⟦13448a423⟧

TextFile

% Copyright (C) 1987 Princeton University

% This file is part of CWEB.
% This program by Silvio Levy is based on a program by D. E. Knuth.
% It is distributed WITHOUT ANY WARRANTY, express or implied.
% $Revision: 1.1 $ -- Dec 1987

% Here is TeX material that gets inserted after \input webmac
\def\hang{\hangindent 3em\indent\ignorespaces}
\font\ninerm=cmr9
\let\mc=\ninerm % medium caps
\def\cee{C}
\def\pb{$\.|\ldots\.|$} % C brackets (|...|)
\def\v{\char'174} % vertical (|) in typewriter font
\def\ceeref{{\it The C Reference Manual}}
\mathchardef\RA="3221 % right arrow
\mathchardef\BA="3224 % double arrow

\def\title{CTANGLE $
  ($Revision: 1.1 $)
  $}
\def\contentspagenumber{1} % should be odd
\def\topofcontents{\null\vfill
  \titlefalse % include headline on the contents page
  \def\rheader{\hfil}
  \centerline{\titlefont The {\ttitlefont CTANGLE} processor}
  \vskip 15pt
  \centerline{$
    ($Revision: 1.1 $)
    $}}
\pageno=\contentspagenumber \advance\pageno by 1
\let\maybe=\iftrue

@* Introduction.
The ``banner line'' defined here should be changed whenever \.{TANGLE}
is modified.

@d banner "This is CTANGLE ($Revision: 1.1 $)\n"

@ \.{TANGLE} has a fairly straightforward outline.  It operates in
two phases: first it reads the source file, saving the \cee\ code in
compressed form; then outputs the code, after shuffling it around.
It can be compiled
with certain optional flags, |DEBUG| and |STAT|, the latter being used
to keep track of how much of \.{TANGLE}'s resources were actually used.

@c
@<Include files@>@/
@<Common code for \.{WEAVE} and \.{TANGLE}@>@/
@<Typedef declarations@>@/
@<Global variables@>@/

main (ac, av)
char **av;
{
  argc=ac; argv=av;
  program=tangle;
  @<Set initial values@>;
  common_init();
  printf(banner); /* print a ``banner line'' */
  phase_one(); /* read all the user's text and compress it into |tok_mem| */
  phase_two(); /* output the contents of the compressed tables */
  wrap_up();
}

@ The following parameters were sufficient in the original \.{TANGLE} to
handle \TeX, so they should be sufficient for most applications of \.{TANGLE}.
If you change |max_bytes|, |max_names| or |hash_size| you should also
change them in the file |"common.web"|.

@d max_bytes 90000 /* the number of bytes in identifiers,
  index entries, and module names; used in |"common.web"| */
@d max_toks 210000 /* number of bytes in compressed \cee\ code */
@d max_names 4000 /* number of identifiers, strings, module names;
  must be less than 10240; used in |"common.web"| */
@d max_texts 2500 /* number of replacement texts, must be less than 10240 */
@d hash_size 353 /* should be prime; used in |"common.web"| */
@d longest_name 400 /* module names shouldn't be longer than this */
@d stack_size 50 /* number of simultaneous levels of macro expansion */
@d buf_size 100 /* for \.{WEAVE} and \.{TANGLE} */

@i common.h

@* Data structures exclusive to {\tt TANGLE}.
We've already seen that the |byte_mem| array holds the names of identifiers,
strings, and modules;
the |tok_mem| array holds the replacement texts
for modules. Allocation is sequential, since things are deleted only
during Phase II, and only in a last-in-first-out manner.

A \&{text} variable is a structure containing a pointer into
|tok_mem|, which tells where the corresponding text starts, and an
integer |text_link|, which, as we shall see later, is used to connect
pieces of text that have the same name.  All the \&{text}s are stored in
the array |text_info|, and we use a |text_pointer| variable to refer
to them.

The first position of |tok_mem| that is unoccupied by
replacement text is called |tok_ptr|, and the first unused location of
|text_info| is called |text_ptr|.  Thus we usually have the identity
|text_ptr->tok_start=tok_ptr|.

If your machine does not support |char unsigned| you should change
the definition of \&{eight\_bits} to |short unsigned|.
@^system dependencies@>

@<Typed...@>=
typedef struct {
  eight_bits *tok_start; /* pointer into |tok_mem| */
  sixteen_bits text_link; /* relates replacement texts */
} text;
typedef text *text_pointer;

@ @<Glob...@>=
text text_info[max_texts];
text_pointer text_info_end=text_info+max_texts-1;
text_pointer text_ptr; /* first unused position in |text_info| */
eight_bits tok_mem[max_toks];
eight_bits *tok_mem_end=tok_mem+max_toks-1;
eight_bits *tok_ptr; /* first unused position in |tok_mem| */

@ @<Set init...@>=
text_info->tok_start=tok_ptr=tok_mem;
text_ptr=text_info+1; text_ptr->tok_start=tok_mem;
  /* this makes replacement text 0 of length zero */

@ If |p| is a pointer to a module name, |p->equiv| is a pointer to its
replacement text, an element of the array |text_info|.

@d equiv equiv_or_xref /* info corresponding to names */

@ @<Set init...@>=
name_dir->equiv=(ASCII *)text_info; /* the undefined module has no replacement text */

@ Here's the procedure that decides whether a name of length |l|
starting at position |first| equals the identifier pointed to by |p|:

@c
names_match(p,first,l)
name_pointer p; /* points to the proposed match */
ASCII *first; /* position of first character of string */
int l; /* length of identifier */
{
  if (length(p)!=l) return 0;
  return !strncmp(first,p->byte_start,l);
}

@ The procedure |init_p| is required so the loader won't complain of its
absence, but it's called only by \.{WEAVE}.

@c
init_node(node)
name_pointer node;
{
    node->equiv=(ASCII *)text_info;
}
init_p() {}

@* Tokens.
Replacement texts, which represent \cee\ code in a compressed format,
appear in |tok_mem| as mentioned above. The codes in
these texts are called `tokens'; some tokens occupy two consecutive
eight-bit byte positions, and the others take just one byte.

If $p$ points to a replacement text, |p->tok_start| is the |tok_mem| position
of the first eight-bit code of that text. If |p->text_link=0|,
this is the replacement text for a macro, otherwise it is the replacement
text for a module. In the latter case |p->text_link| is either equal to
|module_flag|, which means that there is no further text for this module, or
|p->text_link| points to a continuation of this replacement text; such
links are created when several modules have \cee\ texts with the same
name, and they also tie together all the \cee\ texts of unnamed modules.
The replacement text pointer for the first unnamed module appears in
|text_info->text_link|, and the most recent such pointer is |last_unnamed|.

@d module_flag max_texts /* final |text_link| in module replacement texts */

@<Glob...@>=
text_pointer last_unnamed; /* most recent replacement text of unnamed module */

@ @<Set init...@>= last_unnamed=text_info; text_info->text_link=0;

@ If the first byte of a token is less than |0200|, the token occupies a
single byte. Otherwise we make a sixteen-bit token by combining two consecutive
bytes |a| and |b|. If |0200<=a<0250|, then |(a-0200)@t${}\times2^8$@>+b|
to an identifier; if |0250<=a<0320|, then
|(a-0250)@t${}\times2^8$@>+b| points to a module name; otherwise, i.e., if
|0320<=a<0400|, then |(a-0320)@t${}\times2^8$@>+b| is the number of the module
in which the current replacement text appears.

Codes less than 0200 are 7-bit ASCII codes that represent themselves.
In particular, a single-character identifier like `|x|' will be a one-byte
token, while all longer identifiers will occupy two bytes.

Some of the 7-bit ASCII codes will not be present, however, so we can
use them for special purposes. The following symbolic names are used:

\yskip \hang |join| denotes the concatenation of adjacent items with no
space or line breaks allowed between them (the \.{@@\&} operation of \.{WEB}).

\hang |string| denotes the beginning or end of a string, verbatim
construction or numerical constant.
@^ASCII code@>

@d string 02 /* extended ASCII alpha should not appear */
@d join 0177 /* ASCII delete will not appear */

@ The following procedure is used to enter a two-byte value into
|tok_mem| when a replacement text is being generated.

@c store_two_bytes(x)
sixteen_bits x;
{
  if (tok_ptr+2>tok_mem_end) overflow("token");
  *tok_ptr++=x>>8; /* store high byte */
  *tok_ptr++=x&0377; /* store low byte */
}

@* Stacks for output.  The output process uses a stack to keep track
of what is going on at different ``levels'' as the modules are being
written out.  Entries on this stack have five parts:

\yskip\hang |end_field| is the |tok_mem| location where the replacement
text of a particular level will end;

\hang |byte_field| is the |tok_mem| location from which the next token
on a particular level will be read;

\hang |name_field| points to the name corresponding to a particular level;

\hang |repl_field| points to the replacement text currently being read
at a particular level.

\hang |mod_field| is the module number, or zero if this is a macro.

\yskip\noindent The current values of these five quantities are referred to
quite frequently, so they are stored in a separate place instead of in
the |stack| array. We call the current values |cur_end|, |cur_byte|,
|cur_name|, |cur_repl|, and |cur_mod|.

The global variable |stack_ptr| tells how many levels of output are
currently in progress. The end of all output occurs when the stack is
empty, i.e., when |stack_ptr=stack|.

@<Typed...@>=
typedef struct {
  eight_bits *end_field; /* ending location of replacement text */
  eight_bits *byte_field; /* present location within replacement text */
  name_pointer name_field; /* |byte_start| index for text being output */
  text_pointer repl_field; /* |tok_start| index for text being output */
  sixteen_bits mod_field; /* module number or zero if not a module */
} output_state;
typedef output_state *stack_pointer;

@ @d cur_end cur_state.end_field /* current ending location in |tok_mem| */
@d cur_byte cur_state.byte_field /* location of next output byte in |tok_mem|*/
@d cur_name cur_state.name_field /* pointer to current name being expanded */
@d cur_repl cur_state.repl_field /* pointer to current replacement text */
@d cur_mod cur_state.mod_field /* current module number being expanded */

@<Global...@>=
output_state cur_state; /* |cur_end|, |cur_byte|, |cur_name|, |cur_repl|
  and |cur_mod| */
output_state stack[stack_size+1]; /* info for non-current levels */
stack_pointer stack_ptr; /* first unused location in the output state stack */
stack_pointer stack_end=stack+stack_size; /* end of |stack| */

@ To get the output process started, we will perform the following
initialization steps. We may assume that |text_info->text_link| is nonzero,
since it points to the \cee\ text in the first unnamed module that generates
code; if there are no such modules, there is nothing to output, and an
error message will have been generated before we do any of the initialization.

@<Initialize the output stacks@>=
stack_ptr=stack+1; cur_name=name_dir; cur_repl=text_info->text_link+text_info;
cur_byte=cur_repl->tok_start; cur_end=(cur_repl+1)->tok_start; cur_mod=0;

@ When the replacement text for name |p| is to be inserted into the output,
the following subroutine is called to save the old level of output and get
the new one going.

We assume that the C compiler can copy structures.
@^system dependencies@>

@c push_level(p) /* suspends the current level */
name_pointer p;
{
  if (stack_ptr==stack_end) overflow("stack");
  *stack_ptr=cur_state;
  stack_ptr++;
  cur_name=p; cur_repl=(text_pointer)p->equiv;
  cur_byte=cur_repl->tok_start; cur_end=(cur_repl+1)->tok_start;
  cur_mod=0;
}

@ When we come to the end of a replacement text, the |pop_level| subroutine
does the right thing: It either moves to the continuation of this replacement
text or returns the state to the most recently stacked level.

@c pop_level() /* do this when |cur_byte| reaches |cur_end| */
{
  if (cur_repl->text_link<module_flag) { /* link to a continuation */
    cur_repl=cur_repl->text_link+text_info; /* stay on the same level */
    cur_byte=cur_repl->tok_start; cur_end=(cur_repl+1)->tok_start;
    return;
  }
  stack_ptr--; /* go down to the previous level */
  if (stack_ptr>stack) cur_state=*stack_ptr;
}

@ The heart of the output procedure is the |get_output| routine, which produces
the next token of output that is not a reference to a macro. This procedure
handles all the stacking and unstacking that is necessary.
It returns the value |module_number|
if the next output begins or ends the replacement text of some module,
in which case |cur_val| is that module's number (if beginning) or the
negative of that value (if ending). (A module number of 0 indicates
not the beginning or ending of a module, but a \&{\#line} command.)
And it returns the value |identifier|
if the next output is an identifier of length two or more, in which case
|cur_val| points to that identifier name.

@d module_number 0201 /* code returned by |get_output| for module numbers */
@d identifier 0202 /* code returned by |get_output| for identifiers */

@<Global...@>=
int cur_val; /* additional information corresponding to output token */

@ If |get_output| finds that no more output remains, it returns the value zero.

@c sixteen_bits
get_output() /* returns next token after macro expansion */
{
  sixteen_bits a; /* value of current byte */
  restart: if (stack_ptr==stack) return 0;
  if (cur_byte==cur_end) {
    cur_val=-((int)cur_mod); /* cast needed because of sign extension */
    pop_level();
    if (cur_val==0) goto restart;
    out_char(module_number); return;
  }
  a=*cur_byte++;
  if (a<0200) out_char(a); /* one-byte token */
  else {
    a=(a-0200)*0400+*cur_byte++;
    switch (a/024000) { /* |024000=(0250-0200)*0400| */
      case 0: cur_val=a; out_char(identifier); break;
      case 1: @<Expand module |a-024000|, |goto restart|@>;
      default: cur_val=a-050000; if (cur_val>0) cur_mod=cur_val;
	out_char(module_number);
    }
  }
  return 1;
}

@ The user may have forgotten to give any \cee\ text for a module name,
or the \cee\ text may have been associated with a different name by mistake.

@<Expand module |a-...@>=
  a-=024000;
  if ((a+name_dir)->equiv!=(ASCII *)text_info) push_level(a+name_dir);
  else if (a!=0) {
    printf("\n! Not present: <"); print_id(a+name_dir); err_print(">");
@.Not present: <section name>@>
  }
  goto restart;

@* Producing the output.
The |get_output| routine above handles most of the complexity of output
generation, but there is one further consideration that has a nontrivial
effect on \.{TANGLE}'s algorithms.  Namely,
we want to make sure that the output has spaces and line breaks in
the right places (e.g., not in the middle of a string or a constant or an
identifier, not at a `\.{@@\&}' position
where quantities are being joined together, and certainly after a \.=
because the C compiler thinks \.{=-} is ambiguous).

The output process can be in one of following states:

\yskip\hang |num_or_id| means that the last item in the buffer is a number or
identifier, hence a blank space or line break must be inserted if the next
item is also a number or identifier.

\yskip\hang |unbreakable| means that the last item in the buffer was followed
by the \.{@@\&} operation that inhibits spaces between it and the next item.

\yskip\hang |verbatim| means we're copying only character tokens, and
that they are to be output exactly as stored.  This is the case during
strings, verbatim constructions and numerical constants.

\yskip\hang |misc| means none of the above.

\yskip Furthermore, if the variable |protect| is positive, new-lines
are preceded by a `\.\\'.

@d misc 0 /* ``normal'' state */
@d num_or_id 1 /* state associated with numbers and identifiers */
@d unbreakable 3 /* state associated with \.{@@\&} */
@d verbatim 4 /* state in the middle of a string */

@<Global...@>=
eight_bits out_state; /* current status of partial output */
boolean protect; /* current status of partial output */

@ Here is a routine that is invoked when we want to output the current line.
During the output process, |cur_line| equals the number of the next line
to be output.

@c flush_buffer() /* writes one line to output file */
{
  C_putc('\n');
  if (cur_line % 100 == 0) {
    printf(".");
    if (cur_line % 500 == 0) printf("%d",cur_line);
    update_terminal; /* progress report */
  }
  cur_line++;
}

@* The big output switch.  Here then is the routine that does the
output:

@c
phase_two () {
  cur_line=1;
  @<Output macro definitions@>;
  if (text_info->text_link==0) {
    printf("\n! No program text was specified."); mark_harmless;
@.No output was specified@>
  }
  else {
    printf("\nWriting the output file"); update_terminal;
    @<Initialize the output stacks@>;
    while (stack_ptr>stack) get_output();
    flush_buffer();
    printf("\nDone.");
  }
}

@ First we go through the list of replacement texts and copy the ones
that refer to macros, preceded by the |#define| preprocessor command.

@<Output macro def...@>= {
sixteen_bits a;
for (cur_text=text_info+1; cur_text<text_ptr; cur_text++)
  if (cur_text->text_link==0) { /* |cur_text| is the text for a macro */
    cur_byte=cur_text->tok_start;
    cur_end=(cur_text+1)->tok_start;
    C_printf("#define ",0);
    out_state=misc;
    protect=1; /* new-lines should be preceded by |'\\'| */
    while (cur_byte<cur_end) {
      a=*cur_byte++;
      if (cur_byte==cur_end && a=='\n') break; /* disregard a final new-line */
      if (a<0200) out_char(a); /* one-byte token */
      else {
        a=(a-0200)*0400+*cur_byte++;
        if (a<024000) { /* |024000==(0250-0200)*0400| */
          cur_val=a; out_char(identifier);
        }
        else if (a<050000) { confusion("macros defs have strange char");}
	else {
          cur_val=a-050000; cur_mod=cur_val; out_char(module_number);
        }
    /* no other cases */
      }
    }
    protect=0;
    flush_buffer();
  }
}

@ A many-way switch is used to send the output:

@c out_char(cur_char)
eight_bits cur_char;
{
  ASCII *j; /* pointer into |byte_mem| */
    switch (cur_char) {
      case '\n': if (protect) C_putc(' ');
	if (protect || out_state==verbatim) C_putc('\\');
	flush_buffer(); if (out_state!=verbatim) out_state=misc; break;
      @/@t\4@>@<Case of an identifier@>;
      @/@t\4@>@<Case of a module number@>;
      @/@t\4@>@<Cases like \.{!=}@>;
      case '=': C_putc('='); if (out_state!=verbatim) {
	C_putc(' '); out_state=misc;
	}
	break;
      case join: out_state=unbreakable; break; 
      case constant: if (out_state==verbatim) {
	  out_state=num_or_id; break;
	}
        if(out_state==num_or_id) C_putc(' '); out_state=verbatim; break;
      case string: if (out_state==verbatim) out_state=misc;
        else out_state=verbatim; break;
      default: C_putc(cur_char); if (out_state!=verbatim) out_state=misc;
        break;
    }
}

@ @<Cases like \.{!=}@>=
case plus_plus: C_putc('+'); C_putc('+'); out_state=misc; break;
case minus_minus: C_putc('-'); C_putc('-'); out_state=misc; break;
case minus_gt: C_putc('-'); C_putc('>'); out_state=misc; break;
case gt_gt: C_putc('>'); C_putc('>'); out_state=misc; break;
case eq_eq: C_putc('='); C_putc('='); out_state=misc; break;
case lt_lt: C_putc('<'); C_putc('<'); out_state=misc; break;
case gt_eq: C_putc('>'); C_putc('='); out_state=misc; break;
case lt_eq: C_putc('<'); C_putc('='); out_state=misc; break;
case not_eq: C_putc('!'); C_putc('='); out_state=misc; break;
case and_and: C_putc('&'); C_putc('&'); out_state=misc; break;
case or_or: C_putc('|'); C_putc('|'); out_state=misc; break;

@ @<Case of an identifier@>=
case identifier:
  if (out_state==num_or_id) C_putc(' ');
  for (j=(cur_val+name_dir)->byte_start; j<(name_dir+cur_val+1)->byte_start;
    j++) C_putc(*j);
  out_state=num_or_id; break;

@ @<Case of a mod...@>=
case module_number:
  if (cur_val>0) C_printf("/*%d:*/",cur_val);
  else if(cur_val<0) C_printf("/*:%d*/",-cur_val);
  else {
    sixteen_bits a;
    a=0400* *cur_byte++;
    a+=*cur_byte++; /* gets the line number */
    C_printf("\n#line %d \"",a);
    cur_val=*cur_byte++;
    cur_val=0400*(cur_val-0200)+ *cur_byte++; /* points to the file name */
    for (j=(cur_val+name_dir)->byte_start; j<(name_dir+cur_val+1)->byte_start;
      j++) C_putc(*j);
    C_printf("\"\n",0);
  }
  break;

@* Introduction to the input phase.
We have now seen that \.{TANGLE} will be able to output the full
\cee\ program, if we can only get that program into the byte memory in
the proper format. The input process is something like the output process
in reverse, since we compress the text as we read it in and we expand it
as we write it out.

There are three main input routines. The most interesting is the one that gets
the next token of a \cee\ text; the other two are used to scan rapidly past
\TeX\ text in the \.{WEB} source code. One of the latter routines will jump to
the next token that starts with `\.{@@}', and the other skips to the end
of a \cee\ comment.

@ Control codes in \.{WEB} begin with `\.{@@}', and the next character
identifies the code. Some of these are of interest only to \.{WEAVE},
so \.{TANGLE} ignores them; the others are converted by \.{TANGLE} into
internal code numbers by the |control_code| table below. The ordering
of these internal code numbers has been chosen to simplify the program logic;
larger numbers are given to the control codes that denote more significant
milestones.

@d ignore 0 /* control code of no interest to \.{TANGLE} */
@d ascii_constant 0302 /* control code for `\.{@@'}' */
@d control_text 0303 /* control code for `\.{@@t}', `\.{@@\^}', etc. */
@d format 0304 /* control code for `\.{@@f}' */
@d definition 0305 /* control code for `\.{@@d}' */
@d begin_C 0306 /* control code for `\.{@@c}' */
@d module_name 0307 /* control code for `\.{@@<}' */
@d new_module 0310 /* control code for `\.{@@\ }' and `\.{@@*}' */

@<Global...@>=
eight_bits ccode[128]; /* meaning of a char following \.{@@} */

@ @<Set ini...@>= {
  int c; /* must be |int| so the |for| loop will end */
  for (c=0; c<=127; c++) ccode[c]=ignore;
  ccode[' ']=ccode[tab_mark]=ccode['*']=new_module;
  ccode['@@']='@@'; ccode['=']=string;
  ccode['d']=ccode['D']=definition; ccode['f']=ccode['F']=format;
  ccode['c']=ccode['C']=begin_C;
  ccode['^']=ccode[':']=ccode['.']=ccode['t']=ccode['T']=control_text;
  ccode['&']=join; ccode['<']=module_name;
  ccode['\'']=ascii_constant;
}

@ The |skip_ahead| procedure reads through the input at fairly high speed
until finding the next non-ignorable control code, which it returns.

@c eight_bits skip_ahead() /* skip to next control code */
{
  eight_bits c; /* control code found */
  while (1) {
    if (loc>limit && (get_line()==0)) return(new_module);
    *(limit+1)='@@';
    while (*loc!='@@') loc++;
    if (loc<=limit) {
      loc++; c=ccode[*loc]; loc++;
      if (c!=ignore || *(loc-1)=='>') return(c);
    }
  }
}

@ The |skip_comment| procedure reads through the input at somewhat high
speed until finding the end-comment token \.{*/} or a new-line, in which
case |skip_comment| will be called again by |get_next|, since the
comment is not finished.  This is done so that the each newline in the
C part of a module is copied to the output; otherwise the \&{\#line}
commands inserted into the C file by the output routines become useless.
If it comes to the end of the module it prints an error message.

@<Global...@>=
boolean comment_continues=0; /* are we scanning a comment? */

@ @c
skip_comment() /* skips over comments */
{
  ASCII c; /* current character */
  while (1) {
    if (loc>limit)
      if(get_line()) return(comment_continues=1);
      else{
        err_print("! Input ended in mid-comment");
@.Input ended in mid-comment@>
	return(comment_continues=0);
      }
    c=*(loc++);
    if (c=='*' && *loc=='/') { loc++; return(comment_continues=0); }
    if (c=='@@') {
      if (ccode[*loc]==new_module) {
        err_print("! Section name ended in mid-comment"); loc--;
@.Section name ended in mid-comment@>
	return(comment_continues=0);
      }
      else loc++;
    }
  }
}

@* Inputting the next token.

@d constant 03

@<Global...@>=
name_pointer cur_module; /* name of module just scanned */

@ @<Include...@>=
#include "ctype.h" /* definition of |isalpha|, |isdigit| and so on */

@ As one might expect, |get_next| consists mostly of a big switch
that branches to the various special cases that can arise.

@c eight_bits get_next() /* produces the next input token */
{
  static int preprocessing=0;
  eight_bits c; /* the current character */
  while (1) {
    if (loc>limit) {
      if (preprocessing && *(limit-1)!='\\') preprocessing=0;
      if (get_line()==0) return(new_module);
      else if (print_where) {
          print_where=0;
          @<Insert the line number into |tok_mem|@>;
        }
        else return ('\n');
    }
    c=*loc;
    if (comment_continues || (c=='/' && *(loc+1)=='*')) {
      skip_comment(); /* scan to end of comment or newline */
      if (comment_continues) return('\n');
      else continue;
    }
    loc++;
    if (isdigit(c) || c=='\\' || c=='.') @<Get a constant@>@;
    else if (isalpha(c) || c=='_') @<Get an identifier@>@;
    else if (c=='\'' || c=='\"') @<Get a string@>@;
    else if (c=='@@') @<Get control code and possible module name@>@;
    else if (c==' ' || c==tab_mark) {
	if (!preprocessing || loc>limit) continue;
	  /* we don't want a blank after a final backslash */
	else return(' '); /* ignore spaces and tabs, unless preprocessing */
    }
    else if (c=='#' && loc==buffer+1) preprocessing=1;
    mistake: @<Compress two-symbol operator@>@;
    return(c);
  }
}

@ The following code assigns values to the combinations \.{++},
\.{--}, \.{->}, \.{>=}, \.{<=}, \.{==}, \.{<<}, \.{>>}, \.{!=}, \.{||} and
\.{\&\&}.  The compound assignment operators (e.g., \.{+=}) are 
separate tokens, according to \ceeref.

@d compress(c) if (loc++<=limit) return(c)

@<Compress tw...@>=
switch(c) {
  case '+': if (*loc=='+') compress(plus_plus); break;
  case '-': if (*loc=='-') {compress(minus_minus);}
    else if (*loc=='>') compress(minus_gt); break;
  case '=': if (*loc=='=') compress(eq_eq); break;
  case '>': if (*loc=='=') {compress(gt_eq);}
    else if (*loc=='>') compress(gt_gt); break;
  case '<': if (*loc=='=') {compress(lt_eq);}
    else if (*loc=='<') compress(lt_lt); break;
  case '&': if (*loc=='&') compress(and_and); break;
  case '|': if (*loc=='|') compress(or_or); break;
  case '!': if (*loc=='=') compress(not_eq); break;
}

@ @<Get an identifier@>= {
  id_first=--loc;
  while (isalpha(*++loc) || isdigit(*loc) || *loc=='_');
  id_loc=loc; return(identifier);
}

@ @<Get a constant@>= {
  id_first=loc-1;
  if (*id_first=='.' && !isdigit(*loc)) goto mistake; /* not a constant */
  if (*id_first=='\\') while (isdigit(*loc)) loc++; /* octal constant */
  else {
    if (*id_first=='0') {
      if (*loc=='x' || *loc=='X') { /* hex constant */
        loc++; while (isxdigit(*loc)) loc++; goto found;
      }
    }
    while (isdigit(*loc) || *loc=='.') loc++;
    if (*loc=='e' || *loc=='E') { /* float constant */
      if (*++loc=='+' || *loc=='-') loc++;
      while (isdigit(*loc)) loc++;
    }
  }
  found: if (*loc=='l' || *loc=='L') loc++;
  id_loc=loc;
  return(constant);
}

@ \cee\ strings and character constants, delimited by double and single
quotes, respectively, can contain newlines or instances of their own
delimiters if they are protected by a backslash.  We follow this
convention, but do not allow the string to be longer than |longest_name|.

@<Get a string@>= {
  ASCII delim = c; /* what started the string */
  id_first = mod_text+1;
  id_loc = mod_text; *++id_loc=delim;
  while (1) {
    if (loc>=limit) {
      if(*(limit-1)!='\\') {
        err_print("! String didn't end"); loc=limit; break;
@.String didn't end@>
      }
      if(get_line()==0) {
        err_print("! Input ended in middle of string"); loc=buffer; break;
@.Input ended in middle of string@>
      }
      else if (++id_loc<=mod_text_end) *id_loc='\n'; /* will print as
      \.{"\\\\\\n"} */
    }
    if ((c=*loc++)==delim) {
      if (++id_loc<=mod_text_end) *id_loc=c;
      break;
    }
    if (c=='\\') {
      if (loc>=limit) continue;
      if (++id_loc<=mod_text_end) *id_loc = '\\';
      c=*loc++;
    }
    if (++id_loc<=mod_text_end) *id_loc=c;
  }
  if (id_loc>=mod_text_end) {
    printf("\n! String too long: ");
@.String too long@>
    ASCII_write(mod_text+1,25);
    printf("..."); mark_error;
  }
  id_loc++;
  return(string);
}

@ After an \.{@@} sign has been scanned, the next character tells us
whether there is more work to do.

@<Get control code and possible module name@>= {
  c=ccode[*loc++];
  switch(c) {
    case ignore: continue;
    case control_text: while ((c=skip_ahead())=='@@');
      /* only \.{@@@@} and \.{@@>} are expected */
      if (*(loc-1)!='>') err_print("! Improper @@ within control text");
@.Improper {\AT} within control text@>
      continue;
    case module_name: @<Scan the module name and make |cur_module|
      point to it@>;
    case string: @<Scan a verbatim string@>;
    case ascii_constant: @<Scan an ASCII constant@>;
    default: return(c);
  }
}

@ @<Scan an ASCII...@>= 
  id_first=loc;
  if (*loc=='\\') loc++;
  while (*loc!='\'') {
    loc++;
    if (loc>limit) {
        err_print("! String didn't end"); loc=limit-1; break;
    }
  }
  loc++;
  return(ascii_constant);
  	
@ @<Scan the module name...@>= {
  ASCII *k; /* pointer into |mod_text| */
  @<Put module name into |mod_text|@>;
  if (k-mod_text>3 && strncmp(k-2,"...",3)==0) cur_module=prefix_lookup(mod_text+1,k-3);
  else cur_module=mod_lookup(mod_text+1,k);
  return(module_name);
}

@ Module names are placed into the |mod_text| array with consecutive spaces,
tabs, and carriage-returns replaced by single spaces. There will be no
spaces at the beginning or the end. (We set |mod_text[0]=' '| to facilitate
this, since the |mod_lookup| routine uses |mod_text[1]| as the first
character of the name.)

@<Set init...@>=mod_text[0]=' ';

@ @<Put module name...@>=
k=mod_text;
while (1) {
  if (loc>limit && get_line()==0) {
    err_print("! Input ended in section name");
@.Input ended in section name@>
    loc=buffer+1; break;
  }
  c=*loc;
  @<If end of name, |break|@>;
  loc++; if (k<mod_text_end) k++;
  if (c==' ' || c==tab_mark) {
    c=' '; if (*(k-1)==' ') k--;
  }
*k=c;
}
if (k>=mod_text_end) {
  printf("\n! Section name too long: ");
@.Section name too long@>
  ASCII_write(mod_text+1,25);
  printf("..."); mark_harmless;
}
if (*k==' ' && k>mod_text) k--;

@ @<If end of name,...@>=
if (c=='@@') {
  c=*(loc+1);
  if (c=='>') {
    loc+=2; break;
  }
  if (ccode[c]==new_module) {
    err_print("! Section name didn't end"); break;
@.Section name didn't end@>
  }
  *(++k)='@@'; loc++; /* now |c==*loc| again */
}

@ At the present point in the program we
have |*(loc-1)=string|; we set |id_first| to the beginning
of the string itself, and |id_loc| to its ending-plus-one location in the
buffer.  We also set |loc| to the position just after the ending delimiter.

@<Scan a verbatim string@>= {
  id_first=loc++; *(limit+1)='@@'; *(limit+2)='>';
  while (*loc!='@@' || *(loc+1)!='>') loc++;
  if (loc>=limit) err_print("! Verbatim string didn't end");
@.Verbatim string didn't end@>
  id_loc=loc; loc+=2;
  return(string);
}

@* Scanning a macro definition.
The rules for generating the replacement texts corresponding to macros and
\cee\ texts of a module are almost identical; the only differences are that

\yskip \item{a)}Module names are not allowed in macros;
in fact, the appearance of a module name terminates such macros and denotes
the name of the current module.

\item{b)}The symbols \.{@@d} and \.{@@f} and \.{@@c} are not allowed after
module names, while they terminate macro definitions.

\yskip Therefore there is a single procedure |scan_repl| whose parameter
|t| specifies either |macro| or |module_name|. After |scan_repl| has
acted, |cur_text| will point to the replacement text just generated, and
|next_control| will contain the control code that terminated the activity.

@d macro  0
@d app_repl(c)  {if (tok_ptr==tok_mem_end) overflow("token"); *tok_ptr++=c;}

@<Global...@>=
text_pointer cur_text; /* replacement text formed by |scan_repl| */
eight_bits next_control;

@ @c scan_repl(t) /* creates a replacement text */
eight_bits t;
{
  sixteen_bits a; /* the current token */
  if (t==module_name) {@<Insert the line number into |tok_mem|@>;}
  while (1) switch (a=get_next()) {
      @<In cases that |a| is a non-ASCII token (|identifier|,
        |module_name|, etc.), either process it and change |a| to a byte
        that should be stored, or |continue| if |a| should be ignored,
        or |goto done| if |a| signals the end of this replacement text@>@;
      default: app_repl(a); /* store |a| in |tok_mem| */
    }
  done: next_control=(eight_bits) a;
  if (text_ptr>text_info_end) overflow("text");
  cur_text=text_ptr; (++text_ptr)->tok_start=tok_ptr;
}

@ Here is the code for the line number: first a |sixteen_bits| equal
to |0150000|; then, if we're dealing with the change file, the line
number plus |0100000|; or, if we're dealing with the web file, the
line number; or, if we're dealing with an include file, the number 0,
then the line number, followed by the number of characters in the file
name and the file name.

@<Insert the line...@>=
store_two_bytes(0150000);
if (changing) id_first=change_file_name;
else id_first=cur_file_name;
id_loc=id_first+strlen(id_first);
if (changing) store_two_bytes((sixteen_bits)change_line);
else store_two_bytes((sixteen_bits)cur_line);
{int a=id_lookup(id_first,id_loc)-name_dir; app_repl((a / 0400)+0200);
  app_repl(a % 0400);}

@ @<In cases that |a| is...@>=
case identifier: a=id_lookup(id_first,id_loc)-name_dir; app_repl((a / 0400)+0200);
  app_repl(a % 0400); break;
case module_name: if (t!=module_name) goto done;
  else {
    @<Was an `@@' missed here?@>;
    a=cur_module-name_dir;
    app_repl((a / 0400)+0250);
    app_repl(a % 0400);
    @<Insert the line number into |tok_mem|@>; break;
  }
case constant: case string:
  @<Copy a string or verbatim construction or numerical constant@>;
case ascii_constant:
  @<Copy an ASCII constant@>;
case definition: case format: case begin_C: if (t!=module_name) goto done;
  else {
    err_print("! @@d, @@f and @@c are ignored in C text"); continue;
@.{\AT}d, {\AT}f and {\AT}c are ignored in C text@>
  }
case new_module: goto done;

@ @<Was an `@@'...@>= {
  ASCII *try_loc=loc;
  while (*try_loc==' ' && try_loc<limit) try_loc++;
  if (*try_loc=='+' && try_loc<limit) try_loc++;
  while (*try_loc==' ' && try_loc<limit) try_loc++;
  if (*try_loc=='=') err_print ("! Nested named modules.  Missing @@?");
@.Nested named modules@>
}

@ @<Copy a string...@>=
  app_repl(a); /* |string| or |constant| */
  while (id_first < id_loc) { /* simplify \.{@@@@} pairs */
    if (*id_first=='@@') id_first++;
    app_repl(*id_first++);
  }
  app_repl(a); break;

@ @<Copy an ASCII constant@>= {
  int c;
  if (*id_first=='@@') {
    c=xchr[*id_first++];
    if (*id_first!='@@') err_print("! Double @@ within string");
  }
  else if (*id_first=='\\') {
    id_first++;
    switch (*id_first) {
    case 't':c='\t';break;
    case 'n':c='\n';break;
    case 'b':c='\b';break;
    case '0':c='\0';break;
    case '\\':c='\\';break;
    default: err_print("! Unrecognized escape sequence");
    }
  }
  else c=xchr[*id_first];
  app_repl(constant);
  app_repl('0');
  app_repl('0'+(c>>6));
  app_repl('0'+((c-0100*(c>>6))>>3));
  app_repl('0'+(c-010*(c>>3)));
  app_repl(constant);
}
break;
  
@* Scanning a module.
The |scan_module| procedure starts when `\.{@@\ }' or `\.{@@*}' has been
sensed in the input, and it proceeds until the end of that module.  It
uses |module_count| to keep track of the current module number; with luck,
\.{WEAVE} and \.{TANGLE} will both assign the same numbers to modules.

@<Global...@>=
extern sixteen_bits module_count; /* the current module number */

@ The top level of |scan_module| is trivial.
@c scan_module()
{
  name_pointer p; /* module name for the current module */
  text_pointer q; /* text for the current module */
  sixteen_bits a; /* token for left-hand side of definition */
  module_count++;
  if (*(loc-1)=='*') /* starred module */
    printf("*%d",module_count); fflush(stdout);
  @<Scan the definition part of the current module@>;
  @<Scan the \cee\ part of the current module@>;
}

@ @<Scan the definition part...@>=
next_control=0;
while (1) {
  while  (next_control<=format)
  if ((next_control=skip_ahead())==module_name) { /* scan the module name too */
      loc-=2; next_control=get_next();
  }
  if (next_control!=definition) break;
  while ((next_control=get_next())=='\n'); /* allow definition to start on next line */
  if (next_control!=identifier) {
    err_print("! Definition flushed, must start with identifier");
@.Definition flushed...@>
    continue;
  }
  app_repl(((a=id_lookup(id_first,id_loc)-name_dir) / 0400)+0200); /* append the lhs */
  app_repl(a % 0400);
  if (*loc!='(') { /* identifier must be separated from replacement text */
    app_repl(string); app_repl(' '); app_repl(string);
  }
  scan_repl(macro);
  cur_text->text_link=0; /* |text_link=0| characterizes a macro */
}

@ @<Scan the \cee...@>=
switch (next_control) {
  case begin_C: p=name_dir; break;
  case module_name: p=cur_module;
    @<Check that |=| or |==| follows this module name, otherwise |return|@>;
    break;
  default: return;
}
@<Insert the module number into |tok_mem|@>;
scan_repl(module_name); /* now |cur_text| points to the replacement text */
@<Update the data structure so that the replacement text is accessible@>;

@ @<Check that |=|...@>=
while ((next_control=get_next())=='+'); /* allow optional `\.{+=}" */
if (next_control!='=' && next_control!=eq_eq) {
  err_print("! C text flushed, = sign is missing");
@.C text flushed...@>
  while ((next_control=skip_ahead()) != new_module);
  return;
}

@ @<Insert the module number...@>=
store_two_bytes((sixteen_bits)(0150000+module_count));
  /* |0150000==0320*0400| */

@ @<Update the data...@>=
if (p==name_dir||p==0) { /* unnamed module, or bad module name */
  (last_unnamed)->text_link=cur_text-text_info; last_unnamed=cur_text;
}
else if (p->equiv==(ASCII *)text_info) p->equiv=(ASCII *)cur_text;
  /* first module of this name */
else {
  q=(text_pointer)p->equiv;
  while (q->text_link<module_flag) q=q->text_link+text_info; /* find end of list */
  q->text_link=cur_text-text_info;
}
cur_text->text_link=module_flag; /* mark this replacement text as a nonmacro */

@ @c phase_one() {
  phase=1;
  module_count=0;
  reset_input();
  while ((next_control=skip_ahead())!=new_module);
  while (!input_has_ended) scan_module();
  check_complete();
  phase=2;
}

@ @c print_stats() {
  printf("\nMemory usage statistics:\n");
  printf("%d names (out of %d)\n",name_ptr-name_dir,max_names);
  printf("%d replacement texts (out of %d)\n",text_ptr-text_info,max_texts);
  printf("%d bytes (out of %d)\n",byte_ptr-byte_mem,max_bytes);
  printf("%d tokens (out of %d)\n",tok_ptr-tok_mem,max_toks);
}

@* Index.
Here is a cross-reference table for the \.{TANGLE} processor.
All modules in which an identifier is
used are listed with that identifier, except that reserved words are
indexed only when they appear in format definitions, and the appearances
of identifiers in module names are not indexed. Underlined entries
correspond to where the identifier was declared. Error messages and
a few other things like ``ASCII code'' are indexed here too.
DataMuseum.dk

DKUUG/EUUG Conference tapes

⟦13448a423⟧ TextFile

Derivation

TextFile
