parser-scanner.c

     
   1  //! @file parser-scanner.c
   2  //! @author J. Marcel van der Veer
   3  
   4  //! @section Copyright
   5  //!
   6  //! This file is part of Algol68G - an Algol 68 compiler-interpreter.
   7  //! Copyright 2001-2024 J. Marcel van der Veer [algol68g@xs4all.nl].
   8  
   9  //! @section License
  10  //!
  11  //! This program is free software; you can redistribute it and/or modify it 
  12  //! under the terms of the GNU General Public License as published by the 
  13  //! Free Software Foundation; either version 3 of the License, or 
  14  //! (at your option) any later version.
  15  //!
  16  //! This program is distributed in the hope that it will be useful, but 
  17  //! WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 
  18  //! or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 
  19  //! more details. You should have received a copy of the GNU General Public 
  20  //! License along with this program. If not, see [http://www.gnu.org/licenses/].
  21  
  22  //! @section Synopsis 
  23  //!
  24  //! Context-dependent Algol 68 tokeniser.
  25  
  26  #include "a68g.h"
  27  #include "a68g-parser.h"
  28  #include "a68g-prelude.h"
  29  #include "a68g-options.h"
  30  #include "a68g-environ.h"
  31  
  32  // Macros.
  33  
  34  #define SCAN_DIGITS(c)\
  35    while (IS_DIGIT (c)) {\
  36      (sym++)[0] = (c);\
  37      (c) = next_char (ref_l, ref_s, A68_TRUE);\
  38    }
  39  
  40  #define SCAN_EXPONENT_PART(c)\
  41    (sym++)[0] = EXPONENT_CHAR;\
  42    (c) = next_char (ref_l, ref_s, A68_TRUE);\
  43    if ((c) == '+' || (c) == '-') {\
  44      (sym++)[0] = (c);\
  45      (c) = next_char (ref_l, ref_s, A68_TRUE);\
  46    }\
  47    SCAN_ERROR (!IS_DIGIT (c), *start_l, *start_c, ERROR_EXPONENT_DIGIT);\
  48    SCAN_DIGITS (c)
  49  
  50  //! @brief Save scanner state, for character look-ahead.
  51  
  52  void save_state (LINE_T * ref_l, char *ref_s, char ch)
  53  {
  54    SCAN_STATE_L (&A68_JOB) = ref_l;
  55    SCAN_STATE_S (&A68_JOB) = ref_s;
  56    SCAN_STATE_C (&A68_JOB) = ch;
  57  }
  58  
  59  //! @brief Restore scanner state, for character look-ahead.
  60  
  61  void restore_state (LINE_T ** ref_l, char **ref_s, char *ch)
  62  {
  63    *ref_l = SCAN_STATE_L (&A68_JOB);
  64    *ref_s = SCAN_STATE_S (&A68_JOB);
  65    *ch = SCAN_STATE_C (&A68_JOB);
  66  }
  67  
  68  //! @brief New_source_line.
  69  
  70  LINE_T *new_source_line (void)
  71  {
  72    LINE_T *z = (LINE_T *) get_fixed_heap_space ((size_t) SIZE_ALIGNED (LINE_T));
  73    MARKER (z)[0] = NULL_CHAR;
  74    STRING (z) = NO_TEXT;
  75    FILENAME (z) = NO_TEXT;
  76    DIAGNOSTICS (z) = NO_DIAGNOSTIC;
  77    NUMBER (z) = 0;
  78    PRINT_STATUS (z) = 0;
  79    LIST (z) = A68_TRUE;
  80    NEXT (z) = NO_LINE;
  81    PREVIOUS (z) = NO_LINE;
  82    return z;
  83  }
  84  
  85  //! @brief Append a source line to the internal source file.
  86  
  87  void append_source_line (char *str, LINE_T ** ref_l, int *line_num, char *filename)
  88  {
  89    LINE_T *z = new_source_line ();
  90  // Allow shell command in first line, f.i. "#!/usr/share/bin/a68g".
  91    if (*line_num == 1) {
  92      if (strlen (str) >= 2 && strncmp (str, "#!", strlen ("#!")) == 0) {
  93        ABEND (strstr (str, "run-script") != NO_TEXT, ERROR_SHELL_SCRIPT, __func__);
  94        (*line_num)++;
  95        return;
  96      }
  97    }
  98  // Link line into the chain.
  99    STRING (z) = new_fixed_string (str);
 100    FILENAME (z) = filename;
 101    NUMBER (z) = (*line_num)++;
 102    PRINT_STATUS (z) = NOT_PRINTED;
 103    LIST (z) = A68_TRUE;
 104    DIAGNOSTICS (z) = NO_DIAGNOSTIC;
 105    NEXT (z) = NO_LINE;
 106    PREVIOUS (z) = *ref_l;
 107    if (TOP_LINE (&A68_JOB) == NO_LINE) {
 108      TOP_LINE (&A68_JOB) = z;
 109    }
 110    if (*ref_l != NO_LINE) {
 111      NEXT (*ref_l) = z;
 112    }
 113    *ref_l = z;
 114  }
 115  
 116  // Scanner, tokenises the source code.
 117  
 118  //! @brief Whether ch is unworthy.
 119  
 120  void unworthy (LINE_T * u, char *v, char ch)
 121  {
 122    if (IS_PRINT (ch)) {
 123      ASSERT (a68_bufprt (A68 (edit_line), SNPRINTF_SIZE, "*%s", ERROR_UNWORTHY_CHARACTER) >= 0);
 124    } else {
 125      ASSERT (a68_bufprt (A68 (edit_line), SNPRINTF_SIZE, "*%s %s", ERROR_UNWORTHY_CHARACTER, ctrl_char (ch)) >= 0);
 126    }
 127    scan_error (u, v, A68 (edit_line));
 128  }
 129  
 130  //! @brief Concatenate lines that terminate in '\' with next line.
 131  
 132  void concatenate_lines (LINE_T * top)
 133  {
 134    LINE_T *q;
 135  // Work from bottom backwards.
 136    for (q = top; q != NO_LINE && NEXT (q) != NO_LINE; q = NEXT (q)) {
 137      ;
 138    }
 139    for (; q != NO_LINE; BACKWARD (q)) {
 140      char *z = STRING (q);
 141      int len = (int) strlen (z);
 142      if (len >= 2 && z[len - 2] == BACKSLASH_CHAR && z[len - 1] == NEWLINE_CHAR && NEXT (q) != NO_LINE && STRING (NEXT (q)) != NO_TEXT) {
 143        z[len - 2] = NULL_CHAR;
 144        len += (int) strlen (STRING (NEXT (q)));
 145        z = (char *) get_fixed_heap_space ((size_t) (len + 1));
 146        a68_bufcpy (z, STRING (q), len + 1);
 147        a68_bufcat (z, STRING (NEXT (q)), len + 1);
 148        STRING (NEXT (q))[0] = NULL_CHAR;
 149        STRING (q) = z;
 150      }
 151    }
 152  }
 153  
 154  //! @brief Whether u is bold tag v, independent of stropping regime.
 155  
 156  BOOL_T is_bold (char *u, char *v)
 157  {
 158    unt len = (unt) strlen (v);
 159    if (OPTION_STROPPING (&A68_JOB) == QUOTE_STROPPING) {
 160      if (u[0] == '\'') {
 161        return (BOOL_T) (strncmp (++u, v, len) == 0 && u[len] == '\'');
 162      } else {
 163        return A68_FALSE;
 164      }
 165    } else {
 166      return (BOOL_T) (strncmp (u, v, len) == 0 && !IS_UPPER (u[len]));
 167    }
 168  }
 169  
 170  //! @brief Skip string.
 171  
 172  BOOL_T skip_string (LINE_T ** top, char **ch)
 173  {
 174    LINE_T *u = *top;
 175    char *v = *ch;
 176    v++;
 177    while (u != NO_LINE) {
 178      while (v[0] != NULL_CHAR) {
 179        if (v[0] == QUOTE_CHAR && v[1] != QUOTE_CHAR) {
 180          *top = u;
 181          *ch = &v[1];
 182          return A68_TRUE;
 183        } else if (v[0] == QUOTE_CHAR && v[1] == QUOTE_CHAR) {
 184          v += 2;
 185        } else {
 186          v++;
 187        }
 188      }
 189      FORWARD (u);
 190      if (u != NO_LINE) {
 191        v = &(STRING (u)[0]);
 192      } else {
 193        v = NO_TEXT;
 194      }
 195    }
 196    return A68_FALSE;
 197  }
 198  
 199  //! @brief Skip comment.
 200  
 201  BOOL_T skip_comment (LINE_T ** top, char **ch, int delim)
 202  {
 203    LINE_T *u = *top;
 204    char *v = *ch;
 205    BOOL_T qstrop = OPTION_STROPPING (&A68_JOB) == QUOTE_STROPPING;
 206    v++;
 207    while (u != NO_LINE) {
 208      while (v[0] != NULL_CHAR) {
 209        if (is_bold (v, "COMMENT") && delim == BOLD_COMMENT_SYMBOL) {
 210          char *w = &v[strlen(qstrop ? "'COMMENT'" : "COMMENT")];
 211          if (!IS_UPPER(w[0])) {
 212            *top = u;
 213            *ch = w;
 214            return A68_TRUE;
 215          }
 216        } else if (is_bold (v, "CO") && delim == STYLE_I_COMMENT_SYMBOL) {
 217          char *w = &v[strlen(qstrop ? "'CO'" : "CO")];
 218          if (!IS_UPPER(w[0])) {
 219            *top = u;
 220            *ch = w;
 221            return A68_TRUE;
 222          }
 223        } else if (v[0] == '#' && delim == STYLE_II_COMMENT_SYMBOL) {
 224          *top = u;
 225          *ch = &v[1];
 226          return A68_TRUE;
 227        } else {
 228          v++;
 229        }
 230      }
 231      FORWARD (u);
 232      if (u != NO_LINE) {
 233        v = &(STRING (u)[0]);
 234      } else {
 235        v = NO_TEXT;
 236      }
 237    }
 238    return A68_FALSE;
 239  }
 240  
 241  //! @brief Skip rest of pragmat.
 242  
 243  BOOL_T skip_pragmat (LINE_T ** top, char **ch, int delim, BOOL_T whitespace)
 244  {
 245    LINE_T *u = *top;
 246    char *v = *ch;
 247    BOOL_T qstrop = OPTION_STROPPING (&A68_JOB) == QUOTE_STROPPING;
 248    while (u != NO_LINE) {
 249      while (v[0] != NULL_CHAR) {
 250        if (is_bold (v, "PRAGMAT") && delim == BOLD_PRAGMAT_SYMBOL) {
 251          char *w = &v[strlen(qstrop ? "'PRAGMAT'" : "PRAGMAT")];
 252          if (!IS_UPPER(w[0])) {
 253            *top = u;
 254            *ch = w;
 255            return A68_TRUE;
 256          }
 257        } else if (is_bold (v, "PR") && delim == STYLE_I_PRAGMAT_SYMBOL) {
 258          char *w = &v[strlen(qstrop ? "'PR'" : "PR")];
 259          if (!IS_UPPER(w[0])) {
 260            *top = u;
 261            *ch = w;
 262            return A68_TRUE;
 263          }
 264        } else {
 265          if (whitespace && !IS_SPACE (v[0]) && v[0] != NEWLINE_CHAR) {
 266            scan_error (u, v, ERROR_PRAGMENT);
 267          } else if (IS_UPPER (v[0])) {
 268  // Skip a bold word as you may trigger on REPR, for instance ...
 269            while (IS_UPPER (v[0])) {
 270              v++;
 271            }
 272          } else {
 273            v++;
 274          }
 275        }
 276      }
 277      FORWARD (u);
 278      if (u != NO_LINE) {
 279        v = &(STRING (u)[0]);
 280      } else {
 281        v = NO_TEXT;
 282      }
 283    }
 284    return A68_FALSE;
 285  }
 286  
 287  //! @brief Return pointer to next token within pragmat.
 288  
 289  char *get_pragmat_item (LINE_T ** top, char **ch)
 290  {
 291    LINE_T *u = *top;
 292    char *v = *ch;
 293    while (u != NO_LINE) {
 294      while (v[0] != NULL_CHAR) {
 295        if (!IS_SPACE (v[0]) && v[0] != NEWLINE_CHAR) {
 296          *top = u;
 297          *ch = v;
 298          return v;
 299        } else {
 300          v++;
 301        }
 302      }
 303      FORWARD (u);
 304      if (u != NO_LINE) {
 305        v = &(STRING (u)[0]);
 306      } else {
 307        v = NO_TEXT;
 308      }
 309    }
 310    return NO_TEXT;
 311  }
 312  
 313  //! @brief Case insensitive strncmp for at most the number of chars in 'v'.
 314  
 315  int streq (char *u, char *v)
 316  {
 317    int diff;
 318    for (diff = 0; diff == 0 && u[0] != NULL_CHAR && v[0] != NULL_CHAR; u++, v++) {
 319      diff = ((int) TO_LOWER (u[0])) - ((int) TO_LOWER (v[0]));
 320    }
 321    return diff;
 322  }
 323  
 324  //! @brief Scan for next pragmat and yield first pragmat item.
 325  
 326  char *next_preprocessor_item (LINE_T ** top, char **ch, int *delim)
 327  {
 328    LINE_T *u = *top;
 329    char *v = *ch;
 330    *delim = 0;
 331    while (u != NO_LINE) {
 332      while (v[0] != NULL_CHAR) {
 333        LINE_T *start_l = u;
 334        char *start_c = v;
 335  // STRINGs must be skipped.
 336        if (v[0] == QUOTE_CHAR) {
 337          SCAN_ERROR (!skip_string (&u, &v), start_l, start_c, ERROR_UNTERMINATED_STRING);
 338        }
 339  // COMMENTS must be skipped.
 340        else if (is_bold (v, "COMMENT")) {
 341          SCAN_ERROR (!skip_comment (&u, &v, BOLD_COMMENT_SYMBOL), start_l, start_c, ERROR_UNTERMINATED_COMMENT);
 342        } else if (is_bold (v, "CO")) {
 343          SCAN_ERROR (!skip_comment (&u, &v, STYLE_I_COMMENT_SYMBOL), start_l, start_c, ERROR_UNTERMINATED_COMMENT);
 344        } else if (v[0] == '#') {
 345          SCAN_ERROR (!skip_comment (&u, &v, STYLE_II_COMMENT_SYMBOL), start_l, start_c, ERROR_UNTERMINATED_COMMENT);
 346        } else if (is_bold (v, "PRAGMAT") || is_bold (v, "PR")) {
 347  // We caught a PRAGMAT.
 348          char *item;
 349          if (is_bold (v, "PRAGMAT")) {
 350            *delim = BOLD_PRAGMAT_SYMBOL;
 351            v = &v[strlen ("PRAGMAT")];
 352          } else if (is_bold (v, "PR")) {
 353            *delim = STYLE_I_PRAGMAT_SYMBOL;
 354            v = &v[strlen ("PR")];
 355          }
 356          item = get_pragmat_item (&u, &v);
 357          SCAN_ERROR (item == NO_TEXT, start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
 358  // Item "preprocessor" restarts preprocessing if it is off.
 359          if (A68_PARSER (no_preprocessing) && streq (item, "PREPROCESSOR") == 0) {
 360            A68_PARSER (no_preprocessing) = A68_FALSE;
 361            SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68_TRUE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
 362          }
 363  // If preprocessing is switched off, we idle to closing bracket.
 364          else if (A68_PARSER (no_preprocessing)) {
 365            SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68_FALSE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
 366          }
 367  // Item "nopreprocessor" stops preprocessing if it is on.
 368          if (streq (item, "NOPREPROCESSOR") == 0) {
 369            A68_PARSER (no_preprocessing) = A68_TRUE;
 370            SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68_TRUE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
 371          }
 372  // Item "INCLUDE" includes a file.
 373          else if (streq (item, "INCLUDE") == 0) {
 374            *top = u;
 375            *ch = v;
 376            return item;
 377          }
 378  // Item "READ" includes a file.
 379          else if (streq (item, "READ") == 0) {
 380            *top = u;
 381            *ch = v;
 382            return item;
 383          }
 384  // Unrecognised item - probably options handled later by the tokeniser.
 385          else {
 386            SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68_FALSE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
 387          }
 388        } else if (IS_UPPER (v[0])) {
 389  // Skip a bold word as you may trigger on REPR, for instance ...
 390          while (IS_UPPER (v[0])) {
 391            v++;
 392          }
 393        } else {
 394          v++;
 395        }
 396      }
 397      FORWARD (u);
 398      if (u != NO_LINE) {
 399        v = &(STRING (u)[0]);
 400      } else {
 401        v = NO_TEXT;
 402      }
 403    }
 404    *top = u;
 405    *ch = v;
 406    return NO_TEXT;
 407  }
 408  
 409  //! @brief Include files.
 410  
 411  void include_files (LINE_T * top)
 412  {
 413  // include_files
 414  // 
 415  // syntax: PR read "filename" PR
 416  //         PR include "filename" PR
 417  // 
 418  // The file gets inserted before the line containing the pragmat. In this way
 419  // correct line numbers are preserved which helps diagnostics. A file that has
 420  // been included will not be included a second time - it will be ignored. 
 421  // A rigorous fail-safe, but there is no mechanism to prevent recursive includes 
 422  // in A68 source code. User reports do not indicate sophisticated use of INCLUDE, 
 423  // so this is fine for now.
 424  // TODO - some day we might need `app', analogous to `cpp'.
 425    BOOL_T make_pass = A68_TRUE;
 426    while (make_pass) {
 427      LINE_T *s, *t, *u = top;
 428      char *v = &(STRING (u)[0]);
 429      make_pass = A68_FALSE;
 430      errno = 0;
 431      while (u != NO_LINE) {
 432        int pr_lim;
 433        char *item = next_preprocessor_item (&u, &v, &pr_lim);
 434        LINE_T *start_l = u;
 435        char *start_c = v;
 436  // Search for PR include "filename" PR.
 437        if (item != NO_TEXT && (streq (item, "INCLUDE") == 0 || streq (item, "READ") == 0)) {
 438          FILE_T fd;
 439          int n, linum, fsize, k, bytes_read, fnwid;
 440          char *fbuf, delim;
 441          BUFFER fnb;
 442          char *fn;
 443  // Skip to filename.
 444          while (IS_ALPHA (v[0])) {
 445            v++;
 446          }
 447          while (IS_SPACE (v[0])) {
 448            v++;
 449          }
 450  // Scan quoted filename.
 451          SCAN_ERROR ((v[0] != QUOTE_CHAR && v[0] != '\''), start_l, start_c, ERROR_INCORRECT_FILENAME);
 452          delim = (v++)[0];
 453          n = 0;
 454          fnb[0] = NULL_CHAR;
 455  // Scan Algol 68 string (note: "" denotes a ", while in C it concatenates).
 456          do {
 457            SCAN_ERROR (EOL (v[0]), start_l, start_c, ERROR_INCORRECT_FILENAME);
 458            SCAN_ERROR (n == BUFFER_SIZE - 1, start_l, start_c, ERROR_INCORRECT_FILENAME);
 459            if (v[0] == delim) {
 460              while (v[0] == delim && v[1] == delim) {
 461                SCAN_ERROR (n == BUFFER_SIZE - 1, start_l, start_c, ERROR_INCORRECT_FILENAME);
 462                fnb[n++] = delim;
 463                fnb[n] = NULL_CHAR;
 464                v += 2;
 465              }
 466            } else if (IS_PRINT (v[0])) {
 467              fnb[n++] = *(v++);
 468              fnb[n] = NULL_CHAR;
 469            } else {
 470              SCAN_ERROR (A68_TRUE, start_l, start_c, ERROR_INCORRECT_FILENAME);
 471            }
 472          } while (v[0] != delim);
 473  // Insist that the pragmat is closed properly.
 474          v = &v[1];
 475          SCAN_ERROR (!skip_pragmat (&u, &v, pr_lim, A68_TRUE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
 476          SCAN_ERROR (n == 0, start_l, start_c, ERROR_INCORRECT_FILENAME);
 477  // Make the name relative to the position of the source file (C preprocessor standard).
 478          if (FILENAME (u) != NO_TEXT) {
 479            fn = a68_relpath (a68_dirname (FILENAME (u)), a68_dirname (fnb), a68_basename (fnb));
 480          } else {
 481            fn = a68_relpath (FILE_PATH (&A68_JOB), a68_dirname (fnb), a68_basename (fnb));
 482          }
 483  // Do not check errno, since errno may be undefined here after a successful call.
 484          if (fn != NO_TEXT) {
 485            a68_bufcpy (fnb, fn, BUFFER_SIZE);
 486          } else {
 487            SCAN_ERROR (A68_TRUE, NO_LINE, NO_TEXT, ERROR_SOURCE_FILE_INCLUDE_OPEN);
 488          }
 489          fnwid = (int) strlen (fnb) + 1;
 490          fn = (char *) get_fixed_heap_space ((size_t) fnwid);
 491          a68_bufcpy (fn, fnb, fnwid);
 492  // Ignore the file when included more than once.
 493          for (t = top; t != NO_LINE; t = NEXT (t)) {
 494            if (strcmp (FILENAME (t), fn) == 0) {
 495              goto search_next_pragmat;
 496            }
 497          }
 498  // Access the file.
 499          errno = 0;
 500          fd = open (fn, O_RDONLY | O_BINARY);
 501          SCAN_ERROR (fd == -1, start_l, start_c, ERROR_SOURCE_FILE_INCLUDE_OPEN);
 502          errno = 0;
 503          fsize = (int) lseek (fd, 0, SEEK_END);
 504          ASSERT (fsize >= 0);
 505          SCAN_ERROR (errno != 0, start_l, start_c, ERROR_FILE_READ);
 506          fbuf = (char *) get_temp_heap_space ((unt) (8 + fsize));
 507          errno = 0;
 508          ASSERT (lseek (fd, 0, SEEK_SET) >= 0);
 509          SCAN_ERROR (errno != 0, start_l, start_c, ERROR_FILE_READ);
 510          errno = 0;
 511          bytes_read = (int) io_read (fd, fbuf, (size_t) fsize);
 512          SCAN_ERROR (errno != 0 || bytes_read != fsize, start_l, start_c, ERROR_FILE_READ);
 513  // Buffer still usable?.
 514          if (fsize > A68_PARSER (max_scan_buf_length)) {
 515            A68_PARSER (max_scan_buf_length) = fsize;
 516            A68_PARSER (scan_buf) = (char *) get_temp_heap_space ((unt) (8 + A68_PARSER (max_scan_buf_length)));
 517          }
 518  // Link all lines into the list.
 519          linum = 1;
 520          s = u;
 521          t = PREVIOUS (u);
 522          k = 0;
 523          if (fsize == 0) {
 524  // If file is empty, insert single empty line.
 525            A68_PARSER (scan_buf)[0] = NEWLINE_CHAR;
 526            A68_PARSER (scan_buf)[1] = NULL_CHAR;
 527            append_source_line (A68_PARSER (scan_buf), &t, &linum, fn);
 528          } else
 529            while (k < fsize) {
 530              n = 0;
 531              A68_PARSER (scan_buf)[0] = NULL_CHAR;
 532              while (k < fsize && fbuf[k] != NEWLINE_CHAR) {
 533                SCAN_ERROR ((IS_CNTRL (fbuf[k]) && !IS_SPACE (fbuf[k])) || fbuf[k] == STOP_CHAR, start_l, start_c, ERROR_FILE_INCLUDE_CTRL);
 534                A68_PARSER (scan_buf)[n++] = fbuf[k++];
 535                A68_PARSER (scan_buf)[n] = NULL_CHAR;
 536              }
 537              A68_PARSER (scan_buf)[n++] = NEWLINE_CHAR;
 538              A68_PARSER (scan_buf)[n] = NULL_CHAR;
 539              if (k < fsize) {
 540                k++;
 541              }
 542              append_source_line (A68_PARSER (scan_buf), &t, &linum, fn);
 543            }
 544  // Conclude and go find another include directive, if any.
 545          NEXT (t) = s;
 546          PREVIOUS (s) = t;
 547          concatenate_lines (top);
 548          ASSERT (close (fd) == 0);
 549          make_pass = A68_TRUE;
 550        }
 551      search_next_pragmat:_SKIP_;
 552      }
 553    }
 554  }
 555  
 556  //! @brief Size of source file.
 557  
 558  int get_source_size (void)
 559  {
 560    FILE_T f = FILE_SOURCE_FD (&A68_JOB);
 561  // This is why WIN32 must open as "read binary".
 562    return (int) lseek (f, 0, SEEK_END);
 563  }
 564  
 565  //! @brief Append environment source lines.
 566  
 567  void append_environ (char *str[], LINE_T ** ref_l, int *line_num, char *name)
 568  {
 569    for (int k = 0; str[k] != NO_TEXT; k++) {
 570      int zero_line_num = 0;
 571      (*line_num)++;
 572      append_source_line (str[k], ref_l, &zero_line_num, name);
 573    }
 574  }
 575  
 576  //! @brief Read script file and make internal copy.
 577  
 578  BOOL_T read_script_file (void)
 579  {
 580    LINE_T *ref_l = NO_LINE;
 581    int k, n, num;
 582    unt len;
 583    BOOL_T file_end = A68_FALSE;
 584    BUFFER filename, linenum;
 585    char ch, *fn, *line;
 586    char *buffer = (char *) get_temp_heap_space ((unt) (8 + A68_PARSER (source_file_size)));
 587    FILE_T source = FILE_SOURCE_FD (&A68_JOB);
 588    ABEND (source == -1, ERROR_ACTION, __func__);
 589    buffer[0] = NULL_CHAR;
 590    n = 0;
 591    len = (unt) (8 + A68_PARSER (source_file_size));
 592    buffer = (char *) get_temp_heap_space (len);
 593    ASSERT (lseek (source, 0, SEEK_SET) >= 0);
 594    while (!file_end) {
 595  // Read the original file name.
 596      filename[0] = NULL_CHAR;
 597      k = 0;
 598      if (io_read (source, &ch, 1) == 0) {
 599        file_end = A68_TRUE;
 600        continue;
 601      }
 602      while (ch != NEWLINE_CHAR) {
 603        filename[k++] = ch;
 604        ASSERT (io_read (source, &ch, 1) == 1);
 605      }
 606      filename[k] = NULL_CHAR;
 607      fn = TEXT (add_token (&A68 (top_token), filename));
 608  // Read the original file number.
 609      linenum[0] = NULL_CHAR;
 610      k = 0;
 611      ASSERT (io_read (source, &ch, 1) == 1);
 612      while (ch != NEWLINE_CHAR) {
 613        linenum[k++] = ch;
 614        ASSERT (io_read (source, &ch, 1) == 1);
 615      }
 616      linenum[k] = NULL_CHAR;
 617      num = (int) strtol (linenum, NO_VAR, 10);
 618      ABEND (errno == ERANGE, ERROR_INTERNAL_CONSISTENCY, __func__);
 619  // COPY original line into buffer.
 620      ASSERT (io_read (source, &ch, 1) == 1);
 621      line = &buffer[n];
 622      while (ch != NEWLINE_CHAR) {
 623        buffer[n++] = ch;
 624        ASSERT (io_read (source, &ch, 1) == 1);
 625        ABEND ((unt) n >= len, ERROR_ACTION, __func__);
 626      }
 627      buffer[n++] = NEWLINE_CHAR;
 628      buffer[n] = NULL_CHAR;
 629      append_source_line (line, &ref_l, &num, fn);
 630    }
 631    return A68_TRUE;
 632  }
 633  
 634  //! @brief match first non-white characters in string.
 635  
 636  BOOL_T a68_start(char *u, char *v, char **end)
 637  {
 638    *end = NO_TEXT;
 639    while (v[0] != NULL_CHAR) {
 640      if (u[0] == NULL_CHAR) {
 641        return A68_FALSE;
 642      } else if (IS_SPACE (u[0])) {
 643        u++;
 644      } else {
 645        if (u[0] == v[0]) {
 646          u++;
 647          v++;
 648          if (end != NULL) {
 649            *end = u;
 650          }
 651        } else {
 652          return A68_FALSE;
 653        }
 654      }
 655    }
 656    return A68_TRUE;
 657  }
 658  
 659  //! @brief Read source file and make internal copy.
 660  
 661  BOOL_T read_source_file (void)
 662  {
 663    LINE_T *ref_l = NO_LINE;
 664    int line_num = 0, k, bytes_read;
 665    ssize_t l;
 666    FILE_T f = FILE_SOURCE_FD (&A68_JOB);
 667    char **prelude_start, **postlude, *buffer, *text;
 668  // Read the file into a single buffer, so we save on system calls.
 669    line_num = 1;
 670    errno = 0;
 671    text = (char *) get_temp_heap_space ((unt) (8 + A68_PARSER (source_file_size)));
 672    ABEND (errno != 0 || text == NO_TEXT, ERROR_ALLOCATION, __func__);
 673    ASSERT (lseek (f, 0, SEEK_SET) >= 0);
 674    ABEND (errno != 0, ERROR_ACTION, __func__);
 675    errno = 0;
 676    bytes_read = (int) io_read (f, text, (size_t) A68_PARSER (source_file_size));
 677    ABEND (errno != 0 || bytes_read != A68_PARSER (source_file_size), ERROR_ACTION, __func__);
 678  // Little test on stropping.
 679    char *pr1 = "'PR'QUOTESTROPPING'PR'";
 680    char *pr2 = "'PRAGMAT'QUOTESTROPPING'PRAGMAT'";
 681    char *end = NO_TEXT;
 682    if (a68_start (text, pr1, &end)) {
 683      OPTION_STROPPING (&A68_JOB) = QUOTE_STROPPING;
 684      buffer = end;
 685      A68_PARSER (source_file_size) = strlen (buffer);
 686    } else if (a68_start (text, pr2, &end)) {
 687      OPTION_STROPPING (&A68_JOB) = QUOTE_STROPPING;
 688      buffer = end;
 689      A68_PARSER (source_file_size) = strlen (buffer);
 690    } else {
 691      buffer = text;
 692    }
 693  // Prelude.
 694    if (OPTION_STROPPING (&A68_JOB) == UPPER_STROPPING) {
 695      prelude_start = bold_prelude_start;
 696      postlude = bold_postlude;
 697    } else if (OPTION_STROPPING (&A68_JOB) == QUOTE_STROPPING) {
 698      prelude_start = quote_prelude_start;
 699      postlude = quote_postlude;
 700    } else {
 701      prelude_start = postlude = NO_VAR;
 702    }
 703    append_environ (prelude_start, &ref_l, &line_num, "prelude");
 704  // Link all lines into the list.
 705    k = 0;
 706    while (k < A68_PARSER (source_file_size)) {
 707      l = 0;
 708      A68_PARSER (scan_buf)[0] = NULL_CHAR;
 709      while (k < A68_PARSER (source_file_size) && buffer[k] != NEWLINE_CHAR) {
 710        if (k < A68_PARSER (source_file_size) - 1 && buffer[k] == CR_CHAR && buffer[k + 1] == NEWLINE_CHAR) {
 711          k++;
 712        } else {
 713          A68_PARSER (scan_buf)[l++] = buffer[k++];
 714          A68_PARSER (scan_buf)[l] = NULL_CHAR;
 715        }
 716      }
 717      A68_PARSER (scan_buf)[l++] = NEWLINE_CHAR;
 718      A68_PARSER (scan_buf)[l] = NULL_CHAR;
 719      if (k < A68_PARSER (source_file_size)) {
 720        k++;
 721      }
 722      append_source_line (A68_PARSER (scan_buf), &ref_l, &line_num, FILE_SOURCE_NAME (&A68_JOB));
 723      SCAN_ERROR (l != (ssize_t) strlen (A68_PARSER (scan_buf)), NO_LINE, NO_TEXT, ERROR_FILE_SOURCE_CTRL);
 724    }
 725  // Postlude.
 726    append_environ (postlude, &ref_l, &line_num, "postlude");
 727  // Concatenate lines.
 728    concatenate_lines (TOP_LINE (&A68_JOB));
 729  // Include files.
 730    include_files (TOP_LINE (&A68_JOB));
 731    return A68_TRUE;
 732  }
 733  
 734  //! @brief Next_char get next character from internal copy of source file.
 735  
 736  char next_char (LINE_T ** ref_l, char **ref_s, BOOL_T allow_typo)
 737  {
 738    char ch;
 739  #if defined (NO_TYPO)
 740    allow_typo = A68_FALSE;
 741  #endif
 742    LOW_STACK_ALERT (NO_NODE);
 743  // Source empty?.
 744    if (*ref_l == NO_LINE) {
 745      return STOP_CHAR;
 746    } else {
 747      LIST (*ref_l) = (BOOL_T) (OPTION_NODEMASK (&A68_JOB) & SOURCE_MASK ? A68_TRUE : A68_FALSE);
 748  // Take new line?.
 749      if ((*ref_s)[0] == NEWLINE_CHAR || (*ref_s)[0] == NULL_CHAR) {
 750        *ref_l = NEXT (*ref_l);
 751        if (*ref_l == NO_LINE) {
 752          return STOP_CHAR;
 753        }
 754        *ref_s = STRING (*ref_l);
 755      } else {
 756        (*ref_s)++;
 757      }
 758  // Deliver next char.
 759      ch = (*ref_s)[0];
 760      if (allow_typo && (IS_SPACE (ch) || ch == FORMFEED_CHAR)) {
 761        ch = next_char (ref_l, ref_s, allow_typo);
 762      }
 763      return ch;
 764    }
 765  }
 766  
 767  //! @brief Find first character that can start a valid symbol.
 768  
 769  void get_good_char (char *ref_c, LINE_T ** ref_l, char **ref_s)
 770  {
 771    while (*ref_c != STOP_CHAR && (IS_SPACE (*ref_c) || (*ref_c == NULL_CHAR))) {
 772      if (*ref_l != NO_LINE) {
 773        LIST (*ref_l) = (BOOL_T) (OPTION_NODEMASK (&A68_JOB) & SOURCE_MASK ? A68_TRUE : A68_FALSE);
 774      }
 775      *ref_c = next_char (ref_l, ref_s, A68_FALSE);
 776    }
 777  }
 778  
 779  //! @brief Handle a pragment (pragmat or comment).
 780  
 781  char *pragment (int type, LINE_T ** ref_l, char **ref_c)
 782  {
 783  #define INIT_BUFFER {chars_in_buf = 0; A68_PARSER (scan_buf)[chars_in_buf] = NULL_CHAR;}
 784  #define ADD_ONE_CHAR(ch) {A68_PARSER (scan_buf)[chars_in_buf ++] = ch; A68_PARSER (scan_buf)[chars_in_buf] = NULL_CHAR;}
 785    char c = **ref_c, *term_s = NO_TEXT, *start_c = *ref_c;
 786    char *z = NO_TEXT;
 787    LINE_T *start_l = *ref_l;
 788    int term_s_length, chars_in_buf;
 789    BOOL_T stop, pragmat = A68_FALSE;
 790  // Set terminator.
 791    if (OPTION_STROPPING (&A68_JOB) == UPPER_STROPPING) {
 792      if (type == STYLE_I_COMMENT_SYMBOL) {
 793        term_s = "CO";
 794      } else if (type == STYLE_II_COMMENT_SYMBOL) {
 795        term_s = "#";
 796      } else if (type == BOLD_COMMENT_SYMBOL) {
 797        term_s = "COMMENT";
 798      } else if (type == STYLE_I_PRAGMAT_SYMBOL) {
 799        term_s = "PR";
 800        pragmat = A68_TRUE;
 801      } else if (type == BOLD_PRAGMAT_SYMBOL) {
 802        term_s = "PRAGMAT";
 803        pragmat = A68_TRUE;
 804      }
 805    } else if (OPTION_STROPPING (&A68_JOB) == QUOTE_STROPPING) {
 806      if (type == STYLE_I_COMMENT_SYMBOL) {
 807        term_s = "'CO'";
 808      } else if (type == STYLE_II_COMMENT_SYMBOL) {
 809        term_s = "#";
 810      } else if (type == BOLD_COMMENT_SYMBOL) {
 811        term_s = "'COMMENT'";
 812      } else if (type == STYLE_I_PRAGMAT_SYMBOL) {
 813        term_s = "'PR'";
 814        pragmat = A68_TRUE;
 815      } else if (type == BOLD_PRAGMAT_SYMBOL) {
 816        term_s = "'PRAGMAT'";
 817        pragmat = A68_TRUE;
 818      }
 819    }
 820    term_s_length = (int) strlen (term_s);
 821  // Scan for terminator.
 822    INIT_BUFFER;
 823    stop = A68_FALSE;
 824    while (stop == A68_FALSE) {
 825      BOOL_T scan_next = A68_TRUE;
 826      SCAN_ERROR (c == STOP_CHAR, start_l, start_c, ERROR_UNTERMINATED_PRAGMENT);
 827  // A ".." or '..' delimited string in a PRAGMAT.
 828      if (pragmat && (c == QUOTE_CHAR || (c == '\'' && OPTION_STROPPING (&A68_JOB) == UPPER_STROPPING))) {
 829        char delim = c;
 830        BOOL_T eos = A68_FALSE;
 831        ADD_ONE_CHAR (c);
 832        c = next_char (ref_l, ref_c, A68_FALSE);
 833        while (!eos) {
 834          SCAN_ERROR (EOL (c), start_l, start_c, ERROR_LONG_STRING);
 835          if (c == delim) {
 836            ADD_ONE_CHAR (delim);
 837            save_state (*ref_l, *ref_c, c);
 838            c = next_char (ref_l, ref_c, A68_FALSE);
 839            if (c == delim) {
 840              c = next_char (ref_l, ref_c, A68_FALSE);
 841            } else {
 842              restore_state (ref_l, ref_c, &c);
 843              eos = A68_TRUE;
 844            }
 845          } else if (IS_PRINT (c)) {
 846            ADD_ONE_CHAR (c);
 847            c = next_char (ref_l, ref_c, A68_FALSE);
 848          } else {
 849            unworthy (start_l, start_c, c);
 850          }
 851        }
 852      } else if (EOL (c)) {
 853        ADD_ONE_CHAR (NEWLINE_CHAR);
 854      } else if (IS_UPPER (c)) {
 855        while (IS_UPPER (c)) {
 856          ADD_ONE_CHAR (c);
 857          c = next_char (ref_l, ref_c, A68_FALSE);
 858        }
 859        scan_next = FALSE;
 860      } else if (IS_PRINT (c) || IS_SPACE (c)) {
 861        ADD_ONE_CHAR (c);
 862      }
 863      if (chars_in_buf >= term_s_length) {
 864  // Check whether we encountered the terminator.
 865        char *tok = &(A68_PARSER (scan_buf)[chars_in_buf - term_s_length]);
 866        stop = (BOOL_T) (strcmp (term_s, tok) == 0);
 867      }
 868      if (scan_next) {
 869        c = next_char (ref_l, ref_c, A68_FALSE);
 870      }
 871    }
 872    A68_PARSER (scan_buf)[chars_in_buf - term_s_length] = NULL_CHAR;
 873    z = new_string (term_s, A68_PARSER (scan_buf), term_s, NO_TEXT);
 874    if (type == STYLE_I_PRAGMAT_SYMBOL || type == BOLD_PRAGMAT_SYMBOL) {
 875      isolate_options (A68_PARSER (scan_buf), start_l);
 876    }
 877    return z;
 878  #undef ADD_ONE_CHAR
 879  #undef INIT_BUFFER
 880  }
 881  
 882  //! @brief Attribute for format item.
 883  
 884  int get_format_item (char ch)
 885  {
 886    switch (TO_LOWER (ch)) {
 887    case 'a': {
 888        return FORMAT_ITEM_A;
 889      }
 890    case 'b': {
 891        return FORMAT_ITEM_B;
 892      }
 893    case 'c': {
 894        return FORMAT_ITEM_C;
 895      }
 896    case 'd': {
 897        return FORMAT_ITEM_D;
 898      }
 899    case 'e': {
 900        return FORMAT_ITEM_E;
 901      }
 902    case 'f': {
 903        return FORMAT_ITEM_F;
 904      }
 905    case 'g': {
 906        return FORMAT_ITEM_G;
 907      }
 908    case 'h': {
 909        return FORMAT_ITEM_H;
 910      }
 911    case 'i': {
 912        return FORMAT_ITEM_I;
 913      }
 914    case 'j': {
 915        return FORMAT_ITEM_J;
 916      }
 917    case 'k': {
 918        return FORMAT_ITEM_K;
 919      }
 920    case 'l':
 921    case '/': {
 922        return FORMAT_ITEM_L;
 923      }
 924    case 'm': {
 925        return FORMAT_ITEM_M;
 926      }
 927    case 'n': {
 928        return FORMAT_ITEM_N;
 929      }
 930    case 'o': {
 931        return FORMAT_ITEM_O;
 932      }
 933    case 'p': {
 934        return FORMAT_ITEM_P;
 935      }
 936    case 'q': {
 937        return FORMAT_ITEM_Q;
 938      }
 939    case 'r': {
 940        return FORMAT_ITEM_R;
 941      }
 942    case 's': {
 943        return FORMAT_ITEM_S;
 944      }
 945    case 't': {
 946        return FORMAT_ITEM_T;
 947      }
 948    case 'u': {
 949        return FORMAT_ITEM_U;
 950      }
 951    case 'v': {
 952        return FORMAT_ITEM_V;
 953      }
 954    case 'w': {
 955        return FORMAT_ITEM_W;
 956      }
 957    case 'x': {
 958        return FORMAT_ITEM_X;
 959      }
 960    case 'y': {
 961        return FORMAT_ITEM_Y;
 962      }
 963    case 'z': {
 964        return FORMAT_ITEM_Z;
 965      }
 966    case '+': {
 967        return FORMAT_ITEM_PLUS;
 968      }
 969    case '-': {
 970        return FORMAT_ITEM_MINUS;
 971      }
 972    case POINT_CHAR: {
 973        return FORMAT_ITEM_POINT;
 974      }
 975    case '%': {
 976        return FORMAT_ITEM_ESCAPE;
 977      }
 978    default: {
 979        return 0;
 980      }
 981    }
 982  }
 983  
 984  //! @brief Whether input shows exponent character.
 985  
 986  BOOL_T is_exp_char (LINE_T ** ref_l, char **ref_s, char *ch)
 987  {
 988    BOOL_T ret = A68_FALSE;
 989    char exp_syms[3];
 990    if (OPTION_STROPPING (&A68_JOB) == UPPER_STROPPING) {
 991      exp_syms[0] = EXPONENT_CHAR;
 992      exp_syms[1] = (char) TO_UPPER (EXPONENT_CHAR);
 993      exp_syms[2] = NULL_CHAR;
 994    } else {
 995      exp_syms[0] = (char) TO_UPPER (EXPONENT_CHAR);
 996      exp_syms[1] = BACKSLASH_CHAR;
 997      exp_syms[2] = NULL_CHAR;
 998    }
 999    save_state (*ref_l, *ref_s, *ch);
1000    if (strchr (exp_syms, *ch) != NO_TEXT) {
1001      *ch = next_char (ref_l, ref_s, A68_TRUE);
1002      ret = (BOOL_T) (strchr ("+-0123456789", *ch) != NO_TEXT);
1003    }
1004    restore_state (ref_l, ref_s, ch);
1005    return ret;
1006  }
1007  
1008  //! @brief Whether input shows radix character.
1009  
1010  BOOL_T is_radix_char (LINE_T ** ref_l, char **ref_s, char *ch)
1011  {
1012    BOOL_T ret = A68_FALSE;
1013    save_state (*ref_l, *ref_s, *ch);
1014    if (OPTION_STROPPING (&A68_JOB) == QUOTE_STROPPING) {
1015      if (*ch == TO_UPPER (RADIX_CHAR)) {
1016        *ch = next_char (ref_l, ref_s, A68_TRUE);
1017        ret = (BOOL_T) (strchr ("0123456789ABCDEF", *ch) != NO_TEXT);
1018      }
1019    } else {
1020      if (*ch == RADIX_CHAR) {
1021        *ch = next_char (ref_l, ref_s, A68_TRUE);
1022        ret = (BOOL_T) (strchr ("0123456789abcdef", *ch) != NO_TEXT);
1023      }
1024    }
1025    restore_state (ref_l, ref_s, ch);
1026    return ret;
1027  }
1028  
1029  //! @brief Whether input shows decimal point.
1030  
1031  BOOL_T is_decimal_point (LINE_T ** ref_l, char **ref_s, char *ch)
1032  {
1033    BOOL_T ret = A68_FALSE;
1034    save_state (*ref_l, *ref_s, *ch);
1035    if (*ch == POINT_CHAR) {
1036      char exp_syms[3];
1037      if (OPTION_STROPPING (&A68_JOB) == UPPER_STROPPING) {
1038        exp_syms[0] = EXPONENT_CHAR;
1039        exp_syms[1] = (char) TO_UPPER (EXPONENT_CHAR);
1040        exp_syms[2] = NULL_CHAR;
1041      } else {
1042        exp_syms[0] = (char) TO_UPPER (EXPONENT_CHAR);
1043        exp_syms[1] = BACKSLASH_CHAR;
1044        exp_syms[2] = NULL_CHAR;
1045      }
1046      *ch = next_char (ref_l, ref_s, A68_TRUE);
1047      if (strchr (exp_syms, *ch) != NO_TEXT) {
1048        *ch = next_char (ref_l, ref_s, A68_TRUE);
1049        ret = (BOOL_T) (strchr ("+-0123456789", *ch) != NO_TEXT);
1050      } else {
1051        ret = (BOOL_T) (strchr ("0123456789", *ch) != NO_TEXT);
1052      }
1053    }
1054    restore_state (ref_l, ref_s, ch);
1055    return ret;
1056  }
1057  
1058  //! @brief Get next token from internal copy of source file..
1059  
1060  void get_next_token (BOOL_T in_format, LINE_T ** ref_l, char **ref_s, LINE_T ** start_l, char **start_c, int *att)
1061  {
1062    char c = **ref_s, *sym = A68_PARSER (scan_buf);
1063    sym[0] = NULL_CHAR;
1064    get_good_char (&c, ref_l, ref_s);
1065    *start_l = *ref_l;
1066    *start_c = *ref_s;
1067    if (c == STOP_CHAR) {
1068  // We are at EOF.
1069      (sym++)[0] = STOP_CHAR;
1070      sym[0] = NULL_CHAR;
1071      return;
1072    }
1073  // In a format.
1074    if (in_format) {
1075      char *format_items;
1076      if (OPTION_STROPPING (&A68_JOB) == UPPER_STROPPING) {
1077        format_items = "/%\\+-.abcdefghijklmnopqrstuvwxyz";
1078      } else if (OPTION_STROPPING (&A68_JOB) == QUOTE_STROPPING) {
1079        format_items = "/%\\+-.ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1080      } else {
1081        format_items = "/%\\+-.abcdefghijklmnopqrstuvwxyz";
1082      }
1083      if (strchr (format_items, c) != NO_TEXT) {
1084  // General format items.
1085        (sym++)[0] = c;
1086        sym[0] = NULL_CHAR;
1087        *att = get_format_item (c);
1088        (void) next_char (ref_l, ref_s, A68_FALSE);
1089        return;
1090      }
1091      if (IS_DIGIT (c)) {
1092  // INT denotation for static replicator.
1093        SCAN_DIGITS (c);
1094        sym[0] = NULL_CHAR;
1095        *att = STATIC_REPLICATOR;
1096        return;
1097      }
1098    }
1099  // Not in a format.
1100    if (IS_UPPER (c)) {
1101      if (OPTION_STROPPING (&A68_JOB) == UPPER_STROPPING) {
1102  // Upper case word - bold tag.
1103        while (IS_UPPER (c) || c == '_') {
1104          (sym++)[0] = c;
1105          c = next_char (ref_l, ref_s, A68_FALSE);
1106        }
1107        sym[0] = NULL_CHAR;
1108        *att = BOLD_TAG;
1109      } else if (OPTION_STROPPING (&A68_JOB) == QUOTE_STROPPING) {
1110        while (IS_UPPER (c) || IS_DIGIT (c) || c == '_') {
1111          (sym++)[0] = c;
1112          c = next_char (ref_l, ref_s, A68_TRUE);
1113        }
1114        sym[0] = NULL_CHAR;
1115        *att = IDENTIFIER;
1116      }
1117    } else if (c == '\'') {
1118  // Quote, uppercase word, quote - bold tag.
1119      int k = 0;
1120      c = next_char (ref_l, ref_s, A68_FALSE);
1121      while (IS_UPPER (c) || IS_DIGIT (c) || c == '_') {
1122        (sym++)[0] = c;
1123        k++;
1124        c = next_char (ref_l, ref_s, A68_TRUE);
1125      }
1126      SCAN_ERROR (k == 0, *start_l, *start_c, ERROR_QUOTED_BOLD_TAG);
1127      sym[0] = NULL_CHAR;
1128      *att = BOLD_TAG;
1129  // Skip terminating quote, or complain if it is not there.
1130      SCAN_ERROR (c != '\'', *start_l, *start_c, ERROR_QUOTED_BOLD_TAG);
1131      c = next_char (ref_l, ref_s, A68_FALSE);
1132    } else if (IS_LOWER (c)) {
1133  // Lower case word - identifier.
1134      while (IS_LOWER (c) || IS_DIGIT (c) || c == '_') {
1135        (sym++)[0] = c;
1136        c = next_char (ref_l, ref_s, A68_TRUE);
1137      }
1138      sym[0] = NULL_CHAR;
1139      *att = IDENTIFIER;
1140    } else if (c == POINT_CHAR) {
1141  // Begins with a point symbol - point, dotdot, L REAL denotation.
1142      if (is_decimal_point (ref_l, ref_s, &c)) {
1143        (sym++)[0] = '0';
1144        (sym++)[0] = POINT_CHAR;
1145        c = next_char (ref_l, ref_s, A68_TRUE);
1146        SCAN_DIGITS (c);
1147        if (is_exp_char (ref_l, ref_s, &c)) {
1148          SCAN_EXPONENT_PART (c);
1149        }
1150        sym[0] = NULL_CHAR;
1151        *att = REAL_DENOTATION;
1152      } else {
1153        c = next_char (ref_l, ref_s, A68_TRUE);
1154        if (c == POINT_CHAR) {
1155          (sym++)[0] = POINT_CHAR;
1156          (sym++)[0] = POINT_CHAR;
1157          sym[0] = NULL_CHAR;
1158          *att = DOTDOT_SYMBOL;
1159          c = next_char (ref_l, ref_s, A68_FALSE);
1160        } else {
1161          (sym++)[0] = POINT_CHAR;
1162          sym[0] = NULL_CHAR;
1163          *att = POINT_SYMBOL;
1164        }
1165      }
1166    } else if (IS_DIGIT (c)) {
1167  // Something that begins with a digit - L INT denotation, L REAL denotation.
1168      SCAN_DIGITS (c);
1169      if (is_decimal_point (ref_l, ref_s, &c)) {
1170        c = next_char (ref_l, ref_s, A68_TRUE);
1171        if (is_exp_char (ref_l, ref_s, &c)) {
1172          (sym++)[0] = POINT_CHAR;
1173          (sym++)[0] = '0';
1174          SCAN_EXPONENT_PART (c);
1175          *att = REAL_DENOTATION;
1176        } else {
1177          (sym++)[0] = POINT_CHAR;
1178          SCAN_DIGITS (c);
1179          if (is_exp_char (ref_l, ref_s, &c)) {
1180            SCAN_EXPONENT_PART (c);
1181          }
1182          *att = REAL_DENOTATION;
1183        }
1184      } else if (is_exp_char (ref_l, ref_s, &c)) {
1185        SCAN_EXPONENT_PART (c);
1186        *att = REAL_DENOTATION;
1187      } else if (is_radix_char (ref_l, ref_s, &c)) {
1188        (sym++)[0] = c;
1189        c = next_char (ref_l, ref_s, A68_TRUE);
1190        if (OPTION_STROPPING (&A68_JOB) == UPPER_STROPPING) {
1191          while (IS_DIGIT (c) || strchr ("abcdef", c) != NO_TEXT) {
1192            (sym++)[0] = c;
1193            c = next_char (ref_l, ref_s, A68_TRUE);
1194          }
1195        } else {
1196          while (IS_DIGIT (c) || strchr ("ABCDEF", c) != NO_TEXT) {
1197            (sym++)[0] = c;
1198            c = next_char (ref_l, ref_s, A68_TRUE);
1199          }
1200        }
1201        *att = BITS_DENOTATION;
1202      } else {
1203        *att = INT_DENOTATION;
1204      }
1205      sym[0] = NULL_CHAR;
1206    } else if (c == QUOTE_CHAR) {
1207  // STRING denotation.
1208      BOOL_T stop = A68_FALSE;
1209      while (!stop) {
1210        c = next_char (ref_l, ref_s, A68_FALSE);
1211        while (c != QUOTE_CHAR && c != STOP_CHAR) {
1212          SCAN_ERROR (EOL (c), *start_l, *start_c, ERROR_LONG_STRING);
1213          (sym++)[0] = c;
1214          c = next_char (ref_l, ref_s, A68_FALSE);
1215        }
1216        SCAN_ERROR (*ref_l == NO_LINE, *start_l, *start_c, ERROR_UNTERMINATED_STRING);
1217        c = next_char (ref_l, ref_s, A68_FALSE);
1218        if (c == QUOTE_CHAR) {
1219          (sym++)[0] = QUOTE_CHAR;
1220        } else {
1221          stop = A68_TRUE;
1222        }
1223      }
1224      sym[0] = NULL_CHAR;
1225      *att = (in_format ? LITERAL : ROW_CHAR_DENOTATION);
1226    } else if (strchr ("#$()[]{},;@", c) != NO_TEXT) {
1227  // Single character symbols.
1228      (sym++)[0] = c;
1229      (void) next_char (ref_l, ref_s, A68_FALSE);
1230      sym[0] = NULL_CHAR;
1231      *att = 0;
1232    } else if (c == '|') {
1233  // Bar.
1234      (sym++)[0] = c;
1235      c = next_char (ref_l, ref_s, A68_FALSE);
1236      if (c == ':') {
1237        (sym++)[0] = c;
1238        (void) next_char (ref_l, ref_s, A68_FALSE);
1239      }
1240      sym[0] = NULL_CHAR;
1241      *att = 0;
1242    } else if (c == '!' && OPTION_STROPPING (&A68_JOB) == QUOTE_STROPPING) {
1243  // Bar, will be replaced with modern variant.
1244  // For this reason ! is not a MONAD with quote-stropping.
1245      (sym++)[0] = '|';
1246      c = next_char (ref_l, ref_s, A68_FALSE);
1247      if (c == ':') {
1248        (sym++)[0] = c;
1249        (void) next_char (ref_l, ref_s, A68_FALSE);
1250      }
1251      sym[0] = NULL_CHAR;
1252      *att = 0;
1253    } else if (c == ':') {
1254  // Colon, semicolon, IS, ISNT.
1255      (sym++)[0] = c;
1256      c = next_char (ref_l, ref_s, A68_FALSE);
1257      if (c == '=') {
1258        (sym++)[0] = c;
1259        if ((c = next_char (ref_l, ref_s, A68_FALSE)) == ':') {
1260          (sym++)[0] = c;
1261          c = next_char (ref_l, ref_s, A68_FALSE);
1262        }
1263      } else if (c == '/') {
1264        (sym++)[0] = c;
1265        if ((c = next_char (ref_l, ref_s, A68_FALSE)) == '=') {
1266          (sym++)[0] = c;
1267          if ((c = next_char (ref_l, ref_s, A68_FALSE)) == ':') {
1268            (sym++)[0] = c;
1269            c = next_char (ref_l, ref_s, A68_FALSE);
1270          }
1271        }
1272      } else if (c == ':') {
1273        (sym++)[0] = c;
1274        if ((c = next_char (ref_l, ref_s, A68_FALSE)) == '=') {
1275          (sym++)[0] = c;
1276        }
1277      }
1278      sym[0] = NULL_CHAR;
1279      *att = 0;
1280    } else if (c == '=') {
1281  // Operator starting with "=".
1282      char *scanned = sym;
1283      (sym++)[0] = c;
1284      c = next_char (ref_l, ref_s, A68_FALSE);
1285      if (strchr (NOMADS, c) != NO_TEXT) {
1286        (sym++)[0] = c;
1287        c = next_char (ref_l, ref_s, A68_FALSE);
1288      }
1289      if (c == '=') {
1290        (sym++)[0] = c;
1291        if (next_char (ref_l, ref_s, A68_FALSE) == ':') {
1292          (sym++)[0] = ':';
1293          c = next_char (ref_l, ref_s, A68_FALSE);
1294          if (strlen (sym) < 4 && c == '=') {
1295            (sym++)[0] = '=';
1296            (void) next_char (ref_l, ref_s, A68_FALSE);
1297          }
1298        }
1299      } else if (c == ':') {
1300        (sym++)[0] = c;
1301        sym[0] = NULL_CHAR;
1302        if (next_char (ref_l, ref_s, A68_FALSE) == '=') {
1303          (sym++)[0] = '=';
1304          (void) next_char (ref_l, ref_s, A68_FALSE);
1305        } else {
1306          SCAN_ERROR (!(strcmp (scanned, "=:") == 0 || strcmp (scanned, "==:") == 0), *start_l, *start_c, ERROR_INVALID_OPERATOR_TAG);
1307        }
1308      }
1309      sym[0] = NULL_CHAR;
1310      if (strcmp (scanned, "=") == 0) {
1311        *att = EQUALS_SYMBOL;
1312      } else {
1313        *att = OPERATOR;
1314      }
1315    } else if (strchr (MONADS, c) != NO_TEXT || strchr (NOMADS, c) != NO_TEXT) {
1316  // Operator.
1317      char *scanned = sym;
1318      (sym++)[0] = c;
1319      c = next_char (ref_l, ref_s, A68_FALSE);
1320      if (strchr (NOMADS, c) != NO_TEXT) {
1321        (sym++)[0] = c;
1322        c = next_char (ref_l, ref_s, A68_FALSE);
1323      }
1324      if (c == '=') {
1325        (sym++)[0] = c;
1326        if (next_char (ref_l, ref_s, A68_FALSE) == ':') {
1327          (sym++)[0] = ':';
1328          c = next_char (ref_l, ref_s, A68_FALSE);
1329          if (strlen (scanned) < 4 && c == '=') {
1330            (sym++)[0] = '=';
1331            (void) next_char (ref_l, ref_s, A68_FALSE);
1332          }
1333        }
1334      } else if (c == ':') {
1335        (sym++)[0] = c;
1336        sym[0] = NULL_CHAR;
1337        if (next_char (ref_l, ref_s, A68_FALSE) == '=') {
1338          (sym++)[0] = '=';
1339          sym[0] = NULL_CHAR;
1340          (void) next_char (ref_l, ref_s, A68_FALSE);
1341        } else {
1342          SCAN_ERROR (strcmp (&(scanned[1]), "=:") != 0, *start_l, *start_c, ERROR_INVALID_OPERATOR_TAG);
1343        }
1344      }
1345      sym[0] = NULL_CHAR;
1346      *att = OPERATOR;
1347    } else {
1348  // Afuuus ... strange characters!.
1349      unworthy (*start_l, *start_c, (int) c);
1350    }
1351  }
1352  
1353  //! @brief Whether att opens an embedded clause.
1354  
1355  BOOL_T open_nested_clause (int att)
1356  {
1357    switch (att) {
1358    case OPEN_SYMBOL:
1359    case BEGIN_SYMBOL:
1360    case PAR_SYMBOL:
1361    case IF_SYMBOL:
1362    case CASE_SYMBOL:
1363    case FOR_SYMBOL:
1364    case FROM_SYMBOL:
1365    case BY_SYMBOL:
1366    case TO_SYMBOL:
1367    case DOWNTO_SYMBOL:
1368    case WHILE_SYMBOL:
1369    case DO_SYMBOL:
1370    case SUB_SYMBOL:
1371    case ACCO_SYMBOL: {
1372        return A68_TRUE;
1373      }
1374    }
1375    return A68_FALSE;
1376  }
1377  
1378  //! @brief Whether att closes an embedded clause.
1379  
1380  BOOL_T close_nested_clause (int att)
1381  {
1382    switch (att) {
1383    case CLOSE_SYMBOL:
1384    case END_SYMBOL:
1385    case FI_SYMBOL:
1386    case ESAC_SYMBOL:
1387    case OD_SYMBOL:
1388    case BUS_SYMBOL:
1389    case OCCA_SYMBOL: {
1390        return A68_TRUE;
1391      }
1392    }
1393    return A68_FALSE;
1394  }
1395  
1396  //! @brief Cast a string to lower case.
1397  
1398  void make_lower_case (char *p)
1399  {
1400    for (; p != NO_TEXT && p[0] != NULL_CHAR; p++) {
1401      p[0] = (char) TO_LOWER (p[0]);
1402    }
1403  }
1404  
1405  //! @brief Construct a linear list of tokens.
1406  
1407  void tokenise_source (NODE_T ** root, int level, BOOL_T in_format, LINE_T ** l, char **s, LINE_T ** start_l, char **start_c)
1408  {
1409    char *lpr = NO_TEXT;
1410    int lprt = 0;
1411    while (l != NO_VAR && !A68_PARSER (stop_scanner)) {
1412      int att = 0;
1413      get_next_token (in_format, l, s, start_l, start_c, &att);
1414      if (A68_PARSER (scan_buf)[0] == STOP_CHAR) {
1415        A68_PARSER (stop_scanner) = A68_TRUE;
1416      } else if (strlen (A68_PARSER (scan_buf)) > 0 || att == ROW_CHAR_DENOTATION || att == LITERAL) {
1417        KEYWORD_T *kw;
1418        char *c = NO_TEXT;
1419        BOOL_T make_node = A68_TRUE;
1420        char *trailing = NO_TEXT;
1421        if (att != IDENTIFIER) {
1422          kw = find_keyword (A68 (top_keyword), A68_PARSER (scan_buf));
1423        } else {
1424          kw = NO_KEYWORD;
1425        }
1426        if (!(kw != NO_KEYWORD && att != ROW_CHAR_DENOTATION)) {
1427          if (att == IDENTIFIER) {
1428            make_lower_case (A68_PARSER (scan_buf));
1429          }
1430          if (att != ROW_CHAR_DENOTATION && att != LITERAL) {
1431            int len = (int) strlen (A68_PARSER (scan_buf));
1432            while (len >= 1 && A68_PARSER (scan_buf)[len - 1] == '_') {
1433              trailing = "_";
1434              A68_PARSER (scan_buf)[len - 1] = NULL_CHAR;
1435              len--;
1436            }
1437          }
1438          c = TEXT (add_token (&A68 (top_token), A68_PARSER (scan_buf)));
1439        } else {
1440          if (IS (kw, TO_SYMBOL)) {
1441  // Merge GO and TO to GOTO.
1442            if (*root != NO_NODE && IS (*root, GO_SYMBOL)) {
1443              ATTRIBUTE (*root) = GOTO_SYMBOL;
1444              NSYMBOL (*root) = TEXT (find_keyword (A68 (top_keyword), "GOTO"));
1445              make_node = A68_FALSE;
1446            } else {
1447              att = ATTRIBUTE (kw);
1448              c = TEXT (kw);
1449            }
1450          } else {
1451            if (att == 0 || att == BOLD_TAG) {
1452              att = ATTRIBUTE (kw);
1453            }
1454            c = TEXT (kw);
1455  // Handle pragments.
1456            if (att == STYLE_II_COMMENT_SYMBOL || att == STYLE_I_COMMENT_SYMBOL || att == BOLD_COMMENT_SYMBOL) {
1457              char *nlpr = pragment (ATTRIBUTE (kw), l, s);
1458              if (lpr == NO_TEXT || (int) strlen (lpr) == 0) {
1459                lpr = nlpr;
1460              } else {
1461                char *stale = lpr;
1462                lpr = new_string (lpr, "\n\n", nlpr, NO_TEXT);
1463                a68_free (nlpr);
1464                a68_free (stale);
1465              }
1466              lprt = att;
1467              make_node = A68_FALSE;
1468            } else if (att == STYLE_I_PRAGMAT_SYMBOL || att == BOLD_PRAGMAT_SYMBOL) {
1469              char *nlpr = pragment (ATTRIBUTE (kw), l, s);
1470              if (lpr == NO_TEXT || (int) strlen (lpr) == 0) {
1471                lpr = nlpr;
1472              } else {
1473                char *stale = lpr;
1474                lpr = new_string (lpr, "\n\n", nlpr, NO_TEXT);
1475                a68_free (nlpr);
1476                a68_free (stale);
1477              }
1478              lprt = att;
1479              if (!A68_PARSER (stop_scanner)) {
1480                (void) set_options (OPTION_LIST (&A68_JOB), A68_FALSE);
1481                make_node = A68_FALSE;
1482              }
1483            }
1484          }
1485        }
1486  // Add token to the tree.
1487        if (make_node) {
1488          NODE_T *q = new_node ();
1489          INFO (q) = new_node_info ();
1490          switch (att) {
1491          case ASSIGN_SYMBOL:
1492          case END_SYMBOL:
1493          case ESAC_SYMBOL:
1494          case OD_SYMBOL:
1495          case OF_SYMBOL:
1496          case FI_SYMBOL:
1497          case CLOSE_SYMBOL:
1498          case BUS_SYMBOL:
1499          case COLON_SYMBOL:
1500          case COMMA_SYMBOL:
1501          case DOTDOT_SYMBOL:
1502          case SEMI_SYMBOL: {
1503              GINFO (q) = NO_GINFO;
1504              break;
1505            }
1506          default: {
1507              GINFO (q) = new_genie_info ();
1508              break;
1509            }
1510          }
1511          STATUS (q) = OPTION_NODEMASK (&A68_JOB);
1512          LINE (INFO (q)) = *start_l;
1513          CHAR_IN_LINE (INFO (q)) = *start_c;
1514          PRIO (INFO (q)) = 0;
1515          PROCEDURE_LEVEL (INFO (q)) = 0;
1516          ATTRIBUTE (q) = att;
1517          NSYMBOL (q) = c;
1518          PREVIOUS (q) = *root;
1519          SUB (q) = NEXT (q) = NO_NODE;
1520          TABLE (q) = NO_TABLE;
1521          MOID (q) = NO_MOID;
1522          TAX (q) = NO_TAG;
1523          if (lpr != NO_TEXT) {
1524            NPRAGMENT (q) = lpr;
1525            NPRAGMENT_TYPE (q) = lprt;
1526            lpr = NO_TEXT;
1527            lprt = 0;
1528          }
1529          if (*root != NO_NODE) {
1530            NEXT (*root) = q;
1531          }
1532          if (TOP_NODE (&A68_JOB) == NO_NODE) {
1533            TOP_NODE (&A68_JOB) = q;
1534          }
1535          *root = q;
1536          if (trailing != NO_TEXT) {
1537            diagnostic (A68_WARNING | A68_FORCE_DIAGNOSTICS, q, WARNING_TRAILING, trailing, att);
1538          }
1539        }
1540  // Redirection in tokenising formats. The scanner is a recursive-descent type as
1541  // to know when it scans a format text and when not. 
1542        if (in_format && att == FORMAT_DELIMITER_SYMBOL) {
1543          return;
1544        } else if (!in_format && att == FORMAT_DELIMITER_SYMBOL) {
1545          tokenise_source (root, level + 1, A68_TRUE, l, s, start_l, start_c);
1546        } else if (in_format && open_nested_clause (att)) {
1547          NODE_T *z = PREVIOUS (*root);
1548          if (z != NO_NODE && is_one_of (z, FORMAT_ITEM_N, FORMAT_ITEM_G, FORMAT_ITEM_H, FORMAT_ITEM_F, STOP)) {
1549            tokenise_source (root, level, A68_FALSE, l, s, start_l, start_c);
1550          } else if (att == OPEN_SYMBOL) {
1551            ATTRIBUTE (*root) = FORMAT_OPEN_SYMBOL;
1552          } else if (OPTION_BRACKETS (&A68_JOB) && att == SUB_SYMBOL) {
1553            ATTRIBUTE (*root) = FORMAT_OPEN_SYMBOL;
1554          } else if (OPTION_BRACKETS (&A68_JOB) && att == ACCO_SYMBOL) {
1555            ATTRIBUTE (*root) = FORMAT_OPEN_SYMBOL;
1556          }
1557        } else if (!in_format && level > 0 && open_nested_clause (att)) {
1558          tokenise_source (root, level + 1, A68_FALSE, l, s, start_l, start_c);
1559        } else if (!in_format && level > 0 && close_nested_clause (att)) {
1560          return;
1561        } else if (in_format && att == CLOSE_SYMBOL) {
1562          ATTRIBUTE (*root) = FORMAT_CLOSE_SYMBOL;
1563        } else if (OPTION_BRACKETS (&A68_JOB) && in_format && att == BUS_SYMBOL) {
1564          ATTRIBUTE (*root) = FORMAT_CLOSE_SYMBOL;
1565        } else if (OPTION_BRACKETS (&A68_JOB) && in_format && att == OCCA_SYMBOL) {
1566          ATTRIBUTE (*root) = FORMAT_CLOSE_SYMBOL;
1567        }
1568      }
1569    }
1570  }
1571  
1572  //! @brief Tokenise source file, build initial syntax tree.
1573  
1574  BOOL_T lexical_analyser (void)
1575  {
1576    LINE_T *l = NO_LINE, *start_l = NO_LINE;
1577    char *s = NO_TEXT, *start_c = NO_TEXT;
1578    NODE_T *root = NO_NODE;
1579    A68_PARSER (scan_buf) = NO_TEXT;
1580    A68_PARSER (max_scan_buf_length) = A68_PARSER (source_file_size) = get_source_size ();
1581  // Errors in file?.
1582    if (A68_PARSER (max_scan_buf_length) == 0) {
1583      return A68_FALSE;
1584    }
1585    if (OPTION_RUN_SCRIPT (&A68_JOB)) {
1586      A68_PARSER (scan_buf) = (char *) get_temp_heap_space ((unt) (8 + A68_PARSER (max_scan_buf_length)));
1587      if (!read_script_file ()) {
1588        return A68_FALSE;
1589      }
1590    } else {
1591      A68_PARSER (max_scan_buf_length) += KILOBYTE;       // for the environ, more than enough
1592      A68_PARSER (scan_buf) = (char *) get_temp_heap_space ((unt) A68_PARSER (max_scan_buf_length));
1593  // Errors in file?.
1594      if (!read_source_file ()) {
1595        return A68_FALSE;
1596      }
1597    }
1598  // Start tokenising.
1599    A68_PARSER (read_error) = A68_FALSE;
1600    A68_PARSER (stop_scanner) = A68_FALSE;
1601    if ((l = TOP_LINE (&A68_JOB)) != NO_LINE) {
1602      s = STRING (l);
1603    }
1604    tokenise_source (&root, 0, A68_FALSE, &l, &s, &start_l, &start_c);
1605    return A68_TRUE;
1606  }