parser-scanner.c
1 //! @file parser-scanner.c
2 //! @author J. Marcel van der Veer
3
4 //! @section Copyright
5 //!
6 //! This file is part of Algol68G - an Algol 68 compiler-interpreter.
7 //! Copyright 2001-2025 J. Marcel van der Veer [algol68g@xs4all.nl].
8
9 //! @section License
10 //!
11 //! This program is free software; you can redistribute it and/or modify it
12 //! under the terms of the GNU General Public License as published by the
13 //! Free Software Foundation; either version 3 of the License, or
14 //! (at your option) any later version.
15 //!
16 //! This program is distributed in the hope that it will be useful, but
17 //! WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18 //! or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
19 //! more details. You should have received a copy of the GNU General Public
20 //! License along with this program. If not, see [http://www.gnu.org/licenses/].
21
22 //! @section Synopsis
23 //!
24 //! Context-dependent Algol 68 tokeniser.
25
26 #include "a68g.h"
27 #include "a68g-parser.h"
28 #include "a68g-prelude.h"
29 #include "a68g-options.h"
30 #include "a68g-environ.h"
31
32 // Macros.
33
34 #define SCAN_DIGITS(c)\
35 while (IS_DIGIT (c)) {\
36 (sym++)[0] = (c);\
37 (c) = next_char (ref_l, ref_s, A68G_TRUE);\
38 }
39
40 #define SCAN_EXPONENT_PART(c)\
41 (sym++)[0] = EXPONENT_CHAR;\
42 (c) = next_char (ref_l, ref_s, A68G_TRUE);\
43 if ((c) == '+' || (c) == '-') {\
44 (sym++)[0] = (c);\
45 (c) = next_char (ref_l, ref_s, A68G_TRUE);\
46 }\
47 SCAN_ERROR (!IS_DIGIT (c), *start_l, *start_c, ERROR_EXPONENT_DIGIT);\
48 SCAN_DIGITS (c)
49
50 //! @brief Save scanner state, for character look-ahead.
51
52 void save_state (LINE_T * ref_l, char *ref_s, char ch)
53 {
54 SCAN_STATE_L (&A68G_JOB) = ref_l;
55 SCAN_STATE_S (&A68G_JOB) = ref_s;
56 SCAN_STATE_C (&A68G_JOB) = ch;
57 }
58
59 //! @brief Restore scanner state, for character look-ahead.
60
61 void restore_state (LINE_T ** ref_l, char **ref_s, char *ch)
62 {
63 *ref_l = SCAN_STATE_L (&A68G_JOB);
64 *ref_s = SCAN_STATE_S (&A68G_JOB);
65 *ch = SCAN_STATE_C (&A68G_JOB);
66 }
67
68 //! @brief New_source_line.
69
70 LINE_T *new_source_line (void)
71 {
72 LINE_T *z = (LINE_T *) get_fixed_heap_space ((size_t) SIZE_ALIGNED (LINE_T));
73 MARKER (z)[0] = NULL_CHAR;
74 STRING (z) = NO_TEXT;
75 FILENAME (z) = NO_TEXT;
76 DIAGNOSTICS (z) = NO_DIAGNOSTIC;
77 NUMBER (z) = 0;
78 PRINT_STATUS (z) = 0;
79 LIST (z) = A68G_TRUE;
80 NEXT (z) = NO_LINE;
81 PREVIOUS (z) = NO_LINE;
82 return z;
83 }
84
85 //! @brief Append a source line to the internal source file.
86
87 void append_source_line (char *str, LINE_T ** ref_l, int *line_num, char *filename)
88 {
89 LINE_T *z = new_source_line ();
90 // Allow shell command in first line, f.i. "#!/usr/share/bin/a68g".
91 if (*line_num == 1) {
92 if (strlen (str) >= 2 && strncmp (str, "#!", strlen ("#!")) == 0) {
93 ABEND (strstr (str, "run-script") != NO_TEXT, ERROR_SHELL_SCRIPT, __func__);
94 (*line_num)++;
95 return;
96 }
97 }
98 // Link line into the chain.
99 STRING (z) = new_fixed_string (str);
100 FILENAME (z) = filename;
101 NUMBER (z) = (*line_num)++;
102 PRINT_STATUS (z) = NOT_PRINTED;
103 LIST (z) = A68G_TRUE;
104 DIAGNOSTICS (z) = NO_DIAGNOSTIC;
105 NEXT (z) = NO_LINE;
106 PREVIOUS (z) = *ref_l;
107 if (TOP_LINE (&A68G_JOB) == NO_LINE) {
108 TOP_LINE (&A68G_JOB) = z;
109 }
110 if (*ref_l != NO_LINE) {
111 NEXT (*ref_l) = z;
112 }
113 *ref_l = z;
114 }
115
116 // Scanner, tokenises the source code.
117
118 //! @brief Whether ch is unworthy.
119
120 void unworthy (LINE_T * u, char *v, char ch)
121 {
122 if (IS_PRINT (ch)) {
123 ASSERT (a68g_bufprt (A68G (edit_line), SNPRINTF_SIZE, "*%s", ERROR_UNWORTHY_CHARACTER) >= 0);
124 } else {
125 ASSERT (a68g_bufprt (A68G (edit_line), SNPRINTF_SIZE, "*%s %s", ERROR_UNWORTHY_CHARACTER, ctrl_char (ch)) >= 0);
126 }
127 scan_error (u, v, A68G (edit_line));
128 }
129
130 //! @brief Concatenate lines that terminate in '\' with next line.
131
132 void concatenate_lines (LINE_T * top)
133 {
134 LINE_T *q;
135 // Work from bottom backwards.
136 for (q = top; q != NO_LINE && NEXT (q) != NO_LINE; q = NEXT (q)) {
137 ;
138 }
139 for (; q != NO_LINE; BACKWARD (q)) {
140 char *z = STRING (q);
141 int len = (int) strlen (z);
142 if (len >= 2 && z[len - 2] == BACKSLASH_CHAR && z[len - 1] == NEWLINE_CHAR && NEXT (q) != NO_LINE && STRING (NEXT (q)) != NO_TEXT) {
143 z[len - 2] = NULL_CHAR;
144 len += (int) strlen (STRING (NEXT (q)));
145 z = (char *) get_fixed_heap_space ((size_t) (len + 1));
146 a68g_bufcpy (z, STRING (q), len + 1);
147 a68g_bufcat (z, STRING (NEXT (q)), len + 1);
148 STRING (NEXT (q))[0] = NULL_CHAR;
149 STRING (q) = z;
150 }
151 }
152 }
153
154 //! @brief Whether u is bold tag v, independent of stropping regime.
155
156 BOOL_T is_bold (char *u, char *v)
157 {
158 unt len = (unt) strlen (v);
159 if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
160 if (u[0] == '\'') {
161 return (BOOL_T) (strncmp (++u, v, len) == 0 && u[len] == '\'');
162 } else {
163 return A68G_FALSE;
164 }
165 } else {
166 return (BOOL_T) (strncmp (u, v, len) == 0 && !IS_UPPER (u[len]));
167 }
168 }
169
170 //! @brief Skip string.
171
172 BOOL_T skip_string (LINE_T ** top, char **ch)
173 {
174 LINE_T *u = *top;
175 char *v = *ch;
176 v++;
177 while (u != NO_LINE) {
178 while (v[0] != NULL_CHAR) {
179 if (v[0] == QUOTE_CHAR && v[1] != QUOTE_CHAR) {
180 *top = u;
181 *ch = &v[1];
182 return A68G_TRUE;
183 } else if (v[0] == QUOTE_CHAR && v[1] == QUOTE_CHAR) {
184 v += 2;
185 } else {
186 v++;
187 }
188 }
189 FORWARD (u);
190 if (u != NO_LINE) {
191 v = &(STRING (u)[0]);
192 } else {
193 v = NO_TEXT;
194 }
195 }
196 return A68G_FALSE;
197 }
198
199 //! @brief Skip comment.
200
201 BOOL_T skip_comment (LINE_T ** top, char **ch, int delim)
202 {
203 LINE_T *u = *top;
204 char *v = *ch;
205 BOOL_T qstrop = OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING;
206 v++;
207 while (u != NO_LINE) {
208 while (v[0] != NULL_CHAR) {
209 if (is_bold (v, "COMMENT") && delim == BOLD_COMMENT_SYMBOL) {
210 char *w = &v[strlen(qstrop ? "'COMMENT'" : "COMMENT")];
211 if (!IS_UPPER(w[0])) {
212 *top = u;
213 *ch = w;
214 return A68G_TRUE;
215 }
216 } else if (is_bold (v, "CO") && delim == STYLE_I_COMMENT_SYMBOL) {
217 char *w = &v[strlen(qstrop ? "'CO'" : "CO")];
218 if (!IS_UPPER(w[0])) {
219 *top = u;
220 *ch = w;
221 return A68G_TRUE;
222 }
223 } else if (v[0] == '#' && delim == STYLE_II_COMMENT_SYMBOL) {
224 *top = u;
225 *ch = &v[1];
226 return A68G_TRUE;
227 } else {
228 v++;
229 }
230 }
231 FORWARD (u);
232 if (u != NO_LINE) {
233 v = &(STRING (u)[0]);
234 } else {
235 v = NO_TEXT;
236 }
237 }
238 return A68G_FALSE;
239 }
240
241 //! @brief Skip rest of pragmat.
242
243 BOOL_T skip_pragmat (LINE_T ** top, char **ch, int delim, BOOL_T whitespace)
244 {
245 LINE_T *u = *top;
246 char *v = *ch;
247 BOOL_T qstrop = OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING;
248 while (u != NO_LINE) {
249 while (v[0] != NULL_CHAR) {
250 if (is_bold (v, "PRAGMAT") && delim == BOLD_PRAGMAT_SYMBOL) {
251 char *w = &v[strlen(qstrop ? "'PRAGMAT'" : "PRAGMAT")];
252 if (!IS_UPPER(w[0])) {
253 *top = u;
254 *ch = w;
255 return A68G_TRUE;
256 }
257 } else if (is_bold (v, "PR") && delim == STYLE_I_PRAGMAT_SYMBOL) {
258 char *w = &v[strlen(qstrop ? "'PR'" : "PR")];
259 if (!IS_UPPER(w[0])) {
260 *top = u;
261 *ch = w;
262 return A68G_TRUE;
263 }
264 } else {
265 if (whitespace && !IS_SPACE (v[0]) && v[0] != NEWLINE_CHAR) {
266 scan_error (u, v, ERROR_PRAGMENT);
267 } else if (IS_UPPER (v[0])) {
268 // Skip a bold word as you may trigger on REPR, for instance ...
269 while (IS_UPPER (v[0])) {
270 v++;
271 }
272 } else {
273 v++;
274 }
275 }
276 }
277 FORWARD (u);
278 if (u != NO_LINE) {
279 v = &(STRING (u)[0]);
280 } else {
281 v = NO_TEXT;
282 }
283 }
284 return A68G_FALSE;
285 }
286
287 //! @brief Return pointer to next token within pragmat.
288
289 char *get_pragmat_item (LINE_T ** top, char **ch)
290 {
291 LINE_T *u = *top;
292 char *v = *ch;
293 while (u != NO_LINE) {
294 while (v[0] != NULL_CHAR) {
295 if (!IS_SPACE (v[0]) && v[0] != NEWLINE_CHAR) {
296 *top = u;
297 *ch = v;
298 return v;
299 } else {
300 v++;
301 }
302 }
303 FORWARD (u);
304 if (u != NO_LINE) {
305 v = &(STRING (u)[0]);
306 } else {
307 v = NO_TEXT;
308 }
309 }
310 return NO_TEXT;
311 }
312
313 //! @brief Case insensitive strncmp for at most the number of chars in 'v'.
314
315 int streq (char *u, char *v)
316 {
317 int diff;
318 for (diff = 0; diff == 0 && u[0] != NULL_CHAR && v[0] != NULL_CHAR; u++, v++) {
319 diff = ((int) TO_LOWER (u[0])) - ((int) TO_LOWER (v[0]));
320 }
321 return diff;
322 }
323
324 //! @brief Scan for next pragmat and yield first pragmat item.
325
326 char *next_preprocessor_item (LINE_T ** top, char **ch, int *delim)
327 {
328 LINE_T *u = *top;
329 char *v = *ch;
330 *delim = 0;
331 while (u != NO_LINE) {
332 while (v[0] != NULL_CHAR) {
333 LINE_T *start_l = u;
334 char *start_c = v;
335 // STRINGs must be skipped.
336 if (v[0] == QUOTE_CHAR) {
337 SCAN_ERROR (!skip_string (&u, &v), start_l, start_c, ERROR_UNTERMINATED_STRING);
338 }
339 // COMMENTS must be skipped.
340 else if (is_bold (v, "COMMENT")) {
341 SCAN_ERROR (!skip_comment (&u, &v, BOLD_COMMENT_SYMBOL), start_l, start_c, ERROR_UNTERMINATED_COMMENT);
342 } else if (is_bold (v, "CO")) {
343 SCAN_ERROR (!skip_comment (&u, &v, STYLE_I_COMMENT_SYMBOL), start_l, start_c, ERROR_UNTERMINATED_COMMENT);
344 } else if (v[0] == '#') {
345 SCAN_ERROR (!skip_comment (&u, &v, STYLE_II_COMMENT_SYMBOL), start_l, start_c, ERROR_UNTERMINATED_COMMENT);
346 } else if (is_bold (v, "PRAGMAT") || is_bold (v, "PR")) {
347 // We caught a PRAGMAT.
348 char *item;
349 if (is_bold (v, "PRAGMAT")) {
350 *delim = BOLD_PRAGMAT_SYMBOL;
351 v = &v[strlen ("PRAGMAT")];
352 } else if (is_bold (v, "PR")) {
353 *delim = STYLE_I_PRAGMAT_SYMBOL;
354 v = &v[strlen ("PR")];
355 }
356 item = get_pragmat_item (&u, &v);
357 SCAN_ERROR (item == NO_TEXT, start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
358 // Item "preprocessor" restarts preprocessing if it is off.
359 if (A68G_PARSER (no_preprocessing) && streq (item, "PREPROCESSOR") == 0) {
360 A68G_PARSER (no_preprocessing) = A68G_FALSE;
361 SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68G_TRUE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
362 }
363 // If preprocessing is switched off, we idle to closing bracket.
364 else if (A68G_PARSER (no_preprocessing)) {
365 SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68G_FALSE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
366 }
367 // Item "nopreprocessor" stops preprocessing if it is on.
368 if (streq (item, "NOPREPROCESSOR") == 0) {
369 A68G_PARSER (no_preprocessing) = A68G_TRUE;
370 SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68G_TRUE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
371 }
372 // Item "INCLUDE" includes a file.
373 else if (streq (item, "INCLUDE") == 0) {
374 *top = u;
375 *ch = v;
376 return item;
377 }
378 // Item "READ" includes a file.
379 else if (streq (item, "READ") == 0) {
380 *top = u;
381 *ch = v;
382 return item;
383 }
384 // Unrecognised item - probably options handled later by the tokeniser.
385 else {
386 SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68G_FALSE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
387 }
388 } else if (IS_UPPER (v[0])) {
389 // Skip a bold word as you may trigger on REPR, for instance ...
390 while (IS_UPPER (v[0])) {
391 v++;
392 }
393 } else {
394 v++;
395 }
396 }
397 FORWARD (u);
398 if (u != NO_LINE) {
399 v = &(STRING (u)[0]);
400 } else {
401 v = NO_TEXT;
402 }
403 }
404 *top = u;
405 *ch = v;
406 return NO_TEXT;
407 }
408
409 //! @brief Include files.
410
411 void include_files (LINE_T * top)
412 {
413 // include_files
414 //
415 // syntax: PR read "filename" PR
416 // PR include "filename" PR
417 //
418 // The file gets inserted before the line containing the pragmat. In this way
419 // correct line numbers are preserved which helps diagnostics. A file that has
420 // been included will not be included a second time - it will be ignored.
421 // A rigorous fail-safe, but there is no mechanism to prevent recursive includes
422 // in A68 source code. User reports do not indicate sophisticated use of INCLUDE,
423 // so this is fine for now.
424 // TODO - some day we might need `app', analogous to `cpp'.
425 BOOL_T make_pass = A68G_TRUE;
426 while (make_pass) {
427 LINE_T *s, *t, *u = top;
428 char *v = &(STRING (u)[0]);
429 make_pass = A68G_FALSE;
430 errno = 0;
431 while (u != NO_LINE) {
432 int pr_lim;
433 char *item = next_preprocessor_item (&u, &v, &pr_lim);
434 LINE_T *start_l = u;
435 char *start_c = v;
436 // Search for PR include "filename" PR.
437 if (item != NO_TEXT && (streq (item, "INCLUDE") == 0 || streq (item, "READ") == 0)) {
438 FILE_T fd;
439 int n, line_num, fsize, k, bytes_read, fnwid;
440 char *fbuf, delim;
441 BUFFER fnb;
442 char *fn;
443 // Skip to filename.
444 while (IS_ALPHA (v[0])) {
445 v++;
446 }
447 while (IS_SPACE (v[0])) {
448 v++;
449 }
450 // Scan quoted filename.
451 SCAN_ERROR ((v[0] != QUOTE_CHAR && v[0] != '\''), start_l, start_c, ERROR_INCORRECT_FILENAME);
452 delim = (v++)[0];
453 n = 0;
454 fnb[0] = NULL_CHAR;
455 // Scan Algol 68 string (note: "" denotes a ", while in C it concatenates).
456 do {
457 SCAN_ERROR (EOL (v[0]), start_l, start_c, ERROR_INCORRECT_FILENAME);
458 SCAN_ERROR (n == BUFFER_SIZE - 1, start_l, start_c, ERROR_INCORRECT_FILENAME);
459 if (v[0] == delim) {
460 while (v[0] == delim && v[1] == delim) {
461 SCAN_ERROR (n == BUFFER_SIZE - 1, start_l, start_c, ERROR_INCORRECT_FILENAME);
462 fnb[n++] = delim;
463 fnb[n] = NULL_CHAR;
464 v += 2;
465 }
466 } else if (IS_PRINT (v[0])) {
467 fnb[n++] = *(v++);
468 fnb[n] = NULL_CHAR;
469 } else {
470 SCAN_ERROR (A68G_TRUE, start_l, start_c, ERROR_INCORRECT_FILENAME);
471 }
472 } while (v[0] != delim);
473 // Insist that the pragmat is closed properly.
474 v = &v[1];
475 SCAN_ERROR (!skip_pragmat (&u, &v, pr_lim, A68G_TRUE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
476 SCAN_ERROR (n == 0, start_l, start_c, ERROR_INCORRECT_FILENAME);
477 // Make the name relative to the position of the source file (C preprocessor standard).
478 if (FILENAME (u) != NO_TEXT) {
479 fn = a68g_relpath (a68g_dirname (FILENAME (u)), a68g_dirname (fnb), a68g_basename (fnb));
480 } else {
481 fn = a68g_relpath (FILE_PATH (&A68G_JOB), a68g_dirname (fnb), a68g_basename (fnb));
482 }
483 // Do not check errno, since errno may be undefined here after a successful call.
484 if (fn != NO_TEXT) {
485 a68g_bufcpy (fnb, fn, BUFFER_SIZE);
486 } else {
487 SCAN_ERROR_INFO (A68G_TRUE, NO_LINE, NO_TEXT, ERROR_SOURCE_FILE_INCLUDE_OPEN, fnb);
488 }
489 fnwid = (int) strlen (fnb) + 1;
490 fn = (char *) get_fixed_heap_space ((size_t) fnwid);
491 a68g_bufcpy (fn, fnb, fnwid);
492 // Ignore the file when included more than once.
493 for (t = top; t != NO_LINE; t = NEXT (t)) {
494 if (strcmp (FILENAME (t), fn) == 0) {
495 goto search_next_pragmat;
496 }
497 }
498 // Access the file.
499 errno = 0;
500 fd = open (fn, O_RDONLY | O_BINARY);
501 SCAN_ERROR_INFO (fd == -1, start_l, start_c, ERROR_SOURCE_FILE_INCLUDE_OPEN, fnb);
502 errno = 0;
503 fsize = (int) lseek (fd, 0, SEEK_END);
504 ASSERT (fsize >= 0);
505 SCAN_ERROR (errno != 0, start_l, start_c, ERROR_FILE_READ);
506 fbuf = (char *) get_temp_heap_space ((unt) (8 + fsize));
507 errno = 0;
508 ASSERT (lseek (fd, 0, SEEK_SET) >= 0);
509 SCAN_ERROR (errno != 0, start_l, start_c, ERROR_FILE_READ);
510 errno = 0;
511 bytes_read = (int) io_read (fd, fbuf, (size_t) fsize);
512 SCAN_ERROR (errno != 0 || bytes_read != fsize, start_l, start_c, ERROR_FILE_READ);
513 // Buffer still usable?.
514 if (fsize > A68G_PARSER (max_scan_buf_length)) {
515 A68G_PARSER (max_scan_buf_length) = fsize;
516 A68G_PARSER (scan_buf) = (char *) get_temp_heap_space ((unt) (8 + A68G_PARSER (max_scan_buf_length)));
517 }
518 // Link all lines into the list.
519 line_num = 1;
520 s = u;
521 t = PREVIOUS (u);
522 k = 0;
523 if (fsize == 0) {
524 // If file is empty, insert single empty line.
525 A68G_PARSER (scan_buf)[0] = NEWLINE_CHAR;
526 A68G_PARSER (scan_buf)[1] = NULL_CHAR;
527 append_source_line (A68G_PARSER (scan_buf), &t, &line_num, fn);
528 } else
529 while (k < fsize) {
530 n = 0;
531 A68G_PARSER (scan_buf)[0] = NULL_CHAR;
532 while (k < fsize && fbuf[k] != NEWLINE_CHAR) {
533 SCAN_ERROR ((IS_CNTRL (fbuf[k]) && !IS_SPACE (fbuf[k])) || fbuf[k] == STOP_CHAR, start_l, start_c, ERROR_FILE_INCLUDE_CTRL);
534 A68G_PARSER (scan_buf)[n++] = fbuf[k++];
535 A68G_PARSER (scan_buf)[n] = NULL_CHAR;
536 }
537 A68G_PARSER (scan_buf)[n++] = NEWLINE_CHAR;
538 A68G_PARSER (scan_buf)[n] = NULL_CHAR;
539 if (k < fsize) {
540 k++;
541 }
542 append_source_line (A68G_PARSER (scan_buf), &t, &line_num, fn);
543 }
544 // Conclude and go find another include directive, if any.
545 NEXT (t) = s;
546 PREVIOUS (s) = t;
547 concatenate_lines (top);
548 ASSERT (close (fd) == 0);
549 make_pass = A68G_TRUE;
550 }
551 search_next_pragmat:_SKIP_;
552 }
553 }
554 }
555
556 //! @brief Size of source file.
557
558 int get_source_size (void)
559 {
560 FILE_T f = FILE_SOURCE_FD (&A68G_JOB);
561 // This is why WIN32 must open as "read binary".
562 return (int) lseek (f, 0, SEEK_END);
563 }
564
565 //! @brief Append environment source lines.
566
567 void append_environ (char *str[], LINE_T ** ref_l, int *line_num, char *name)
568 {
569 for (int k = 0; str[k] != NO_TEXT; k++) {
570 int zero_line_num = 0;
571 (void) line_num;
572 append_source_line (str[k], ref_l, &zero_line_num, name);
573 }
574 }
575
576 //! @brief Read script file and make internal copy.
577
578 BOOL_T read_script_file (void)
579 {
580 LINE_T *ref_l = NO_LINE;
581 int k, n, num;
582 unt len;
583 BOOL_T file_end = A68G_FALSE;
584 BUFFER filename, linenum;
585 char ch, *fn, *line;
586 char *buffer = (char *) get_temp_heap_space ((unt) (8 + A68G_PARSER (source_file_size)));
587 FILE_T source = FILE_SOURCE_FD (&A68G_JOB);
588 ABEND (source == -1, ERROR_ACTION, __func__);
589 buffer[0] = NULL_CHAR;
590 n = 0;
591 len = (unt) (8 + A68G_PARSER (source_file_size));
592 buffer = (char *) get_temp_heap_space (len);
593 ASSERT (lseek (source, 0, SEEK_SET) >= 0);
594 while (!file_end) {
595 // Read the original file name.
596 filename[0] = NULL_CHAR;
597 k = 0;
598 if (io_read (source, &ch, 1) == 0) {
599 file_end = A68G_TRUE;
600 continue;
601 }
602 while (ch != NEWLINE_CHAR) {
603 filename[k++] = ch;
604 ASSERT (io_read (source, &ch, 1) == 1);
605 }
606 filename[k] = NULL_CHAR;
607 fn = TEXT (add_token (&A68G (top_token), filename));
608 // Read the original file number.
609 linenum[0] = NULL_CHAR;
610 k = 0;
611 ASSERT (io_read (source, &ch, 1) == 1);
612 while (ch != NEWLINE_CHAR) {
613 linenum[k++] = ch;
614 ASSERT (io_read (source, &ch, 1) == 1);
615 }
616 linenum[k] = NULL_CHAR;
617 num = (int) strtol (linenum, NO_REF, 10);
618 ABEND (errno == ERANGE, ERROR_INTERNAL_CONSISTENCY, __func__);
619 // COPY original line into buffer.
620 ASSERT (io_read (source, &ch, 1) == 1);
621 line = &buffer[n];
622 while (ch != NEWLINE_CHAR) {
623 buffer[n++] = ch;
624 ASSERT (io_read (source, &ch, 1) == 1);
625 ABEND ((unt) n >= len, ERROR_ACTION, __func__);
626 }
627 buffer[n++] = NEWLINE_CHAR;
628 buffer[n] = NULL_CHAR;
629 append_source_line (line, &ref_l, &num, fn);
630 }
631 return A68G_TRUE;
632 }
633
634 //! @brief match first non-white characters in string.
635
636 BOOL_T a68g_start(char *u, char *v, char **end)
637 {
638 *end = NO_TEXT;
639 while (v[0] != NULL_CHAR) {
640 if (u[0] == NULL_CHAR) {
641 return A68G_FALSE;
642 } else if (IS_SPACE (u[0])) {
643 u++;
644 } else {
645 if (u[0] == v[0]) {
646 u++;
647 v++;
648 if (end != NULL) {
649 *end = u;
650 }
651 } else {
652 return A68G_FALSE;
653 }
654 }
655 }
656 return A68G_TRUE;
657 }
658
659 //! @brief Read source file and make internal copy.
660
661 BOOL_T read_source_file (void)
662 {
663 LINE_T *ref_l = NO_LINE;
664 int line_num = 0, k, bytes_read;
665 ssize_t l;
666 FILE_T f = FILE_SOURCE_FD (&A68G_JOB);
667 char **prelude_start, **postlude, *buffer, *text;
668 // Read the file into a single buffer, so we save on system calls.
669 line_num = 1;
670 errno = 0;
671 text = (char *) get_temp_heap_space ((unt) (8 + A68G_PARSER (source_file_size)));
672 ABEND (errno != 0 || text == NO_TEXT, ERROR_ALLOCATION, __func__);
673 ASSERT (lseek (f, 0, SEEK_SET) >= 0);
674 ABEND (errno != 0, ERROR_ACTION, __func__);
675 errno = 0;
676 bytes_read = (int) io_read (f, text, (size_t) A68G_PARSER (source_file_size));
677 ABEND (errno != 0 || bytes_read != A68G_PARSER (source_file_size), ERROR_ACTION, __func__);
678 // Little test on stropping.
679 char *pr1 = "'PR'QUOTESTROPPING'PR'";
680 char *pr2 = "'PRAGMAT'QUOTESTROPPING'PRAGMAT'";
681 char *end = NO_TEXT;
682 if (a68g_start (text, pr1, &end)) {
683 OPTION_STROPPING (&A68G_JOB) = QUOTE_STROPPING;
684 buffer = end;
685 A68G_PARSER (source_file_size) = strlen (buffer);
686 } else if (a68g_start (text, pr2, &end)) {
687 OPTION_STROPPING (&A68G_JOB) = QUOTE_STROPPING;
688 buffer = end;
689 A68G_PARSER (source_file_size) = strlen (buffer);
690 } else {
691 buffer = text;
692 }
693 // Prelude.
694 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
695 prelude_start = bold_prelude_start;
696 postlude = bold_postlude;
697 } else if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
698 prelude_start = quote_prelude_start;
699 postlude = quote_postlude;
700 } else {
701 prelude_start = postlude = NO_REF;
702 }
703 append_environ (prelude_start, &ref_l, &line_num, "prelude");
704 // Link all lines into the list.
705 k = 0;
706 while (k < A68G_PARSER (source_file_size)) {
707 l = 0;
708 A68G_PARSER (scan_buf)[0] = NULL_CHAR;
709 while (k < A68G_PARSER (source_file_size) && buffer[k] != NEWLINE_CHAR) {
710 if (k < A68G_PARSER (source_file_size) - 1 && buffer[k] == CR_CHAR && buffer[k + 1] == NEWLINE_CHAR) {
711 k++;
712 } else {
713 A68G_PARSER (scan_buf)[l++] = buffer[k++];
714 A68G_PARSER (scan_buf)[l] = NULL_CHAR;
715 }
716 }
717 A68G_PARSER (scan_buf)[l++] = NEWLINE_CHAR;
718 A68G_PARSER (scan_buf)[l] = NULL_CHAR;
719 if (k < A68G_PARSER (source_file_size)) {
720 k++;
721 }
722 append_source_line (A68G_PARSER (scan_buf), &ref_l, &line_num, FILE_SOURCE_NAME (&A68G_JOB));
723 SCAN_ERROR (l != (ssize_t) strlen (A68G_PARSER (scan_buf)), NO_LINE, NO_TEXT, ERROR_FILE_SOURCE_CTRL);
724 }
725 // Postlude.
726 append_environ (postlude, &ref_l, &line_num, "postlude");
727 // Concatenate lines.
728 concatenate_lines (TOP_LINE (&A68G_JOB));
729 // Include files.
730 include_files (TOP_LINE (&A68G_JOB));
731 return A68G_TRUE;
732 }
733
734 //! @brief Next_char get next character from internal copy of source file.
735
736 char next_char (LINE_T ** ref_l, char **ref_s, BOOL_T allow_typo)
737 {
738 char ch;
739 #if defined (NO_TYPO)
740 allow_typo = A68G_FALSE;
741 #endif
742 LOW_STACK_ALERT (NO_NODE);
743 // Source empty?.
744 if (*ref_l == NO_LINE) {
745 return STOP_CHAR;
746 } else {
747 LIST (*ref_l) = (BOOL_T) (OPTION_NODEMASK (&A68G_JOB) & SOURCE_MASK ? A68G_TRUE : A68G_FALSE);
748 // Take new line?.
749 if ((*ref_s)[0] == NEWLINE_CHAR || (*ref_s)[0] == NULL_CHAR) {
750 *ref_l = NEXT (*ref_l);
751 if (*ref_l == NO_LINE) {
752 return STOP_CHAR;
753 }
754 *ref_s = STRING (*ref_l);
755 } else {
756 (*ref_s)++;
757 }
758 // Deliver next char.
759 ch = (*ref_s)[0];
760 if (allow_typo && (IS_SPACE (ch) || ch == FORMFEED_CHAR)) {
761 ch = next_char (ref_l, ref_s, allow_typo);
762 }
763 return ch;
764 }
765 }
766
767 //! @brief Find first character that can start a valid symbol.
768
769 void get_good_char (char *ref_c, LINE_T ** ref_l, char **ref_s)
770 {
771 while (*ref_c != STOP_CHAR && (IS_SPACE (*ref_c) || (*ref_c == NULL_CHAR))) {
772 if (*ref_l != NO_LINE) {
773 LIST (*ref_l) = (BOOL_T) (OPTION_NODEMASK (&A68G_JOB) & SOURCE_MASK ? A68G_TRUE : A68G_FALSE);
774 }
775 *ref_c = next_char (ref_l, ref_s, A68G_FALSE);
776 }
777 }
778
779 //! @brief Handle a pragment (pragmat or comment).
780
781 char *pragment (int type, LINE_T ** ref_l, char **ref_c)
782 {
783 #define INIT_BUFFER {chars_in_buf = 0; A68G_PARSER (scan_buf)[chars_in_buf] = NULL_CHAR;}
784 #define ADD_ONE_CHAR(ch) {A68G_PARSER (scan_buf)[chars_in_buf ++] = ch; A68G_PARSER (scan_buf)[chars_in_buf] = NULL_CHAR;}
785 char c = **ref_c, *term_s = NO_TEXT, *start_c = *ref_c;
786 char *z = NO_TEXT;
787 LINE_T *start_l = *ref_l;
788 int term_s_length, chars_in_buf;
789 BOOL_T stop, pragmat = A68G_FALSE;
790 // Set terminator.
791 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
792 if (type == STYLE_I_COMMENT_SYMBOL) {
793 term_s = "CO";
794 } else if (type == STYLE_II_COMMENT_SYMBOL) {
795 term_s = "#";
796 } else if (type == BOLD_COMMENT_SYMBOL) {
797 term_s = "COMMENT";
798 } else if (type == STYLE_I_PRAGMAT_SYMBOL) {
799 term_s = "PR";
800 pragmat = A68G_TRUE;
801 } else if (type == BOLD_PRAGMAT_SYMBOL) {
802 term_s = "PRAGMAT";
803 pragmat = A68G_TRUE;
804 }
805 } else if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
806 if (type == STYLE_I_COMMENT_SYMBOL) {
807 term_s = "'CO'";
808 } else if (type == STYLE_II_COMMENT_SYMBOL) {
809 term_s = "#";
810 } else if (type == BOLD_COMMENT_SYMBOL) {
811 term_s = "'COMMENT'";
812 } else if (type == STYLE_I_PRAGMAT_SYMBOL) {
813 term_s = "'PR'";
814 pragmat = A68G_TRUE;
815 } else if (type == BOLD_PRAGMAT_SYMBOL) {
816 term_s = "'PRAGMAT'";
817 pragmat = A68G_TRUE;
818 }
819 }
820 term_s_length = (int) strlen (term_s);
821 // Scan for terminator.
822 INIT_BUFFER;
823 stop = A68G_FALSE;
824 while (stop == A68G_FALSE) {
825 BOOL_T scan_next = A68G_TRUE;
826 SCAN_ERROR (c == STOP_CHAR, start_l, start_c, ERROR_UNTERMINATED_PRAGMENT);
827 // A ".." or '..' delimited string in a PRAGMAT.
828 if (pragmat && (c == QUOTE_CHAR || (c == '\'' && OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING))) {
829 char delim = c;
830 BOOL_T eos = A68G_FALSE;
831 ADD_ONE_CHAR (c);
832 c = next_char (ref_l, ref_c, A68G_FALSE);
833 while (!eos) {
834 SCAN_ERROR (EOL (c), start_l, start_c, ERROR_LONG_STRING);
835 if (c == delim) {
836 ADD_ONE_CHAR (delim);
837 save_state (*ref_l, *ref_c, c);
838 c = next_char (ref_l, ref_c, A68G_FALSE);
839 if (c == delim) {
840 c = next_char (ref_l, ref_c, A68G_FALSE);
841 } else {
842 restore_state (ref_l, ref_c, &c);
843 eos = A68G_TRUE;
844 }
845 } else if (IS_PRINT (c)) {
846 ADD_ONE_CHAR (c);
847 c = next_char (ref_l, ref_c, A68G_FALSE);
848 } else {
849 unworthy (start_l, start_c, c);
850 }
851 }
852 } else if (EOL (c)) {
853 ADD_ONE_CHAR (NEWLINE_CHAR);
854 } else if (IS_UPPER (c)) {
855 while (IS_UPPER (c)) {
856 ADD_ONE_CHAR (c);
857 c = next_char (ref_l, ref_c, A68G_FALSE);
858 }
859 scan_next = A68G_FALSE;
860 } else if (IS_PRINT (c) || IS_SPACE (c)) {
861 ADD_ONE_CHAR (c);
862 }
863 if (chars_in_buf >= term_s_length) {
864 // Check whether we encountered the terminator.
865 char *tok = &(A68G_PARSER (scan_buf)[chars_in_buf - term_s_length]);
866 stop = (BOOL_T) (strcmp (term_s, tok) == 0);
867 }
868 if (scan_next) {
869 c = next_char (ref_l, ref_c, A68G_FALSE);
870 }
871 }
872 A68G_PARSER (scan_buf)[chars_in_buf - term_s_length] = NULL_CHAR;
873 z = new_string (term_s, A68G_PARSER (scan_buf), term_s, NO_TEXT);
874 if (type == STYLE_I_PRAGMAT_SYMBOL || type == BOLD_PRAGMAT_SYMBOL) {
875 isolate_options (A68G_PARSER (scan_buf), start_l);
876 }
877 return z;
878 #undef ADD_ONE_CHAR
879 #undef INIT_BUFFER
880 }
881
882 //! @brief Attribute for format item.
883
884 int get_format_item (char ch)
885 {
886 switch (TO_LOWER (ch)) {
887 case 'a': {
888 return FORMAT_ITEM_A;
889 }
890 case 'b': {
891 return FORMAT_ITEM_B;
892 }
893 case 'c': {
894 return FORMAT_ITEM_C;
895 }
896 case 'd': {
897 return FORMAT_ITEM_D;
898 }
899 case 'e': {
900 return FORMAT_ITEM_E;
901 }
902 case 'f': {
903 return FORMAT_ITEM_F;
904 }
905 case 'g': {
906 return FORMAT_ITEM_G;
907 }
908 case 'h': {
909 return FORMAT_ITEM_H;
910 }
911 case 'i': {
912 return FORMAT_ITEM_I;
913 }
914 case 'j': {
915 return FORMAT_ITEM_J;
916 }
917 case 'k': {
918 return FORMAT_ITEM_K;
919 }
920 case 'l':
921 case '/': {
922 return FORMAT_ITEM_L;
923 }
924 case 'm': {
925 return FORMAT_ITEM_M;
926 }
927 case 'n': {
928 return FORMAT_ITEM_N;
929 }
930 case 'o': {
931 return FORMAT_ITEM_O;
932 }
933 case 'p': {
934 return FORMAT_ITEM_P;
935 }
936 case 'q': {
937 return FORMAT_ITEM_Q;
938 }
939 case 'r': {
940 return FORMAT_ITEM_R;
941 }
942 case 's': {
943 return FORMAT_ITEM_S;
944 }
945 case 't': {
946 return FORMAT_ITEM_T;
947 }
948 case 'u': {
949 return FORMAT_ITEM_U;
950 }
951 case 'v': {
952 return FORMAT_ITEM_V;
953 }
954 case 'w': {
955 return FORMAT_ITEM_W;
956 }
957 case 'x': {
958 return FORMAT_ITEM_X;
959 }
960 case 'y': {
961 return FORMAT_ITEM_Y;
962 }
963 case 'z': {
964 return FORMAT_ITEM_Z;
965 }
966 case '+': {
967 return FORMAT_ITEM_PLUS;
968 }
969 case '-': {
970 return FORMAT_ITEM_MINUS;
971 }
972 case POINT_CHAR: {
973 return FORMAT_ITEM_POINT;
974 }
975 case '%': {
976 return FORMAT_ITEM_ESCAPE;
977 }
978 default: {
979 return 0;
980 }
981 }
982 }
983
984 //! @brief Whether input shows exponent character.
985
986 BOOL_T is_exp_char (LINE_T ** ref_l, char **ref_s, char *ch)
987 {
988 BOOL_T ret = A68G_FALSE;
989 char exp_syms[3];
990 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
991 exp_syms[0] = EXPONENT_CHAR;
992 exp_syms[1] = (char) TO_UPPER (EXPONENT_CHAR);
993 exp_syms[2] = NULL_CHAR;
994 } else {
995 exp_syms[0] = (char) TO_UPPER (EXPONENT_CHAR);
996 exp_syms[1] = BACKSLASH_CHAR;
997 exp_syms[2] = NULL_CHAR;
998 }
999 save_state (*ref_l, *ref_s, *ch);
1000 if (strchr (exp_syms, *ch) != NO_TEXT) {
1001 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1002 ret = (BOOL_T) (strchr ("+-0123456789", *ch) != NO_TEXT);
1003 }
1004 restore_state (ref_l, ref_s, ch);
1005 return ret;
1006 }
1007
1008 //! @brief Whether input shows radix character.
1009
1010 BOOL_T is_radix_char (LINE_T ** ref_l, char **ref_s, char *ch)
1011 {
1012 BOOL_T ret = A68G_FALSE;
1013 save_state (*ref_l, *ref_s, *ch);
1014 if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
1015 if (*ch == TO_UPPER (RADIX_CHAR)) {
1016 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1017 ret = (BOOL_T) (strchr ("0123456789ABCDEF", *ch) != NO_TEXT);
1018 }
1019 } else {
1020 if (*ch == RADIX_CHAR) {
1021 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1022 ret = (BOOL_T) (strchr ("0123456789abcdef", *ch) != NO_TEXT);
1023 }
1024 }
1025 restore_state (ref_l, ref_s, ch);
1026 return ret;
1027 }
1028
1029 //! @brief Whether input shows decimal point.
1030
1031 BOOL_T is_decimal_point (LINE_T ** ref_l, char **ref_s, char *ch)
1032 {
1033 BOOL_T ret = A68G_FALSE;
1034 save_state (*ref_l, *ref_s, *ch);
1035 if (*ch == POINT_CHAR) {
1036 char exp_syms[3];
1037 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
1038 exp_syms[0] = EXPONENT_CHAR;
1039 exp_syms[1] = (char) TO_UPPER (EXPONENT_CHAR);
1040 exp_syms[2] = NULL_CHAR;
1041 } else {
1042 exp_syms[0] = (char) TO_UPPER (EXPONENT_CHAR);
1043 exp_syms[1] = BACKSLASH_CHAR;
1044 exp_syms[2] = NULL_CHAR;
1045 }
1046 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1047 if (strchr (exp_syms, *ch) != NO_TEXT) {
1048 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1049 ret = (BOOL_T) (strchr ("+-0123456789", *ch) != NO_TEXT);
1050 } else {
1051 ret = (BOOL_T) (strchr ("0123456789", *ch) != NO_TEXT);
1052 }
1053 }
1054 restore_state (ref_l, ref_s, ch);
1055 return ret;
1056 }
1057
1058 //! @brief Get next token from internal copy of source file..
1059
1060 void get_next_token (BOOL_T in_format, LINE_T ** ref_l, char **ref_s, LINE_T ** start_l, char **start_c, int *att)
1061 {
1062 char c = **ref_s, *sym = A68G_PARSER (scan_buf);
1063 sym[0] = NULL_CHAR;
1064 get_good_char (&c, ref_l, ref_s);
1065 *start_l = *ref_l;
1066 *start_c = *ref_s;
1067 if (c == STOP_CHAR) {
1068 // We are at EOF.
1069 (sym++)[0] = STOP_CHAR;
1070 sym[0] = NULL_CHAR;
1071 return;
1072 }
1073 // In a format.
1074 if (in_format) {
1075 char *format_items;
1076 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
1077 format_items = "/%\\+-.abcdefghijklmnopqrstuvwxyz";
1078 } else if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
1079 format_items = "/%\\+-.ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1080 } else {
1081 format_items = "/%\\+-.abcdefghijklmnopqrstuvwxyz";
1082 }
1083 if (strchr (format_items, c) != NO_TEXT) {
1084 // General format items.
1085 (sym++)[0] = c;
1086 sym[0] = NULL_CHAR;
1087 *att = get_format_item (c);
1088 (void) next_char (ref_l, ref_s, A68G_FALSE);
1089 return;
1090 }
1091 if (IS_DIGIT (c)) {
1092 // INT denotation for static replicator.
1093 SCAN_DIGITS (c);
1094 sym[0] = NULL_CHAR;
1095 *att = STATIC_REPLICATOR;
1096 return;
1097 }
1098 }
1099 // Not in a format.
1100 if (IS_UPPER (c)) {
1101 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
1102 // Upper case word - bold tag.
1103 while (IS_UPPER (c) || c == '_') {
1104 (sym++)[0] = c;
1105 c = next_char (ref_l, ref_s, A68G_FALSE);
1106 }
1107 sym[0] = NULL_CHAR;
1108 *att = BOLD_TAG;
1109 } else if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
1110 while (IS_UPPER (c) || IS_DIGIT (c) || c == '_') {
1111 (sym++)[0] = c;
1112 c = next_char (ref_l, ref_s, A68G_TRUE);
1113 }
1114 sym[0] = NULL_CHAR;
1115 *att = IDENTIFIER;
1116 }
1117 } else if (c == '\'') {
1118 // Quote, uppercase word, quote - bold tag.
1119 int k = 0;
1120 c = next_char (ref_l, ref_s, A68G_FALSE);
1121 while (IS_UPPER (c) || IS_DIGIT (c) || c == '_') {
1122 (sym++)[0] = c;
1123 k++;
1124 c = next_char (ref_l, ref_s, A68G_TRUE);
1125 }
1126 SCAN_ERROR (k == 0, *start_l, *start_c, ERROR_QUOTED_BOLD_TAG);
1127 sym[0] = NULL_CHAR;
1128 *att = BOLD_TAG;
1129 // Skip terminating quote, or complain if it is not there.
1130 SCAN_ERROR (c != '\'', *start_l, *start_c, ERROR_QUOTED_BOLD_TAG);
1131 c = next_char (ref_l, ref_s, A68G_FALSE);
1132 } else if (IS_LOWER (c)) {
1133 // Lower case word - identifier.
1134 while (IS_LOWER (c) || IS_DIGIT (c) || c == '_') {
1135 (sym++)[0] = c;
1136 c = next_char (ref_l, ref_s, A68G_TRUE);
1137 }
1138 sym[0] = NULL_CHAR;
1139 *att = IDENTIFIER;
1140 } else if (c == POINT_CHAR) {
1141 // Begins with a point symbol - point, dotdot, L REAL denotation.
1142 if (is_decimal_point (ref_l, ref_s, &c)) {
1143 (sym++)[0] = '0';
1144 (sym++)[0] = POINT_CHAR;
1145 c = next_char (ref_l, ref_s, A68G_TRUE);
1146 SCAN_DIGITS (c);
1147 if (is_exp_char (ref_l, ref_s, &c)) {
1148 SCAN_EXPONENT_PART (c);
1149 }
1150 sym[0] = NULL_CHAR;
1151 *att = REAL_DENOTATION;
1152 } else {
1153 c = next_char (ref_l, ref_s, A68G_TRUE);
1154 if (c == POINT_CHAR) {
1155 (sym++)[0] = POINT_CHAR;
1156 (sym++)[0] = POINT_CHAR;
1157 sym[0] = NULL_CHAR;
1158 *att = DOTDOT_SYMBOL;
1159 c = next_char (ref_l, ref_s, A68G_FALSE);
1160 } else {
1161 (sym++)[0] = POINT_CHAR;
1162 sym[0] = NULL_CHAR;
1163 *att = POINT_SYMBOL;
1164 }
1165 }
1166 } else if (IS_DIGIT (c)) {
1167 // Something that begins with a digit - L INT denotation, L REAL denotation.
1168 SCAN_DIGITS (c);
1169 if (is_decimal_point (ref_l, ref_s, &c)) {
1170 c = next_char (ref_l, ref_s, A68G_TRUE);
1171 if (is_exp_char (ref_l, ref_s, &c)) {
1172 (sym++)[0] = POINT_CHAR;
1173 (sym++)[0] = '0';
1174 SCAN_EXPONENT_PART (c);
1175 *att = REAL_DENOTATION;
1176 } else {
1177 (sym++)[0] = POINT_CHAR;
1178 SCAN_DIGITS (c);
1179 if (is_exp_char (ref_l, ref_s, &c)) {
1180 SCAN_EXPONENT_PART (c);
1181 }
1182 *att = REAL_DENOTATION;
1183 }
1184 } else if (is_exp_char (ref_l, ref_s, &c)) {
1185 SCAN_EXPONENT_PART (c);
1186 *att = REAL_DENOTATION;
1187 } else if (is_radix_char (ref_l, ref_s, &c)) {
1188 (sym++)[0] = c;
1189 c = next_char (ref_l, ref_s, A68G_TRUE);
1190 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
1191 while (IS_DIGIT (c) || strchr ("abcdef", c) != NO_TEXT) {
1192 (sym++)[0] = c;
1193 c = next_char (ref_l, ref_s, A68G_TRUE);
1194 }
1195 } else {
1196 while (IS_DIGIT (c) || strchr ("ABCDEF", c) != NO_TEXT) {
1197 (sym++)[0] = c;
1198 c = next_char (ref_l, ref_s, A68G_TRUE);
1199 }
1200 }
1201 *att = BITS_DENOTATION;
1202 } else {
1203 *att = INT_DENOTATION;
1204 }
1205 sym[0] = NULL_CHAR;
1206 } else if (c == QUOTE_CHAR) {
1207 // STRING denotation.
1208 BOOL_T stop = A68G_FALSE;
1209 while (!stop) {
1210 c = next_char (ref_l, ref_s, A68G_FALSE);
1211 while (c != QUOTE_CHAR && c != STOP_CHAR) {
1212 SCAN_ERROR (EOL (c), *start_l, *start_c, ERROR_LONG_STRING);
1213 (sym++)[0] = c;
1214 c = next_char (ref_l, ref_s, A68G_FALSE);
1215 }
1216 SCAN_ERROR (*ref_l == NO_LINE, *start_l, *start_c, ERROR_UNTERMINATED_STRING);
1217 c = next_char (ref_l, ref_s, A68G_FALSE);
1218 if (c == QUOTE_CHAR) {
1219 (sym++)[0] = QUOTE_CHAR;
1220 } else {
1221 stop = A68G_TRUE;
1222 }
1223 }
1224 sym[0] = NULL_CHAR;
1225 *att = (in_format ? LITERAL : ROW_CHAR_DENOTATION);
1226 } else if (strchr ("#$()[]{},;@", c) != NO_TEXT) {
1227 // Single character symbols.
1228 (sym++)[0] = c;
1229 (void) next_char (ref_l, ref_s, A68G_FALSE);
1230 sym[0] = NULL_CHAR;
1231 *att = 0;
1232 } else if (c == '|') {
1233 // Bar.
1234 (sym++)[0] = c;
1235 c = next_char (ref_l, ref_s, A68G_FALSE);
1236 if (c == ':') {
1237 (sym++)[0] = c;
1238 (void) next_char (ref_l, ref_s, A68G_FALSE);
1239 }
1240 sym[0] = NULL_CHAR;
1241 *att = 0;
1242 } else if (c == '!' && OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
1243 // Bar, will be replaced with modern variant.
1244 // For this reason ! is not a MONAD with quote-stropping.
1245 (sym++)[0] = '|';
1246 c = next_char (ref_l, ref_s, A68G_FALSE);
1247 if (c == ':') {
1248 (sym++)[0] = c;
1249 (void) next_char (ref_l, ref_s, A68G_FALSE);
1250 }
1251 sym[0] = NULL_CHAR;
1252 *att = 0;
1253 } else if (c == ':') {
1254 // Colon, semicolon, IS, ISNT.
1255 (sym++)[0] = c;
1256 c = next_char (ref_l, ref_s, A68G_FALSE);
1257 if (c == '=') {
1258 (sym++)[0] = c;
1259 if ((c = next_char (ref_l, ref_s, A68G_FALSE)) == ':') {
1260 (sym++)[0] = c;
1261 c = next_char (ref_l, ref_s, A68G_FALSE);
1262 }
1263 } else if (c == '/') {
1264 (sym++)[0] = c;
1265 if ((c = next_char (ref_l, ref_s, A68G_FALSE)) == '=') {
1266 (sym++)[0] = c;
1267 if ((c = next_char (ref_l, ref_s, A68G_FALSE)) == ':') {
1268 (sym++)[0] = c;
1269 c = next_char (ref_l, ref_s, A68G_FALSE);
1270 }
1271 }
1272 } else if (c == ':') {
1273 (sym++)[0] = c;
1274 if ((c = next_char (ref_l, ref_s, A68G_FALSE)) == '=') {
1275 (sym++)[0] = c;
1276 }
1277 }
1278 sym[0] = NULL_CHAR;
1279 *att = 0;
1280 } else if (c == '=') {
1281 // Operator starting with "=".
1282 char *scanned = sym;
1283 (sym++)[0] = c;
1284 c = next_char (ref_l, ref_s, A68G_FALSE);
1285 if (strchr (NOMADS, c) != NO_TEXT) {
1286 (sym++)[0] = c;
1287 c = next_char (ref_l, ref_s, A68G_FALSE);
1288 }
1289 if (c == '=') {
1290 (sym++)[0] = c;
1291 if (next_char (ref_l, ref_s, A68G_FALSE) == ':') {
1292 (sym++)[0] = ':';
1293 c = next_char (ref_l, ref_s, A68G_FALSE);
1294 if (strlen (sym) < 4 && c == '=') {
1295 (sym++)[0] = '=';
1296 (void) next_char (ref_l, ref_s, A68G_FALSE);
1297 }
1298 }
1299 } else if (c == ':') {
1300 (sym++)[0] = c;
1301 sym[0] = NULL_CHAR;
1302 if (next_char (ref_l, ref_s, A68G_FALSE) == '=') {
1303 (sym++)[0] = '=';
1304 (void) next_char (ref_l, ref_s, A68G_FALSE);
1305 } else {
1306 SCAN_ERROR (!(strcmp (scanned, "=:") == 0 || strcmp (scanned, "==:") == 0), *start_l, *start_c, ERROR_INVALID_OPERATOR_TAG);
1307 }
1308 }
1309 sym[0] = NULL_CHAR;
1310 if (strcmp (scanned, "=") == 0) {
1311 *att = EQUALS_SYMBOL;
1312 } else {
1313 *att = OPERATOR;
1314 }
1315 } else if (strchr (MONADS, c) != NO_TEXT || strchr (NOMADS, c) != NO_TEXT) {
1316 // Operator.
1317 char *scanned = sym;
1318 (sym++)[0] = c;
1319 c = next_char (ref_l, ref_s, A68G_FALSE);
1320 if (strchr (NOMADS, c) != NO_TEXT) {
1321 (sym++)[0] = c;
1322 c = next_char (ref_l, ref_s, A68G_FALSE);
1323 }
1324 if (c == '=') {
1325 (sym++)[0] = c;
1326 if (next_char (ref_l, ref_s, A68G_FALSE) == ':') {
1327 (sym++)[0] = ':';
1328 c = next_char (ref_l, ref_s, A68G_FALSE);
1329 if (strlen (scanned) < 4 && c == '=') {
1330 (sym++)[0] = '=';
1331 (void) next_char (ref_l, ref_s, A68G_FALSE);
1332 }
1333 }
1334 } else if (c == ':') {
1335 (sym++)[0] = c;
1336 sym[0] = NULL_CHAR;
1337 if (next_char (ref_l, ref_s, A68G_FALSE) == '=') {
1338 (sym++)[0] = '=';
1339 sym[0] = NULL_CHAR;
1340 (void) next_char (ref_l, ref_s, A68G_FALSE);
1341 } else {
1342 SCAN_ERROR (strcmp (&(scanned[1]), "=:") != 0, *start_l, *start_c, ERROR_INVALID_OPERATOR_TAG);
1343 }
1344 }
1345 sym[0] = NULL_CHAR;
1346 *att = OPERATOR;
1347 } else {
1348 // Afuuus ... strange characters!.
1349 unworthy (*start_l, *start_c, (int) c);
1350 }
1351 }
1352
1353 //! @brief Whether att opens an embedded clause.
1354
1355 BOOL_T open_nested_clause (int att)
1356 {
1357 switch (att) {
1358 case OPEN_SYMBOL:
1359 case BEGIN_SYMBOL:
1360 case PAR_SYMBOL:
1361 case IF_SYMBOL:
1362 case CASE_SYMBOL:
1363 case FOR_SYMBOL:
1364 case FROM_SYMBOL:
1365 case BY_SYMBOL:
1366 case TO_SYMBOL:
1367 case DOWNTO_SYMBOL:
1368 case WHILE_SYMBOL:
1369 case DO_SYMBOL:
1370 case SUB_SYMBOL:
1371 case ACCO_SYMBOL: {
1372 return A68G_TRUE;
1373 }
1374 }
1375 return A68G_FALSE;
1376 }
1377
1378 //! @brief Whether att closes an embedded clause.
1379
1380 BOOL_T close_nested_clause (int att)
1381 {
1382 switch (att) {
1383 case CLOSE_SYMBOL:
1384 case END_SYMBOL:
1385 case FI_SYMBOL:
1386 case ESAC_SYMBOL:
1387 case OD_SYMBOL:
1388 case BUS_SYMBOL:
1389 case OCCA_SYMBOL: {
1390 return A68G_TRUE;
1391 }
1392 }
1393 return A68G_FALSE;
1394 }
1395
1396 //! @brief Cast a string to lower case.
1397
1398 void make_lower_case (char *p)
1399 {
1400 for (; p != NO_TEXT && p[0] != NULL_CHAR; p++) {
1401 p[0] = (char) TO_LOWER (p[0]);
1402 }
1403 }
1404
1405 //! @brief Construct a linear list of tokens.
1406
1407 void tokenise_source (NODE_T ** root, int level, BOOL_T in_format, LINE_T ** l, char **s, LINE_T ** start_l, char **start_c)
1408 {
1409 char *lpr = NO_TEXT;
1410 int lprt = 0;
1411 while (l != NO_REF && !A68G_PARSER (stop_scanner)) {
1412 int att = 0;
1413 get_next_token (in_format, l, s, start_l, start_c, &att);
1414 if (A68G_PARSER (scan_buf)[0] == STOP_CHAR) {
1415 A68G_PARSER (stop_scanner) = A68G_TRUE;
1416 } else if (strlen (A68G_PARSER (scan_buf)) > 0 || att == ROW_CHAR_DENOTATION || att == LITERAL) {
1417 KEYWORD_T *kw;
1418 char *c = NO_TEXT;
1419 BOOL_T make_node = A68G_TRUE;
1420 char *trailing = NO_TEXT;
1421 if (att != IDENTIFIER) {
1422 kw = find_keyword (A68G (top_keyword), A68G_PARSER (scan_buf));
1423 } else {
1424 kw = NO_KEYWORD;
1425 }
1426 if (!(kw != NO_KEYWORD && att != ROW_CHAR_DENOTATION)) {
1427 if (att == IDENTIFIER) {
1428 make_lower_case (A68G_PARSER (scan_buf));
1429 }
1430 if (att != ROW_CHAR_DENOTATION && att != LITERAL) {
1431 int len = (int) strlen (A68G_PARSER (scan_buf));
1432 while (len >= 1 && A68G_PARSER (scan_buf)[len - 1] == '_') {
1433 trailing = "_";
1434 A68G_PARSER (scan_buf)[len - 1] = NULL_CHAR;
1435 len--;
1436 }
1437 }
1438 c = TEXT (add_token (&A68G (top_token), A68G_PARSER (scan_buf)));
1439 } else {
1440 if (IS (kw, TO_SYMBOL)) {
1441 // Merge GO and TO to GOTO.
1442 if (*root != NO_NODE && IS (*root, GO_SYMBOL)) {
1443 ATTRIBUTE (*root) = GOTO_SYMBOL;
1444 NSYMBOL (*root) = TEXT (find_keyword (A68G (top_keyword), "GOTO"));
1445 make_node = A68G_FALSE;
1446 } else {
1447 att = ATTRIBUTE (kw);
1448 c = TEXT (kw);
1449 }
1450 } else {
1451 if (att == 0 || att == BOLD_TAG) {
1452 att = ATTRIBUTE (kw);
1453 }
1454 c = TEXT (kw);
1455 // Handle pragments.
1456 if (att == STYLE_II_COMMENT_SYMBOL || att == STYLE_I_COMMENT_SYMBOL || att == BOLD_COMMENT_SYMBOL) {
1457 char *nlpr = pragment (ATTRIBUTE (kw), l, s);
1458 if (lpr == NO_TEXT || (int) strlen (lpr) == 0) {
1459 lpr = nlpr;
1460 } else {
1461 char *stale = lpr;
1462 lpr = new_string (lpr, "\n\n", nlpr, NO_TEXT);
1463 a68g_free (nlpr);
1464 a68g_free (stale);
1465 }
1466 lprt = att;
1467 make_node = A68G_FALSE;
1468 } else if (att == STYLE_I_PRAGMAT_SYMBOL || att == BOLD_PRAGMAT_SYMBOL) {
1469 char *nlpr = pragment (ATTRIBUTE (kw), l, s);
1470 if (lpr == NO_TEXT || (int) strlen (lpr) == 0) {
1471 lpr = nlpr;
1472 } else {
1473 char *stale = lpr;
1474 lpr = new_string (lpr, "\n\n", nlpr, NO_TEXT);
1475 a68g_free (nlpr);
1476 a68g_free (stale);
1477 }
1478 lprt = att;
1479 if (!A68G_PARSER (stop_scanner)) {
1480 (void) set_options (OPTION_LIST (&A68G_JOB), A68G_FALSE);
1481 make_node = A68G_FALSE;
1482 }
1483 }
1484 }
1485 }
1486 // Add token to the tree.
1487 if (make_node) {
1488 NODE_T *q = new_node ();
1489 INFO (q) = new_node_info ();
1490 switch (att) {
1491 case ASSIGN_SYMBOL:
1492 case END_SYMBOL:
1493 case ESAC_SYMBOL:
1494 case OD_SYMBOL:
1495 case OF_SYMBOL:
1496 case FI_SYMBOL:
1497 case CLOSE_SYMBOL:
1498 case BUS_SYMBOL:
1499 case COLON_SYMBOL:
1500 case COMMA_SYMBOL:
1501 case DOTDOT_SYMBOL:
1502 case SEMI_SYMBOL: {
1503 GINFO (q) = NO_GINFO;
1504 break;
1505 }
1506 default: {
1507 GINFO (q) = new_genie_info ();
1508 break;
1509 }
1510 }
1511 STATUS (q) = OPTION_NODEMASK (&A68G_JOB);
1512 LINE (INFO (q)) = *start_l;
1513 CHAR_IN_LINE (INFO (q)) = *start_c;
1514 PRIO (INFO (q)) = 0;
1515 PROCEDURE_LEVEL (INFO (q)) = 0;
1516 ATTRIBUTE (q) = att;
1517 NSYMBOL (q) = c;
1518 PREVIOUS (q) = *root;
1519 SUB (q) = NEXT (q) = NO_NODE;
1520 TABLE (q) = NO_TABLE;
1521 MOID (q) = NO_MOID;
1522 TAX (q) = NO_TAG;
1523 if (lpr != NO_TEXT) {
1524 NPRAGMENT (q) = lpr;
1525 NPRAGMENT_TYPE (q) = lprt;
1526 lpr = NO_TEXT;
1527 lprt = 0;
1528 }
1529 if (*root != NO_NODE) {
1530 NEXT (*root) = q;
1531 }
1532 if (TOP_NODE (&A68G_JOB) == NO_NODE) {
1533 TOP_NODE (&A68G_JOB) = q;
1534 }
1535 *root = q;
1536 if (trailing != NO_TEXT) {
1537 diagnostic (A68G_WARNING | A68G_FORCE_DIAGNOSTICS, q, WARNING_TRAILING, trailing, att);
1538 }
1539 }
1540 // Redirection in tokenising formats. The scanner is a recursive-descent type as
1541 // to know when it scans a format text and when not.
1542 if (in_format && att == FORMAT_DELIMITER_SYMBOL) {
1543 return;
1544 } else if (!in_format && att == FORMAT_DELIMITER_SYMBOL) {
1545 tokenise_source (root, level + 1, A68G_TRUE, l, s, start_l, start_c);
1546 } else if (in_format && open_nested_clause (att)) {
1547 NODE_T *z = PREVIOUS (*root);
1548 if (z != NO_NODE && is_one_of (z, FORMAT_ITEM_N, FORMAT_ITEM_G, FORMAT_ITEM_H, FORMAT_ITEM_F, STOP)) {
1549 tokenise_source (root, level, A68G_FALSE, l, s, start_l, start_c);
1550 } else if (att == OPEN_SYMBOL) {
1551 ATTRIBUTE (*root) = FORMAT_OPEN_SYMBOL;
1552 } else if (OPTION_BRACKETS (&A68G_JOB) && att == SUB_SYMBOL) {
1553 ATTRIBUTE (*root) = FORMAT_OPEN_SYMBOL;
1554 } else if (OPTION_BRACKETS (&A68G_JOB) && att == ACCO_SYMBOL) {
1555 ATTRIBUTE (*root) = FORMAT_OPEN_SYMBOL;
1556 }
1557 } else if (!in_format && level > 0 && open_nested_clause (att)) {
1558 tokenise_source (root, level + 1, A68G_FALSE, l, s, start_l, start_c);
1559 } else if (!in_format && level > 0 && close_nested_clause (att)) {
1560 return;
1561 } else if (in_format && att == CLOSE_SYMBOL) {
1562 ATTRIBUTE (*root) = FORMAT_CLOSE_SYMBOL;
1563 } else if (OPTION_BRACKETS (&A68G_JOB) && in_format && att == BUS_SYMBOL) {
1564 ATTRIBUTE (*root) = FORMAT_CLOSE_SYMBOL;
1565 } else if (OPTION_BRACKETS (&A68G_JOB) && in_format && att == OCCA_SYMBOL) {
1566 ATTRIBUTE (*root) = FORMAT_CLOSE_SYMBOL;
1567 }
1568 }
1569 }
1570 }
1571
1572 //! @brief Tokenise source file, build initial syntax tree.
1573
1574 BOOL_T lexical_analyser (void)
1575 {
1576 LINE_T *l = NO_LINE, *start_l = NO_LINE;
1577 char *s = NO_TEXT, *start_c = NO_TEXT;
1578 NODE_T *root = NO_NODE;
1579 A68G_PARSER (scan_buf) = NO_TEXT;
1580 A68G_PARSER (max_scan_buf_length) = A68G_PARSER (source_file_size) = get_source_size ();
1581 // Errors in file?.
1582 if (A68G_PARSER (max_scan_buf_length) == 0) {
1583 return A68G_FALSE;
1584 }
1585 if (OPTION_RUN_SCRIPT (&A68G_JOB)) {
1586 A68G_PARSER (scan_buf) = (char *) get_temp_heap_space ((unt) (8 + A68G_PARSER (max_scan_buf_length)));
1587 if (!read_script_file ()) {
1588 return A68G_FALSE;
1589 }
1590 } else {
1591 A68G_PARSER (max_scan_buf_length) += KILOBYTE; // for the environ, more than enough
1592 A68G_PARSER (scan_buf) = (char *) get_temp_heap_space ((unt) A68G_PARSER (max_scan_buf_length));
1593 // Errors in file?.
1594 if (!read_source_file ()) {
1595 return A68G_FALSE;
1596 }
1597 }
1598 // Start tokenising.
1599 A68G_PARSER (read_error) = A68G_FALSE;
1600 A68G_PARSER (stop_scanner) = A68G_FALSE;
1601 if ((l = TOP_LINE (&A68G_JOB)) != NO_LINE) {
1602 s = STRING (l);
1603 }
1604 tokenise_source (&root, 0, A68G_FALSE, &l, &s, &start_l, &start_c);
1605 return A68G_TRUE;
1606 }
|
© 2002-2025 J.M. van der Veer (jmvdveer@xs4all.nl)
|
