parser-scanner.c
1 //! @file parser-scanner.c
2 //! @author J. Marcel van der Veer
3
4 //! @section Copyright
5 //!
6 //! This file is part of Algol68G - an Algol 68 compiler-interpreter.
7 //! Copyright 2001-2026 J. Marcel van der Veer [algol68g@xs4all.nl].
8
9 //! @section License
10 //!
11 //! This program is free software; you can redistribute it and/or modify it
12 //! under the terms of the GNU General Public License as published by the
13 //! Free Software Foundation; either version 3 of the License, or
14 //! (at your option) any later version.
15 //!
16 //! This program is distributed in the hope that it will be useful, but
17 //! WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18 //! or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
19 //! more details. You should have received a copy of the GNU General Public
20 //! License along with this program. If not, see [http://www.gnu.org/licenses/].
21
22 //! @section Synopsis
23 //!
24 //! Context-dependent Algol 68 tokeniser.
25
26 #include "a68g.h"
27 #include "a68g-parser.h"
28 #include "a68g-prelude.h"
29 #include "a68g-options.h"
30 #include "a68g-environ.h"
31
32 // Macros.
33
34 #define SCAN_DIGITS(c)\
35 while (IS_DIGIT (c)) {\
36 (sym++)[0] = (c);\
37 (c) = next_char (ref_l, ref_s, A68G_TRUE);\
38 }
39
40 #define SCAN_EXPONENT_PART(c)\
41 (sym++)[0] = EXPONENT_CHAR;\
42 (c) = next_char (ref_l, ref_s, A68G_TRUE);\
43 if ((c) == '+' || (c) == '-') {\
44 (sym++)[0] = (c);\
45 (c) = next_char (ref_l, ref_s, A68G_TRUE);\
46 }\
47 SCAN_ERROR (!IS_DIGIT (c), *start_l, *start_c, ERROR_EXPONENT_DIGIT);\
48 SCAN_DIGITS (c)
49
50 //! @brief Save scanner state, for character look-ahead.
51
52 void save_state (LINE_T * ref_l, char *ref_s, char ch)
53 {
54 SCAN_STATE_L (&A68G_JOB) = ref_l;
55 SCAN_STATE_S (&A68G_JOB) = ref_s;
56 SCAN_STATE_C (&A68G_JOB) = ch;
57 }
58
59 //! @brief Restore scanner state, for character look-ahead.
60
61 void restore_state (LINE_T ** ref_l, char **ref_s, char *ch)
62 {
63 *ref_l = SCAN_STATE_L (&A68G_JOB);
64 *ref_s = SCAN_STATE_S (&A68G_JOB);
65 *ch = SCAN_STATE_C (&A68G_JOB);
66 }
67
68 //! @brief New_source_line.
69
70 LINE_T *new_source_line (void)
71 {
72 LINE_T *z = (LINE_T *) get_fixed_heap_space (SIZE_ALIGNED (LINE_T));
73 MARKER (z)[0] = NULL_CHAR;
74 STRING (z) = NO_TEXT;
75 FILENAME (z) = NO_TEXT;
76 DIAGNOSTICS (z) = NO_DIAGNOSTIC;
77 NUMBER (z) = 0;
78 PRINT_STATUS (z) = 0;
79 LIST (z) = A68G_TRUE;
80 NEXT (z) = NO_LINE;
81 PREVIOUS (z) = NO_LINE;
82 return z;
83 }
84
85 //! @brief Append a source line to the internal source file.
86
87 void append_source_line (char *str, LINE_T ** ref_l, int *line_num, char *filename)
88 {
89 LINE_T *z = new_source_line ();
90 // Allow shell command in first line, f.i. "#!/usr/share/bin/a68g".
91 if (*line_num == 1) {
92 if (strlen (str) >= 2 && strncmp (str, "#!", strlen ("#!")) == 0) {
93 ABEND (strstr (str, "run-script") != NO_TEXT, ERROR_SHELL_SCRIPT, NO_TEXT);
94 (*line_num)++;
95 return;
96 }
97 }
98 // Link line into the chain.
99 STRING (z) = new_fixed_string (str);
100 FILENAME (z) = filename;
101 NUMBER (z) = (*line_num)++;
102 PRINT_STATUS (z) = NOT_PRINTED;
103 LIST (z) = A68G_TRUE;
104 DIAGNOSTICS (z) = NO_DIAGNOSTIC;
105 NEXT (z) = NO_LINE;
106 PREVIOUS (z) = *ref_l;
107 if (TOP_LINE (&A68G_JOB) == NO_LINE) {
108 TOP_LINE (&A68G_JOB) = z;
109 }
110 if (*ref_l != NO_LINE) {
111 NEXT (*ref_l) = z;
112 }
113 *ref_l = z;
114 }
115
116 // Scanner, tokenises the source code.
117
118 //! @brief Whether ch is unworthy.
119
120 void unworthy (LINE_T * u, char *v, char ch)
121 {
122 if (IS_PRINT (ch)) {
123 ASSERT (a68g_bufprt (A68G (edit_line), SNPRINTF_SIZE, "*%s", ERROR_UNWORTHY_CHARACTER) >= 0);
124 } else {
125 ASSERT (a68g_bufprt (A68G (edit_line), SNPRINTF_SIZE, "*%s %s", ERROR_UNWORTHY_CHARACTER, ctrl_char (ch)) >= 0);
126 }
127 scan_error (u, v, A68G (edit_line));
128 }
129
130 //! @brief Concatenate lines that terminate in '\' with next line.
131
132 void concatenate_lines (LINE_T * top)
133 {
134 LINE_T *q;
135 // Work from bottom backwards.
136 for (q = top; q != NO_LINE && NEXT (q) != NO_LINE; q = NEXT (q)) {
137 ;
138 }
139 for (; q != NO_LINE; BACKWARD (q)) {
140 char *z = STRING (q);
141 size_t len = strlen (z);
142 if (len >= 2 && z[len - 2] == BACKSLASH_CHAR && z[len - 1] == NEWLINE_CHAR && NEXT (q) != NO_LINE && STRING (NEXT (q)) != NO_TEXT) {
143 z[len - 2] = NULL_CHAR;
144 len += strlen (STRING (NEXT (q)));
145 z = (char *) get_fixed_heap_space (len + 1);
146 a68g_bufcpy (z, STRING (q), len + 1);
147 a68g_bufcat (z, STRING (NEXT (q)), len + 1);
148 STRING (NEXT (q))[0] = NULL_CHAR;
149 STRING (q) = z;
150 }
151 }
152 }
153
154 //! @brief Whether u is bold tag v, independent of stropping regime.
155
156 BOOL_T is_bold (char *u, char *v)
157 {
158 size_t len = strlen (v);
159 if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
160 if (u[0] == '\'') {
161 return (BOOL_T) (strncmp (++u, v, len) == 0 && u[len] == '\'');
162 } else {
163 return A68G_FALSE;
164 }
165 } else {
166 return (BOOL_T) (strncmp (u, v, len) == 0 && !IS_UPPER (u[len]));
167 }
168 }
169
170 //! @brief Skip string.
171
172 BOOL_T skip_string (LINE_T ** top, char **ch)
173 {
174 LINE_T *u = *top;
175 char *v = *ch;
176 v++;
177 while (u != NO_LINE) {
178 while (v[0] != NULL_CHAR) {
179 if (v[0] == QUOTE_CHAR && v[1] != QUOTE_CHAR) {
180 *top = u;
181 *ch = &v[1];
182 return A68G_TRUE;
183 } else if (v[0] == QUOTE_CHAR && v[1] == QUOTE_CHAR) {
184 v += 2;
185 } else {
186 v++;
187 }
188 }
189 FORWARD (u);
190 if (u != NO_LINE) {
191 v = &(STRING (u)[0]);
192 } else {
193 v = NO_TEXT;
194 }
195 }
196 return A68G_FALSE;
197 }
198
199 //! @brief Skip comment.
200
201 BOOL_T skip_comment (LINE_T ** top, char **ch, int delim)
202 {
203 LINE_T *u = *top;
204 char *v = *ch;
205 BOOL_T qstrop = OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING;
206 v++;
207 while (u != NO_LINE) {
208 while (v[0] != NULL_CHAR) {
209 if (is_bold (v, "COMMENT") && delim == BOLD_COMMENT_SYMBOL) {
210 char *w = &v[strlen(qstrop ? "'COMMENT'" : "COMMENT")];
211 if (!IS_UPPER(w[0])) {
212 *top = u;
213 *ch = w;
214 return A68G_TRUE;
215 }
216 } else if (is_bold (v, "CO") && delim == STYLE_I_COMMENT_SYMBOL) {
217 char *w = &v[strlen(qstrop ? "'CO'" : "CO")];
218 if (!IS_UPPER(w[0])) {
219 *top = u;
220 *ch = w;
221 return A68G_TRUE;
222 }
223 } else if (v[0] == '#' && delim == STYLE_II_COMMENT_SYMBOL) {
224 *top = u;
225 *ch = &v[1];
226 return A68G_TRUE;
227 } else {
228 v++;
229 }
230 }
231 FORWARD (u);
232 if (u != NO_LINE) {
233 v = &(STRING (u)[0]);
234 } else {
235 v = NO_TEXT;
236 }
237 }
238 return A68G_FALSE;
239 }
240
241 //! @brief Skip rest of pragmat.
242
243 BOOL_T skip_pragmat (LINE_T ** top, char **ch, int delim, BOOL_T whitespace)
244 {
245 LINE_T *u = *top;
246 char *v = *ch;
247 BOOL_T qstrop = OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING;
248 while (u != NO_LINE) {
249 while (v[0] != NULL_CHAR) {
250 if (is_bold (v, "PRAGMAT") && delim == BOLD_PRAGMAT_SYMBOL) {
251 char *w = &v[strlen(qstrop ? "'PRAGMAT'" : "PRAGMAT")];
252 if (!IS_UPPER(w[0])) {
253 *top = u;
254 *ch = w;
255 return A68G_TRUE;
256 }
257 } else if (is_bold (v, "PR") && delim == STYLE_I_PRAGMAT_SYMBOL) {
258 char *w = &v[strlen(qstrop ? "'PR'" : "PR")];
259 if (!IS_UPPER(w[0])) {
260 *top = u;
261 *ch = w;
262 return A68G_TRUE;
263 }
264 } else {
265 if (whitespace && !IS_SPACE (v[0]) && v[0] != NEWLINE_CHAR) {
266 scan_error (u, v, ERROR_PRAGMENT);
267 } else if (IS_UPPER (v[0])) {
268 // Skip a bold word as you may trigger on REPR, for instance ...
269 while (IS_UPPER (v[0])) {
270 v++;
271 }
272 } else {
273 v++;
274 }
275 }
276 }
277 FORWARD (u);
278 if (u != NO_LINE) {
279 v = &(STRING (u)[0]);
280 } else {
281 v = NO_TEXT;
282 }
283 }
284 return A68G_FALSE;
285 }
286
287 //! @brief Return pointer to next token within pragmat.
288
289 char *get_pragmat_item (LINE_T ** top, char **ch)
290 {
291 LINE_T *u = *top;
292 char *v = *ch;
293 while (u != NO_LINE) {
294 while (v[0] != NULL_CHAR) {
295 if (!IS_SPACE (v[0]) && v[0] != NEWLINE_CHAR) {
296 *top = u;
297 *ch = v;
298 return v;
299 } else {
300 v++;
301 }
302 }
303 FORWARD (u);
304 if (u != NO_LINE) {
305 v = &(STRING (u)[0]);
306 } else {
307 v = NO_TEXT;
308 }
309 }
310 return NO_TEXT;
311 }
312
313 //! @brief Case insensitive strncmp for at most the number of chars in 'v'.
314
315 int streq (char *u, char *v)
316 {
317 int diff;
318 for (diff = 0; diff == 0 && u[0] != NULL_CHAR && v[0] != NULL_CHAR; u++, v++) {
319 diff = ((int) TO_LOWER (u[0])) - ((int) TO_LOWER (v[0]));
320 }
321 return diff;
322 }
323
324 //! @brief Scan for next pragmat and yield first pragmat item.
325
326 char *next_preprocessor_item (LINE_T ** top, char **ch, int *delim)
327 {
328 LINE_T *u = *top;
329 char *v = *ch;
330 *delim = 0;
331 while (u != NO_LINE) {
332 while (v[0] != NULL_CHAR) {
333 LINE_T *start_l = u;
334 char *start_c = v;
335 // STRINGs must be skipped.
336 if (v[0] == QUOTE_CHAR) {
337 SCAN_ERROR (!skip_string (&u, &v), start_l, start_c, ERROR_UNTERMINATED_STRING);
338 }
339 // COMMENTS must be skipped.
340 else if (is_bold (v, "COMMENT")) {
341 SCAN_ERROR (!skip_comment (&u, &v, BOLD_COMMENT_SYMBOL), start_l, start_c, ERROR_UNTERMINATED_COMMENT);
342 } else if (is_bold (v, "CO")) {
343 SCAN_ERROR (!skip_comment (&u, &v, STYLE_I_COMMENT_SYMBOL), start_l, start_c, ERROR_UNTERMINATED_COMMENT);
344 } else if (v[0] == '#') {
345 SCAN_ERROR (!skip_comment (&u, &v, STYLE_II_COMMENT_SYMBOL), start_l, start_c, ERROR_UNTERMINATED_COMMENT);
346 } else if (is_bold (v, "PRAGMAT") || is_bold (v, "PR")) {
347 // We caught a PRAGMAT.
348 char *item;
349 if (is_bold (v, "PRAGMAT")) {
350 *delim = BOLD_PRAGMAT_SYMBOL;
351 v = &v[strlen ("PRAGMAT")];
352 } else if (is_bold (v, "PR")) {
353 *delim = STYLE_I_PRAGMAT_SYMBOL;
354 v = &v[strlen ("PR")];
355 }
356 item = get_pragmat_item (&u, &v);
357 SCAN_ERROR (item == NO_TEXT, start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
358 // Item "preprocessor" restarts preprocessing if it is off.
359 if (A68G_PARSER (no_preprocessing) && streq (item, "PREPROCESSOR") == 0) {
360 A68G_PARSER (no_preprocessing) = A68G_FALSE;
361 SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68G_TRUE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
362 }
363 // If preprocessing is switched off, we idle to closing bracket.
364 else if (A68G_PARSER (no_preprocessing)) {
365 SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68G_FALSE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
366 }
367 // Item "nopreprocessor" stops preprocessing if it is on.
368 if (streq (item, "NOPREPROCESSOR") == 0) {
369 A68G_PARSER (no_preprocessing) = A68G_TRUE;
370 SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68G_TRUE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
371 }
372 // Item "INCLUDE" includes a file.
373 else if (streq (item, "INCLUDE") == 0) {
374 *top = u;
375 *ch = v;
376 return item;
377 }
378 // Item "READ" includes a file.
379 else if (streq (item, "READ") == 0) {
380 *top = u;
381 *ch = v;
382 return item;
383 }
384 // Unrecognised item - probably options handled later by the tokeniser.
385 else {
386 SCAN_ERROR (!skip_pragmat (&u, &v, *delim, A68G_FALSE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
387 }
388 } else if (IS_UPPER (v[0])) {
389 // Skip a bold word as you may trigger on REPR, for instance ...
390 while (IS_UPPER (v[0])) {
391 v++;
392 }
393 } else {
394 v++;
395 }
396 }
397 FORWARD (u);
398 if (u != NO_LINE) {
399 v = &(STRING (u)[0]);
400 } else {
401 v = NO_TEXT;
402 }
403 }
404 *top = u;
405 *ch = v;
406 return NO_TEXT;
407 }
408
409 //! @brief Include files.
410
411 void include_files (LINE_T * top)
412 {
413 // include_files
414 //
415 // syntax: PR read "filename" PR
416 // PR include "filename" PR
417 //
418 // The file gets inserted before the line containing the pragmat. In this way
419 // correct line numbers are preserved which helps diagnostics. A file that has
420 // been included will not be included a second time - it will be ignored. This
421 // is a rigorous fail-safe, but there is no mechanism to prevent recursive includes
422 // in A68 source code. User reports do not indicate sophisticated use of INCLUDE,
423 // so this is fine for now.
424 // TODO - some day we might need `app', analogous to `cpp'.
425 BOOL_T make_pass = A68G_TRUE;
426 while (make_pass) {
427 LINE_T *s, *t, *u = top;
428 char *v = &(STRING (u)[0]);
429 make_pass = A68G_FALSE;
430 errno = 0;
431 while (u != NO_LINE) {
432 int pr_lim;
433 char *item = next_preprocessor_item (&u, &v, &pr_lim);
434 LINE_T *start_l = u;
435 char *start_c = v;
436 // Search for PR include "filename" PR.
437 if (item != NO_TEXT && (streq (item, "INCLUDE") == 0 || streq (item, "READ") == 0)) {
438 FILE_T fd;
439 int n, line_num, k, bytes_read;
440 char *fbuf, delim;
441 BUFFER fnb;
442 char *fn;
443 // Skip to filename.
444 while (IS_ALPHA (v[0])) {
445 v++;
446 }
447 while (IS_SPACE (v[0])) {
448 v++;
449 }
450 // Scan quoted filename.
451 SCAN_ERROR ((v[0] != QUOTE_CHAR && v[0] != '\''), start_l, start_c, ERROR_INCORRECT_FILENAME);
452 delim = (v++)[0];
453 n = 0;
454 fnb[0] = NULL_CHAR;
455 // Scan Algol 68 string (note: "" denotes a ", while in C it concatenates).
456 do {
457 SCAN_ERROR (EOL (v[0]), start_l, start_c, ERROR_INCORRECT_FILENAME);
458 SCAN_ERROR (n == BUFFER_SIZE - 1, start_l, start_c, ERROR_INCORRECT_FILENAME);
459 if (v[0] == delim) {
460 while (v[0] == delim && v[1] == delim) {
461 SCAN_ERROR (n == BUFFER_SIZE - 1, start_l, start_c, ERROR_INCORRECT_FILENAME);
462 fnb[n++] = delim;
463 fnb[n] = NULL_CHAR;
464 v += 2;
465 }
466 } else if (IS_PRINT (v[0])) {
467 fnb[n++] = *(v++);
468 fnb[n] = NULL_CHAR;
469 } else {
470 SCAN_ERROR (A68G_TRUE, start_l, start_c, ERROR_INCORRECT_FILENAME);
471 }
472 } while (v[0] != delim);
473 // Insist that the pragmat is closed properly.
474 v = &v[1];
475 SCAN_ERROR (!skip_pragmat (&u, &v, pr_lim, A68G_TRUE), start_l, start_c, ERROR_UNTERMINATED_PRAGMAT);
476 SCAN_ERROR (n == 0, start_l, start_c, ERROR_INCORRECT_FILENAME);
477 // Make the name relative to the position of the source file (C preprocessor standard).
478 if (FILENAME (u) != NO_TEXT) {
479 fn = a68g_relpath (a68g_dirname (FILENAME (u)), a68g_dirname (fnb), a68g_basename (fnb));
480 } else {
481 fn = a68g_relpath (FILE_PATH (&A68G_JOB), a68g_dirname (fnb), a68g_basename (fnb));
482 }
483 // Do not check errno, since errno may be undefined here after a successful call.
484 if (fn != NO_TEXT) {
485 a68g_bufcpy (fnb, fn, BUFFER_SIZE);
486 } else {
487 SCAN_ERROR_INFO (A68G_TRUE, NO_LINE, NO_TEXT, ERROR_SOURCE_FILE_INCLUDE_OPEN, fnb);
488 }
489 size_t fnwid = strlen (fnb) + 1;
490 fn = (char *) get_fixed_heap_space (fnwid);
491 a68g_bufcpy (fn, fnb, fnwid);
492 // Ignore the file when included more than once.
493 for (t = top; t != NO_LINE; t = NEXT (t)) {
494 if (strcmp (FILENAME (t), fn) == 0) {
495 goto search_next_pragmat;
496 }
497 }
498 // Access the file.
499 errno = 0;
500 fd = open (fn, O_RDONLY | O_BINARY);
501 SCAN_ERROR_INFO (fd == -1, start_l, start_c, ERROR_SOURCE_FILE_INCLUDE_OPEN, fnb);
502 errno = 0;
503 a68g_off_t fsize = (a68g_off_t) lseek (fd, 0, SEEK_END);
504 ASSERT (fsize >= 0);
505 SCAN_ERROR (errno != 0, start_l, start_c, ERROR_FILE_READ);
506 fbuf = (char *) get_temp_heap_space ((size_t) (8 + fsize));
507 errno = 0;
508 ASSERT (lseek (fd, 0, SEEK_SET) >= 0);
509 SCAN_ERROR (errno != 0, start_l, start_c, ERROR_FILE_READ);
510 errno = 0;
511 bytes_read = (int) io_read (fd, fbuf, (size_t) fsize);
512 SCAN_ERROR (errno != 0 || bytes_read != fsize, start_l, start_c, ERROR_FILE_READ);
513 // Buffer still usable?.
514 if (fsize > A68G_PARSER (max_scan_buf_length)) {
515 A68G_PARSER (max_scan_buf_length) = fsize;
516 A68G_PARSER (scan_buf) = (char *) get_temp_heap_space (8 + A68G_PARSER (max_scan_buf_length));
517 }
518 // Link all lines into the list.
519 line_num = 1;
520 s = u;
521 t = PREVIOUS (u);
522 k = 0;
523 if (fsize == 0) {
524 // If file is empty, insert single empty line.
525 A68G_PARSER (scan_buf)[0] = NEWLINE_CHAR;
526 A68G_PARSER (scan_buf)[1] = NULL_CHAR;
527 append_source_line (A68G_PARSER (scan_buf), &t, &line_num, fn);
528 } else
529 while (k < fsize) {
530 n = 0;
531 A68G_PARSER (scan_buf)[0] = NULL_CHAR;
532 while (k < fsize && fbuf[k] != NEWLINE_CHAR) {
533 SCAN_ERROR ((IS_CNTRL (fbuf[k]) && !IS_SPACE (fbuf[k])) || fbuf[k] == STOP_CHAR, start_l, start_c, ERROR_FILE_INCLUDE_CTRL);
534 A68G_PARSER (scan_buf)[n++] = fbuf[k++];
535 A68G_PARSER (scan_buf)[n] = NULL_CHAR;
536 }
537 A68G_PARSER (scan_buf)[n++] = NEWLINE_CHAR;
538 A68G_PARSER (scan_buf)[n] = NULL_CHAR;
539 if (k < fsize) {
540 k++;
541 }
542 append_source_line (A68G_PARSER (scan_buf), &t, &line_num, fn);
543 }
544 // Conclude and go find another include directive, if any.
545 NEXT (t) = s;
546 PREVIOUS (s) = t;
547 concatenate_lines (top);
548 ASSERT (close (fd) == 0);
549 make_pass = A68G_TRUE;
550 }
551 search_next_pragmat:_SKIP_;
552 }
553 }
554 }
555
556 //! @brief Size of source file.
557
558 a68g_off_t get_source_size (void)
559 {
560 FILE_T f = FILE_SOURCE_FD (&A68G_JOB);
561 // This is why WIN32/64 must open as "read binary".
562 return (a68g_off_t) lseek (f, 0, SEEK_END);
563 }
564
565 //! @brief Append environment source lines.
566
567 void append_environ (char *str[], LINE_T ** ref_l, int *line_num, char *name)
568 {
569 for (int k = 0; str[k] != NO_TEXT; k++) {
570 int zero_line_num = 0;
571 (void) line_num;
572 append_source_line (str[k], ref_l, &zero_line_num, name);
573 }
574 }
575
576 //! @brief Read script file and make internal copy.
577
578 BOOL_T read_script_file (void)
579 {
580 LINE_T *ref_l = NO_LINE;
581 int num;
582 BOOL_T file_end = A68G_FALSE;
583 BUFFER filename, linenum;
584 char *buffer = (char *) get_temp_heap_space (8 + A68G_PARSER (source_file_size));
585 FILE_T source = FILE_SOURCE_FD (&A68G_JOB);
586 ABEND (source == -1, ERROR_ACTION, NO_TEXT);
587 buffer[0] = NULL_CHAR;
588 UNSIGNED_T k, n = 0;
589 size_t len = 8 + A68G_PARSER (source_file_size);
590 buffer = (char *) get_temp_heap_space (len);
591 ASSERT (lseek (source, 0, SEEK_SET) >= 0);
592 while (!file_end) {
593 char ch;
594 // Read the original file name.
595 filename[0] = NULL_CHAR;
596 k = 0;
597 if (io_read (source, &ch, 1) == 0) {
598 file_end = A68G_TRUE;
599 continue;
600 }
601 while (ch != NEWLINE_CHAR) {
602 filename[k++] = ch;
603 ASSERT (io_read (source, &ch, 1) == 1);
604 }
605 filename[k] = NULL_CHAR;
606 char *fn = TEXT (add_token (&A68G (top_token), filename));
607 // Read the original file number.
608 linenum[0] = NULL_CHAR;
609 k = 0;
610 ASSERT (io_read (source, &ch, 1) == 1);
611 while (ch != NEWLINE_CHAR) {
612 linenum[k++] = ch;
613 ASSERT (io_read (source, &ch, 1) == 1);
614 }
615 linenum[k] = NULL_CHAR;
616 num = (int) strtol (linenum, NO_REF, 10);
617 ABEND (errno == ERANGE, ERROR_INTERNAL_CONSISTENCY, NO_TEXT);
618 // COPY original line into buffer.
619 ASSERT (io_read (source, &ch, 1) == 1);
620 char *line = &buffer[n];
621 while (ch != NEWLINE_CHAR) {
622 buffer[n++] = ch;
623 ASSERT (io_read (source, &ch, 1) == 1);
624 ABEND (n >= len, ERROR_ACTION, NO_TEXT);
625 }
626 buffer[n++] = NEWLINE_CHAR;
627 buffer[n] = NULL_CHAR;
628 append_source_line (line, &ref_l, &num, fn);
629 }
630 return A68G_TRUE;
631 }
632
633 //! @brief match first non-white characters in string.
634
635 BOOL_T a68g_start(char *u, char *v, char **end)
636 {
637 *end = NO_TEXT;
638 while (v[0] != NULL_CHAR) {
639 if (u[0] == NULL_CHAR) {
640 return A68G_FALSE;
641 } else if (IS_SPACE (u[0])) {
642 u++;
643 } else {
644 if (u[0] == v[0]) {
645 u++;
646 v++;
647 if (end != NULL) {
648 *end = u;
649 }
650 } else {
651 return A68G_FALSE;
652 }
653 }
654 }
655 return A68G_TRUE;
656 }
657
658 //! @brief Read source file and make internal copy.
659
660 BOOL_T read_source_file (void)
661 {
662 LINE_T *ref_l = NO_LINE;
663 int line_num = 0, k, bytes_read;
664 FILE_T f = FILE_SOURCE_FD (&A68G_JOB);
665 char **prelude_start, **postlude, *buffer, *text;
666 // Read the file into a single buffer, so we save on system calls.
667 line_num = 1;
668 errno = 0;
669 text = (char *) get_temp_heap_space (8 + A68G_PARSER (source_file_size));
670 ABEND (errno != 0 || text == NO_TEXT, ERROR_MEMORY_FULL, NO_TEXT);
671 ASSERT (lseek (f, 0, SEEK_SET) >= 0);
672 ABEND (errno != 0, ERROR_ACTION, NO_TEXT);
673 errno = 0;
674 bytes_read = (int) io_read (f, text, A68G_PARSER (source_file_size));
675 ABEND (errno != 0 || bytes_read != A68G_PARSER (source_file_size), ERROR_ACTION, NO_TEXT);
676 // Little test on stropping.
677 char *pr1 = "'PR'QUOTESTROPPING'PR'";
678 char *pr2 = "'PRAGMAT'QUOTESTROPPING'PRAGMAT'";
679 char *end = NO_TEXT;
680 if (a68g_start (text, pr1, &end)) {
681 OPTION_STROPPING (&A68G_JOB) = QUOTE_STROPPING;
682 buffer = end;
683 A68G_PARSER (source_file_size) = strlen (buffer);
684 } else if (a68g_start (text, pr2, &end)) {
685 OPTION_STROPPING (&A68G_JOB) = QUOTE_STROPPING;
686 buffer = end;
687 A68G_PARSER (source_file_size) = strlen (buffer);
688 } else {
689 buffer = text;
690 }
691 // Prelude.
692 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
693 prelude_start = bold_prelude_start;
694 postlude = bold_postlude;
695 } else if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
696 prelude_start = quote_prelude_start;
697 postlude = quote_postlude;
698 } else {
699 prelude_start = postlude = NO_REF;
700 }
701 append_environ (prelude_start, &ref_l, &line_num, "prelude");
702 // Link all lines into the list.
703 k = 0;
704 while (k < A68G_PARSER (source_file_size)) {
705 ssize_t l = 0;
706 A68G_PARSER (scan_buf)[0] = NULL_CHAR;
707 while (k < A68G_PARSER (source_file_size) && buffer[k] != NEWLINE_CHAR) {
708 if (k < A68G_PARSER (source_file_size) - 1 && buffer[k] == CR_CHAR && buffer[k + 1] == NEWLINE_CHAR) {
709 k++;
710 } else {
711 A68G_PARSER (scan_buf)[l++] = buffer[k++];
712 A68G_PARSER (scan_buf)[l] = NULL_CHAR;
713 }
714 }
715 A68G_PARSER (scan_buf)[l++] = NEWLINE_CHAR;
716 A68G_PARSER (scan_buf)[l] = NULL_CHAR;
717 if (k < A68G_PARSER (source_file_size)) {
718 k++;
719 }
720 append_source_line (A68G_PARSER (scan_buf), &ref_l, &line_num, FILE_SOURCE_NAME (&A68G_JOB));
721 SCAN_ERROR (l != strlen (A68G_PARSER (scan_buf)), NO_LINE, NO_TEXT, ERROR_FILE_SOURCE_CTRL);
722 }
723 // Postlude.
724 append_environ (postlude, &ref_l, &line_num, "postlude");
725 // Concatenate lines.
726 concatenate_lines (TOP_LINE (&A68G_JOB));
727 // Include files.
728 include_files (TOP_LINE (&A68G_JOB));
729 return A68G_TRUE;
730 }
731
732 //! @brief Next_char get next character from internal copy of source file.
733
734 char next_char (LINE_T ** ref_l, char **ref_s, BOOL_T allow_typo)
735 {
736 char ch;
737 #if defined (NO_TYPO)
738 allow_typo = A68G_FALSE;
739 #endif
740 LOW_STACK_ALERT (NO_NODE);
741 // Source empty?.
742 if (*ref_l == NO_LINE) {
743 return STOP_CHAR;
744 } else {
745 LIST (*ref_l) = (BOOL_T) (OPTION_NODEMASK (&A68G_JOB) & SOURCE_MASK ? A68G_TRUE : A68G_FALSE);
746 // Take new line?.
747 if ((*ref_s)[0] == NEWLINE_CHAR || (*ref_s)[0] == NULL_CHAR) {
748 *ref_l = NEXT (*ref_l);
749 if (*ref_l == NO_LINE) {
750 return STOP_CHAR;
751 }
752 *ref_s = STRING (*ref_l);
753 } else {
754 (*ref_s)++;
755 }
756 // Deliver next char.
757 ch = (*ref_s)[0];
758 if (allow_typo && (IS_SPACE (ch) || ch == FORMFEED_CHAR)) {
759 ch = next_char (ref_l, ref_s, allow_typo);
760 }
761 return ch;
762 }
763 }
764
765 //! @brief Find first character that can start a valid symbol.
766
767 void get_good_char (char *ref_c, LINE_T ** ref_l, char **ref_s)
768 {
769 while (*ref_c != STOP_CHAR && (IS_SPACE (*ref_c) || (*ref_c == NULL_CHAR))) {
770 if (*ref_l != NO_LINE) {
771 LIST (*ref_l) = (BOOL_T) (OPTION_NODEMASK (&A68G_JOB) & SOURCE_MASK ? A68G_TRUE : A68G_FALSE);
772 }
773 *ref_c = next_char (ref_l, ref_s, A68G_FALSE);
774 }
775 }
776
777 //! @brief Handle a pragment (pragmat or comment).
778
779 char *pragment (int type, LINE_T ** ref_l, char **ref_c)
780 {
781 size_t chars_in_buf;
782 #define INIT_BUFFER {chars_in_buf = 0; A68G_PARSER (scan_buf)[chars_in_buf] = NULL_CHAR;}
783 #define ADD_ONE_CHAR(ch) {A68G_PARSER (scan_buf)[chars_in_buf ++] = ch; A68G_PARSER (scan_buf)[chars_in_buf] = NULL_CHAR;}
784 //
785 char c = **ref_c, *term_s = NO_TEXT, *start_c = *ref_c;
786 char *z = NO_TEXT;
787 LINE_T *start_l = *ref_l;
788 BOOL_T stop, pragmat = A68G_FALSE;
789 // Set terminator.
790 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
791 if (type == STYLE_I_COMMENT_SYMBOL) {
792 term_s = "CO";
793 } else if (type == STYLE_II_COMMENT_SYMBOL) {
794 term_s = "#";
795 } else if (type == BOLD_COMMENT_SYMBOL) {
796 term_s = "COMMENT";
797 } else if (type == STYLE_I_PRAGMAT_SYMBOL) {
798 term_s = "PR";
799 pragmat = A68G_TRUE;
800 } else if (type == BOLD_PRAGMAT_SYMBOL) {
801 term_s = "PRAGMAT";
802 pragmat = A68G_TRUE;
803 }
804 } else if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
805 if (type == STYLE_I_COMMENT_SYMBOL) {
806 term_s = "'CO'";
807 } else if (type == STYLE_II_COMMENT_SYMBOL) {
808 term_s = "#";
809 } else if (type == BOLD_COMMENT_SYMBOL) {
810 term_s = "'COMMENT'";
811 } else if (type == STYLE_I_PRAGMAT_SYMBOL) {
812 term_s = "'PR'";
813 pragmat = A68G_TRUE;
814 } else if (type == BOLD_PRAGMAT_SYMBOL) {
815 term_s = "'PRAGMAT'";
816 pragmat = A68G_TRUE;
817 }
818 }
819 size_t term_s_length = strlen (term_s);
820 // Scan for terminator.
821 INIT_BUFFER;
822 stop = A68G_FALSE;
823 while (stop == A68G_FALSE) {
824 BOOL_T scan_next = A68G_TRUE;
825 SCAN_ERROR (c == STOP_CHAR, start_l, start_c, ERROR_UNTERMINATED_PRAGMENT);
826 // A ".." or '..' delimited string in a PRAGMAT.
827 if (pragmat && (c == QUOTE_CHAR || (c == '\'' && OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING))) {
828 char delim = c;
829 BOOL_T eos = A68G_FALSE;
830 ADD_ONE_CHAR (c);
831 c = next_char (ref_l, ref_c, A68G_FALSE);
832 while (!eos) {
833 SCAN_ERROR (EOL (c), start_l, start_c, ERROR_LONG_STRING);
834 if (c == delim) {
835 ADD_ONE_CHAR (delim);
836 save_state (*ref_l, *ref_c, c);
837 c = next_char (ref_l, ref_c, A68G_FALSE);
838 if (c == delim) {
839 c = next_char (ref_l, ref_c, A68G_FALSE);
840 } else {
841 restore_state (ref_l, ref_c, &c);
842 eos = A68G_TRUE;
843 }
844 } else if (IS_PRINT (c)) {
845 ADD_ONE_CHAR (c);
846 c = next_char (ref_l, ref_c, A68G_FALSE);
847 } else {
848 unworthy (start_l, start_c, c);
849 }
850 }
851 } else if (EOL (c)) {
852 ADD_ONE_CHAR (NEWLINE_CHAR);
853 } else if (IS_UPPER (c)) {
854 while (IS_UPPER (c)) {
855 ADD_ONE_CHAR (c);
856 c = next_char (ref_l, ref_c, A68G_FALSE);
857 }
858 scan_next = A68G_FALSE;
859 } else if (IS_PRINT (c) || IS_SPACE (c)) {
860 ADD_ONE_CHAR (c);
861 }
862 if (chars_in_buf >= term_s_length) {
863 // Check whether we encountered the terminator.
864 char *tok = &(A68G_PARSER (scan_buf)[chars_in_buf - term_s_length]);
865 stop = (BOOL_T) (strcmp (term_s, tok) == 0);
866 }
867 if (scan_next) {
868 c = next_char (ref_l, ref_c, A68G_FALSE);
869 }
870 }
871 A68G_PARSER (scan_buf)[chars_in_buf - term_s_length] = NULL_CHAR;
872 z = new_string (term_s, A68G_PARSER (scan_buf), term_s, NO_TEXT);
873 if (type == STYLE_I_PRAGMAT_SYMBOL || type == BOLD_PRAGMAT_SYMBOL) {
874 isolate_options (A68G_PARSER (scan_buf), start_l);
875 }
876 return z;
877 #undef ADD_ONE_CHAR
878 #undef INIT_BUFFER
879 }
880
881 //! @brief Attribute for format item.
882
883 int get_format_item (char ch)
884 {
885 switch (TO_LOWER (ch)) {
886 case 'a': {
887 return FORMAT_ITEM_A;
888 }
889 case 'b': {
890 return FORMAT_ITEM_B;
891 }
892 case 'c': {
893 return FORMAT_ITEM_C;
894 }
895 case 'd': {
896 return FORMAT_ITEM_D;
897 }
898 case 'e': {
899 return FORMAT_ITEM_E;
900 }
901 case 'f': {
902 return FORMAT_ITEM_F;
903 }
904 case 'g': {
905 return FORMAT_ITEM_G;
906 }
907 case 'h': {
908 return FORMAT_ITEM_H;
909 }
910 case 'i': {
911 return FORMAT_ITEM_I;
912 }
913 case 'j': {
914 return FORMAT_ITEM_J;
915 }
916 case 'k': {
917 return FORMAT_ITEM_K;
918 }
919 case 'l':
920 case '/': {
921 return FORMAT_ITEM_L;
922 }
923 case 'm': {
924 return FORMAT_ITEM_M;
925 }
926 case 'n': {
927 return FORMAT_ITEM_N;
928 }
929 case 'o': {
930 return FORMAT_ITEM_O;
931 }
932 case 'p': {
933 return FORMAT_ITEM_P;
934 }
935 case 'q': {
936 return FORMAT_ITEM_Q;
937 }
938 case 'r': {
939 return FORMAT_ITEM_R;
940 }
941 case 's': {
942 return FORMAT_ITEM_S;
943 }
944 case 't': {
945 return FORMAT_ITEM_T;
946 }
947 case 'u': {
948 return FORMAT_ITEM_U;
949 }
950 case 'v': {
951 return FORMAT_ITEM_V;
952 }
953 case 'w': {
954 return FORMAT_ITEM_W;
955 }
956 case 'x': {
957 return FORMAT_ITEM_X;
958 }
959 case 'y': {
960 return FORMAT_ITEM_Y;
961 }
962 case 'z': {
963 return FORMAT_ITEM_Z;
964 }
965 case '+': {
966 return FORMAT_ITEM_PLUS;
967 }
968 case '-': {
969 return FORMAT_ITEM_MINUS;
970 }
971 case POINT_CHAR: {
972 return FORMAT_ITEM_POINT;
973 }
974 case '%': {
975 return FORMAT_ITEM_ESCAPE;
976 }
977 default: {
978 return 0;
979 }
980 }
981 }
982
983 //! @brief Whether input shows exponent character.
984
985 BOOL_T is_exp_char (LINE_T ** ref_l, char **ref_s, char *ch)
986 {
987 BOOL_T ret = A68G_FALSE;
988 char exp_syms[3];
989 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
990 exp_syms[0] = EXPONENT_CHAR;
991 exp_syms[1] = (char) TO_UPPER (EXPONENT_CHAR);
992 exp_syms[2] = NULL_CHAR;
993 } else {
994 exp_syms[0] = (char) TO_UPPER (EXPONENT_CHAR);
995 exp_syms[1] = BACKSLASH_CHAR;
996 exp_syms[2] = NULL_CHAR;
997 }
998 save_state (*ref_l, *ref_s, *ch);
999 if (strchr (exp_syms, *ch) != NO_TEXT) {
1000 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1001 ret = (BOOL_T) (strchr ("+-0123456789", *ch) != NO_TEXT);
1002 }
1003 restore_state (ref_l, ref_s, ch);
1004 return ret;
1005 }
1006
1007 //! @brief Whether input shows radix character.
1008
1009 BOOL_T is_radix_char (LINE_T ** ref_l, char **ref_s, char *ch)
1010 {
1011 BOOL_T ret = A68G_FALSE;
1012 save_state (*ref_l, *ref_s, *ch);
1013 if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
1014 if (*ch == TO_UPPER (RADIX_CHAR)) {
1015 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1016 ret = (BOOL_T) (strchr ("0123456789ABCDEF", *ch) != NO_TEXT);
1017 }
1018 } else {
1019 if (*ch == RADIX_CHAR) {
1020 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1021 ret = (BOOL_T) (strchr ("0123456789abcdef", *ch) != NO_TEXT);
1022 }
1023 }
1024 restore_state (ref_l, ref_s, ch);
1025 return ret;
1026 }
1027
1028 //! @brief Whether input shows decimal point.
1029
1030 BOOL_T is_decimal_point (LINE_T ** ref_l, char **ref_s, char *ch)
1031 {
1032 BOOL_T ret = A68G_FALSE;
1033 save_state (*ref_l, *ref_s, *ch);
1034 if (*ch == POINT_CHAR) {
1035 char exp_syms[3];
1036 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
1037 exp_syms[0] = EXPONENT_CHAR;
1038 exp_syms[1] = (char) TO_UPPER (EXPONENT_CHAR);
1039 exp_syms[2] = NULL_CHAR;
1040 } else {
1041 exp_syms[0] = (char) TO_UPPER (EXPONENT_CHAR);
1042 exp_syms[1] = BACKSLASH_CHAR;
1043 exp_syms[2] = NULL_CHAR;
1044 }
1045 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1046 if (strchr (exp_syms, *ch) != NO_TEXT) {
1047 *ch = next_char (ref_l, ref_s, A68G_TRUE);
1048 ret = (BOOL_T) (strchr ("+-0123456789", *ch) != NO_TEXT);
1049 } else {
1050 ret = (BOOL_T) (strchr ("0123456789", *ch) != NO_TEXT);
1051 }
1052 }
1053 restore_state (ref_l, ref_s, ch);
1054 return ret;
1055 }
1056
1057 //! @brief Get next token from internal copy of source file..
1058
1059 void get_next_token (BOOL_T in_format, LINE_T ** ref_l, char **ref_s, LINE_T ** start_l, char **start_c, int *att)
1060 {
1061 char c = **ref_s, *sym = A68G_PARSER (scan_buf);
1062 sym[0] = NULL_CHAR;
1063 get_good_char (&c, ref_l, ref_s);
1064 *start_l = *ref_l;
1065 *start_c = *ref_s;
1066 if (c == STOP_CHAR) {
1067 // We are at EOF.
1068 (sym++)[0] = STOP_CHAR;
1069 sym[0] = NULL_CHAR;
1070 return;
1071 }
1072 // In a format.
1073 if (in_format) {
1074 char *format_items;
1075 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
1076 format_items = "/%\\+-.abcdefghijklmnopqrstuvwxyz";
1077 } else if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
1078 format_items = "/%\\+-.ABCDEFGHIJKLMNOPQRSTUVWXYZ";
1079 } else {
1080 format_items = "/%\\+-.abcdefghijklmnopqrstuvwxyz";
1081 }
1082 if (strchr (format_items, c) != NO_TEXT) {
1083 // General format items.
1084 (sym++)[0] = c;
1085 sym[0] = NULL_CHAR;
1086 *att = get_format_item (c);
1087 (void) next_char (ref_l, ref_s, A68G_FALSE);
1088 return;
1089 }
1090 if (IS_DIGIT (c)) {
1091 // INT denotation for static replicator.
1092 SCAN_DIGITS (c);
1093 sym[0] = NULL_CHAR;
1094 *att = STATIC_REPLICATOR;
1095 return;
1096 }
1097 }
1098 // Not in a format.
1099 if (IS_UPPER (c)) {
1100 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
1101 // Upper case word - bold tag.
1102 while (IS_UPPER (c) || c == '_') {
1103 (sym++)[0] = c;
1104 c = next_char (ref_l, ref_s, A68G_FALSE);
1105 }
1106 sym[0] = NULL_CHAR;
1107 *att = BOLD_TAG;
1108 } else if (OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
1109 while (IS_UPPER (c) || IS_DIGIT (c) || c == '_') {
1110 (sym++)[0] = c;
1111 c = next_char (ref_l, ref_s, A68G_TRUE);
1112 }
1113 sym[0] = NULL_CHAR;
1114 *att = IDENTIFIER;
1115 }
1116 } else if (c == '\'') {
1117 // Quote, uppercase word, quote - bold tag.
1118 int k = 0;
1119 c = next_char (ref_l, ref_s, A68G_FALSE);
1120 while (IS_UPPER (c) || IS_DIGIT (c) || c == '_') {
1121 (sym++)[0] = c;
1122 k++;
1123 c = next_char (ref_l, ref_s, A68G_TRUE);
1124 }
1125 SCAN_ERROR (k == 0, *start_l, *start_c, ERROR_QUOTED_BOLD_TAG);
1126 sym[0] = NULL_CHAR;
1127 *att = BOLD_TAG;
1128 // Skip terminating quote, or complain if it is not there.
1129 SCAN_ERROR (c != '\'', *start_l, *start_c, ERROR_QUOTED_BOLD_TAG);
1130 c = next_char (ref_l, ref_s, A68G_FALSE);
1131 } else if (IS_LOWER (c)) {
1132 // Lower case word - identifier.
1133 while (IS_LOWER (c) || IS_DIGIT (c) || c == '_') {
1134 (sym++)[0] = c;
1135 c = next_char (ref_l, ref_s, A68G_TRUE);
1136 }
1137 sym[0] = NULL_CHAR;
1138 *att = IDENTIFIER;
1139 } else if (c == POINT_CHAR) {
1140 // Begins with a point symbol - point, dotdot, L REAL denotation.
1141 if (is_decimal_point (ref_l, ref_s, &c)) {
1142 (sym++)[0] = '0';
1143 (sym++)[0] = POINT_CHAR;
1144 c = next_char (ref_l, ref_s, A68G_TRUE);
1145 SCAN_DIGITS (c);
1146 if (is_exp_char (ref_l, ref_s, &c)) {
1147 SCAN_EXPONENT_PART (c);
1148 }
1149 sym[0] = NULL_CHAR;
1150 *att = REAL_DENOTATION;
1151 } else {
1152 c = next_char (ref_l, ref_s, A68G_TRUE);
1153 if (c == POINT_CHAR) {
1154 (sym++)[0] = POINT_CHAR;
1155 (sym++)[0] = POINT_CHAR;
1156 sym[0] = NULL_CHAR;
1157 *att = DOTDOT_SYMBOL;
1158 c = next_char (ref_l, ref_s, A68G_FALSE);
1159 } else {
1160 (sym++)[0] = POINT_CHAR;
1161 sym[0] = NULL_CHAR;
1162 *att = POINT_SYMBOL;
1163 }
1164 }
1165 } else if (IS_DIGIT (c)) {
1166 // Something that begins with a digit - L INT denotation, L REAL denotation.
1167 SCAN_DIGITS (c);
1168 if (is_decimal_point (ref_l, ref_s, &c)) {
1169 c = next_char (ref_l, ref_s, A68G_TRUE);
1170 if (is_exp_char (ref_l, ref_s, &c)) {
1171 (sym++)[0] = POINT_CHAR;
1172 (sym++)[0] = '0';
1173 SCAN_EXPONENT_PART (c);
1174 *att = REAL_DENOTATION;
1175 } else {
1176 (sym++)[0] = POINT_CHAR;
1177 SCAN_DIGITS (c);
1178 if (is_exp_char (ref_l, ref_s, &c)) {
1179 SCAN_EXPONENT_PART (c);
1180 }
1181 *att = REAL_DENOTATION;
1182 }
1183 } else if (is_exp_char (ref_l, ref_s, &c)) {
1184 SCAN_EXPONENT_PART (c);
1185 *att = REAL_DENOTATION;
1186 } else if (is_radix_char (ref_l, ref_s, &c)) {
1187 (sym++)[0] = c;
1188 c = next_char (ref_l, ref_s, A68G_TRUE);
1189 if (OPTION_STROPPING (&A68G_JOB) == UPPER_STROPPING) {
1190 while (IS_DIGIT (c) || strchr ("abcdef", c) != NO_TEXT) {
1191 (sym++)[0] = c;
1192 c = next_char (ref_l, ref_s, A68G_TRUE);
1193 }
1194 } else {
1195 while (IS_DIGIT (c) || strchr ("ABCDEF", c) != NO_TEXT) {
1196 (sym++)[0] = c;
1197 c = next_char (ref_l, ref_s, A68G_TRUE);
1198 }
1199 }
1200 *att = BITS_DENOTATION;
1201 } else {
1202 *att = INT_DENOTATION;
1203 }
1204 sym[0] = NULL_CHAR;
1205 } else if (c == QUOTE_CHAR) {
1206 // STRING denotation.
1207 BOOL_T stop = A68G_FALSE;
1208 while (!stop) {
1209 c = next_char (ref_l, ref_s, A68G_FALSE);
1210 while (c != QUOTE_CHAR && c != STOP_CHAR) {
1211 SCAN_ERROR (EOL (c), *start_l, *start_c, ERROR_LONG_STRING);
1212 (sym++)[0] = c;
1213 c = next_char (ref_l, ref_s, A68G_FALSE);
1214 }
1215 SCAN_ERROR (*ref_l == NO_LINE, *start_l, *start_c, ERROR_UNTERMINATED_STRING);
1216 c = next_char (ref_l, ref_s, A68G_FALSE);
1217 if (c == QUOTE_CHAR) {
1218 (sym++)[0] = QUOTE_CHAR;
1219 } else {
1220 stop = A68G_TRUE;
1221 }
1222 }
1223 sym[0] = NULL_CHAR;
1224 *att = (in_format ? LITERAL : ROW_CHAR_DENOTATION);
1225 } else if (strchr ("#$()[]{},;@", c) != NO_TEXT) {
1226 // Single character symbols.
1227 (sym++)[0] = c;
1228 (void) next_char (ref_l, ref_s, A68G_FALSE);
1229 sym[0] = NULL_CHAR;
1230 *att = 0;
1231 } else if (c == '|') {
1232 // Bar.
1233 (sym++)[0] = c;
1234 c = next_char (ref_l, ref_s, A68G_FALSE);
1235 if (c == ':') {
1236 (sym++)[0] = c;
1237 (void) next_char (ref_l, ref_s, A68G_FALSE);
1238 }
1239 sym[0] = NULL_CHAR;
1240 *att = 0;
1241 } else if (c == '!' && OPTION_STROPPING (&A68G_JOB) == QUOTE_STROPPING) {
1242 // Bar, will be replaced with modern variant.
1243 // For this reason ! is not a MONAD with quote-stropping.
1244 (sym++)[0] = '|';
1245 c = next_char (ref_l, ref_s, A68G_FALSE);
1246 if (c == ':') {
1247 (sym++)[0] = c;
1248 (void) next_char (ref_l, ref_s, A68G_FALSE);
1249 }
1250 sym[0] = NULL_CHAR;
1251 *att = 0;
1252 } else if (c == ':') {
1253 // Colon, semicolon, IS, ISNT.
1254 (sym++)[0] = c;
1255 c = next_char (ref_l, ref_s, A68G_FALSE);
1256 if (c == '=') {
1257 (sym++)[0] = c;
1258 if ((c = next_char (ref_l, ref_s, A68G_FALSE)) == ':') {
1259 (sym++)[0] = c;
1260 c = next_char (ref_l, ref_s, A68G_FALSE);
1261 }
1262 } else if (c == '/') {
1263 (sym++)[0] = c;
1264 if ((c = next_char (ref_l, ref_s, A68G_FALSE)) == '=') {
1265 (sym++)[0] = c;
1266 if ((c = next_char (ref_l, ref_s, A68G_FALSE)) == ':') {
1267 (sym++)[0] = c;
1268 c = next_char (ref_l, ref_s, A68G_FALSE);
1269 }
1270 }
1271 } else if (c == ':') {
1272 (sym++)[0] = c;
1273 if ((c = next_char (ref_l, ref_s, A68G_FALSE)) == '=') {
1274 (sym++)[0] = c;
1275 }
1276 }
1277 sym[0] = NULL_CHAR;
1278 *att = 0;
1279 } else if (c == '=') {
1280 // Operator starting with "=".
1281 char *scanned = sym;
1282 (sym++)[0] = c;
1283 c = next_char (ref_l, ref_s, A68G_FALSE);
1284 if (strchr (NOMADS, c) != NO_TEXT) {
1285 (sym++)[0] = c;
1286 c = next_char (ref_l, ref_s, A68G_FALSE);
1287 }
1288 if (c == '=') {
1289 (sym++)[0] = c;
1290 if (next_char (ref_l, ref_s, A68G_FALSE) == ':') {
1291 (sym++)[0] = ':';
1292 c = next_char (ref_l, ref_s, A68G_FALSE);
1293 if (strlen (sym) < 4 && c == '=') {
1294 (sym++)[0] = '=';
1295 (void) next_char (ref_l, ref_s, A68G_FALSE);
1296 }
1297 }
1298 } else if (c == ':') {
1299 (sym++)[0] = c;
1300 sym[0] = NULL_CHAR;
1301 if (next_char (ref_l, ref_s, A68G_FALSE) == '=') {
1302 (sym++)[0] = '=';
1303 (void) next_char (ref_l, ref_s, A68G_FALSE);
1304 } else {
1305 SCAN_ERROR (!(strcmp (scanned, "=:") == 0 || strcmp (scanned, "==:") == 0), *start_l, *start_c, ERROR_INVALID_OPERATOR_TAG);
1306 }
1307 }
1308 sym[0] = NULL_CHAR;
1309 if (strcmp (scanned, "=") == 0) {
1310 *att = EQUALS_SYMBOL;
1311 } else {
1312 *att = OPERATOR;
1313 }
1314 } else if (strchr (MONADS, c) != NO_TEXT || strchr (NOMADS, c) != NO_TEXT) {
1315 // Operator.
1316 char *scanned = sym;
1317 (sym++)[0] = c;
1318 c = next_char (ref_l, ref_s, A68G_FALSE);
1319 if (strchr (NOMADS, c) != NO_TEXT) {
1320 (sym++)[0] = c;
1321 c = next_char (ref_l, ref_s, A68G_FALSE);
1322 }
1323 if (c == '=') {
1324 (sym++)[0] = c;
1325 if (next_char (ref_l, ref_s, A68G_FALSE) == ':') {
1326 (sym++)[0] = ':';
1327 c = next_char (ref_l, ref_s, A68G_FALSE);
1328 if (strlen (scanned) < 4 && c == '=') {
1329 (sym++)[0] = '=';
1330 (void) next_char (ref_l, ref_s, A68G_FALSE);
1331 }
1332 }
1333 } else if (c == ':') {
1334 (sym++)[0] = c;
1335 sym[0] = NULL_CHAR;
1336 if (next_char (ref_l, ref_s, A68G_FALSE) == '=') {
1337 (sym++)[0] = '=';
1338 sym[0] = NULL_CHAR;
1339 (void) next_char (ref_l, ref_s, A68G_FALSE);
1340 } else {
1341 SCAN_ERROR (strcmp (&(scanned[1]), "=:") != 0, *start_l, *start_c, ERROR_INVALID_OPERATOR_TAG);
1342 }
1343 }
1344 sym[0] = NULL_CHAR;
1345 *att = OPERATOR;
1346 } else {
1347 // Afuuus ... strange characters!.
1348 unworthy (*start_l, *start_c, (int) c);
1349 }
1350 }
1351
1352 //! @brief Whether att opens an embedded clause.
1353
1354 BOOL_T open_nested_clause (int att)
1355 {
1356 switch (att) {
1357 case OPEN_SYMBOL:
1358 case BEGIN_SYMBOL:
1359 case PAR_SYMBOL:
1360 case IF_SYMBOL:
1361 case CASE_SYMBOL:
1362 case FOR_SYMBOL:
1363 case FROM_SYMBOL:
1364 case BY_SYMBOL:
1365 case TO_SYMBOL:
1366 case DOWNTO_SYMBOL:
1367 case WHILE_SYMBOL:
1368 case DO_SYMBOL:
1369 case SUB_SYMBOL:
1370 case ACCO_SYMBOL: {
1371 return A68G_TRUE;
1372 }
1373 }
1374 return A68G_FALSE;
1375 }
1376
1377 //! @brief Whether att closes an embedded clause.
1378
1379 BOOL_T close_nested_clause (int att)
1380 {
1381 switch (att) {
1382 case CLOSE_SYMBOL:
1383 case END_SYMBOL:
1384 case FI_SYMBOL:
1385 case ESAC_SYMBOL:
1386 case OD_SYMBOL:
1387 case BUS_SYMBOL:
1388 case OCCA_SYMBOL: {
1389 return A68G_TRUE;
1390 }
1391 }
1392 return A68G_FALSE;
1393 }
1394
1395 //! @brief Cast a string to lower case.
1396
1397 void make_lower_case (char *p)
1398 {
1399 for (; p != NO_TEXT && p[0] != NULL_CHAR; p++) {
1400 p[0] = (char) TO_LOWER (p[0]);
1401 }
1402 }
1403
1404 //! @brief Construct a linear list of tokens.
1405
1406 void tokenise_source (NODE_T ** root, int level, BOOL_T in_format, LINE_T ** l, char **s, LINE_T ** start_l, char **start_c)
1407 {
1408 char *lpr = NO_TEXT;
1409 int lprt = 0;
1410 while (l != NO_REF && !A68G_PARSER (stop_scanner)) {
1411 int att = 0;
1412 get_next_token (in_format, l, s, start_l, start_c, &att);
1413 if (A68G_PARSER (scan_buf)[0] == STOP_CHAR) {
1414 A68G_PARSER (stop_scanner) = A68G_TRUE;
1415 } else if (strlen (A68G_PARSER (scan_buf)) > 0 || att == ROW_CHAR_DENOTATION || att == LITERAL) {
1416 KEYWORD_T *kw;
1417 char *c = NO_TEXT;
1418 BOOL_T make_node = A68G_TRUE;
1419 char *trailing = NO_TEXT;
1420 if (att != IDENTIFIER) {
1421 kw = find_keyword (A68G (top_keyword), A68G_PARSER (scan_buf));
1422 } else {
1423 kw = NO_KEYWORD;
1424 }
1425 if (!(kw != NO_KEYWORD && att != ROW_CHAR_DENOTATION)) {
1426 if (att == IDENTIFIER) {
1427 make_lower_case (A68G_PARSER (scan_buf));
1428 }
1429 if (att != ROW_CHAR_DENOTATION && att != LITERAL) {
1430 size_t len = strlen (A68G_PARSER (scan_buf));
1431 while (len >= 1 && A68G_PARSER (scan_buf)[len - 1] == '_') {
1432 trailing = "_";
1433 A68G_PARSER (scan_buf)[len - 1] = NULL_CHAR;
1434 len--;
1435 }
1436 }
1437 c = TEXT (add_token (&A68G (top_token), A68G_PARSER (scan_buf)));
1438 } else {
1439 if (IS (kw, TO_SYMBOL)) {
1440 // Merge GO and TO to GOTO.
1441 if (*root != NO_NODE && IS (*root, GO_SYMBOL)) {
1442 ATTRIBUTE (*root) = GOTO_SYMBOL;
1443 NSYMBOL (*root) = TEXT (find_keyword (A68G (top_keyword), "GOTO"));
1444 make_node = A68G_FALSE;
1445 } else {
1446 att = ATTRIBUTE (kw);
1447 c = TEXT (kw);
1448 }
1449 } else {
1450 if (att == 0 || att == BOLD_TAG) {
1451 att = ATTRIBUTE (kw);
1452 }
1453 c = TEXT (kw);
1454 // Handle pragments.
1455 if (att == STYLE_II_COMMENT_SYMBOL || att == STYLE_I_COMMENT_SYMBOL || att == BOLD_COMMENT_SYMBOL) {
1456 char *nlpr = pragment (ATTRIBUTE (kw), l, s);
1457 if (lpr == NO_TEXT || strlen (lpr) == 0) {
1458 lpr = nlpr;
1459 } else {
1460 char *stale = lpr;
1461 lpr = new_string (lpr, "\n\n", nlpr, NO_TEXT);
1462 a68g_free (nlpr);
1463 a68g_free (stale);
1464 }
1465 lprt = att;
1466 make_node = A68G_FALSE;
1467 } else if (att == STYLE_I_PRAGMAT_SYMBOL || att == BOLD_PRAGMAT_SYMBOL) {
1468 char *nlpr = pragment (ATTRIBUTE (kw), l, s);
1469 if (lpr == NO_TEXT || strlen (lpr) == 0) {
1470 lpr = nlpr;
1471 } else {
1472 char *stale = lpr;
1473 lpr = new_string (lpr, "\n\n", nlpr, NO_TEXT);
1474 a68g_free (nlpr);
1475 a68g_free (stale);
1476 }
1477 lprt = att;
1478 if (!A68G_PARSER (stop_scanner)) {
1479 (void) set_options (OPTION_LIST (&A68G_JOB), A68G_FALSE);
1480 make_node = A68G_FALSE;
1481 }
1482 }
1483 }
1484 }
1485 // Add token to the tree.
1486 if (make_node) {
1487 NODE_T *q = new_node ();
1488 INFO (q) = new_node_info ();
1489 switch (att) {
1490 case ASSIGN_SYMBOL:
1491 case END_SYMBOL:
1492 case ESAC_SYMBOL:
1493 case OD_SYMBOL:
1494 case OF_SYMBOL:
1495 case FI_SYMBOL:
1496 case CLOSE_SYMBOL:
1497 case BUS_SYMBOL:
1498 case COLON_SYMBOL:
1499 case COMMA_SYMBOL:
1500 case DOTDOT_SYMBOL:
1501 case SEMI_SYMBOL: {
1502 GINFO (q) = NO_GINFO;
1503 break;
1504 }
1505 default: {
1506 GINFO (q) = new_genie_info ();
1507 break;
1508 }
1509 }
1510 STATUS (q) = OPTION_NODEMASK (&A68G_JOB);
1511 LINE (INFO (q)) = *start_l;
1512 CHAR_IN_LINE (INFO (q)) = *start_c;
1513 PRIO (INFO (q)) = 0;
1514 PROCEDURE_LEVEL (INFO (q)) = 0;
1515 ATTRIBUTE (q) = att;
1516 NSYMBOL (q) = c;
1517 PREVIOUS (q) = *root;
1518 SUB (q) = NEXT (q) = NO_NODE;
1519 TABLE (q) = NO_TABLE;
1520 MOID (q) = NO_MOID;
1521 TAX (q) = NO_TAG;
1522 if (lpr != NO_TEXT) {
1523 NPRAGMENT (q) = lpr;
1524 NPRAGMENT_TYPE (q) = lprt;
1525 lpr = NO_TEXT;
1526 lprt = 0;
1527 }
1528 if (*root != NO_NODE) {
1529 NEXT (*root) = q;
1530 }
1531 if (TOP_NODE (&A68G_JOB) == NO_NODE) {
1532 TOP_NODE (&A68G_JOB) = q;
1533 }
1534 *root = q;
1535 if (trailing != NO_TEXT) {
1536 diagnostic (A68G_WARNING, q, WARNING_TRAILING, trailing, att);
1537 }
1538 }
1539 // Redirection in tokenising formats. The scanner is a recursive-descent type as
1540 // to know when it scans a format text and when not.
1541 if (in_format && att == FORMAT_DELIMITER_SYMBOL) {
1542 return;
1543 } else if (!in_format && att == FORMAT_DELIMITER_SYMBOL) {
1544 tokenise_source (root, level + 1, A68G_TRUE, l, s, start_l, start_c);
1545 } else if (in_format && open_nested_clause (att)) {
1546 NODE_T *z = PREVIOUS (*root);
1547 if (z != NO_NODE && is_one_of (z, FORMAT_ITEM_N, FORMAT_ITEM_G, FORMAT_ITEM_H, FORMAT_ITEM_F, STOP)) {
1548 tokenise_source (root, level, A68G_FALSE, l, s, start_l, start_c);
1549 } else if (att == OPEN_SYMBOL) {
1550 ATTRIBUTE (*root) = FORMAT_OPEN_SYMBOL;
1551 } else if (OPTION_BRACKETS (&A68G_JOB) && att == SUB_SYMBOL) {
1552 ATTRIBUTE (*root) = FORMAT_OPEN_SYMBOL;
1553 } else if (OPTION_BRACKETS (&A68G_JOB) && att == ACCO_SYMBOL) {
1554 ATTRIBUTE (*root) = FORMAT_OPEN_SYMBOL;
1555 }
1556 } else if (!in_format && level > 0 && open_nested_clause (att)) {
1557 tokenise_source (root, level + 1, A68G_FALSE, l, s, start_l, start_c);
1558 } else if (!in_format && level > 0 && close_nested_clause (att)) {
1559 return;
1560 } else if (in_format && att == CLOSE_SYMBOL) {
1561 ATTRIBUTE (*root) = FORMAT_CLOSE_SYMBOL;
1562 } else if (OPTION_BRACKETS (&A68G_JOB) && in_format && att == BUS_SYMBOL) {
1563 ATTRIBUTE (*root) = FORMAT_CLOSE_SYMBOL;
1564 } else if (OPTION_BRACKETS (&A68G_JOB) && in_format && att == OCCA_SYMBOL) {
1565 ATTRIBUTE (*root) = FORMAT_CLOSE_SYMBOL;
1566 }
1567 }
1568 }
1569 }
1570
1571 //! @brief Tokenise source file, build initial syntax tree.
1572
1573 BOOL_T lexical_analyser (void)
1574 {
1575 LINE_T *l = NO_LINE, *start_l = NO_LINE;
1576 char *s = NO_TEXT, *start_c = NO_TEXT;
1577 NODE_T *root = NO_NODE;
1578 A68G_PARSER (scan_buf) = NO_TEXT;
1579 A68G_PARSER (max_scan_buf_length) = A68G_PARSER (source_file_size) = get_source_size ();
1580 // Errors in file?.
1581 if (A68G_PARSER (max_scan_buf_length) == 0) {
1582 return A68G_FALSE;
1583 }
1584 if (OPTION_RUN_SCRIPT (&A68G_JOB)) {
1585 A68G_PARSER (scan_buf) = (char *) get_temp_heap_space (8 + A68G_PARSER (max_scan_buf_length));
1586 if (!read_script_file ()) {
1587 return A68G_FALSE;
1588 }
1589 } else {
1590 A68G_PARSER (max_scan_buf_length) += A68G_KILO; // for the environ, more than enough
1591 A68G_PARSER (scan_buf) = (char *) get_temp_heap_space (A68G_PARSER (max_scan_buf_length));
1592 // Errors in file?.
1593 if (!read_source_file ()) {
1594 return A68G_FALSE;
1595 }
1596 }
1597 // Start tokenising.
1598 A68G_PARSER (read_error) = A68G_FALSE;
1599 A68G_PARSER (stop_scanner) = A68G_FALSE;
1600 if ((l = TOP_LINE (&A68G_JOB)) != NO_LINE) {
1601 s = STRING (l);
1602 }
1603 tokenise_source (&root, 0, A68G_FALSE, &l, &s, &start_l, &start_c);
1604 return A68G_TRUE;
1605 }
|
© 2002-2026 J.M. van der Veer (jmvdveer@xs4all.nl)
|
