#ifndef LEX_TOKENS_H
#define LEX_TOKENS_H
/* Define LEX_DEBUG to enable token printing and describing functions. */
enum {
/*
* EOF is not emitted by lexer, but may be used by driver after
* last buffer is processed.
*/
LEX_TOK_EOF = 0,
/*
* Either EOB or EOS is emitted as the last token before exit,
* or also ABORT in some lexers. Unterminated string or comment
* will be emitted immediately one of these when relevant.
*
* It may be useful to redefine lex_emit_eos and lex_emit_eob to
* produce LEX_TOK_EOF or error directly for simple string lexing.
*/
LEX_TOK_EOB = 1,
LEX_TOK_EOS = 2,
/*
* ABORT can be used for early exit by some lexers while other
* lexers may choose to run to buffer end regardless of input (with
* the exception of deeply nested comments).
*/
LEX_TOK_ABORT = 3,
/*
* Byte order marker. Only happen if lexer was started in bom mode
* and the input stream contains a leading bom marker.
* The token can only be the first token in the stream. Utf-8 is the
* only supported bom, but the lexeme may be checked in case other
* boms are added later. Normally it is routed to lex_emit_other
* along with comments so it just ignores the bom if present. It is
* generally recommended to consume utf-8 bom for interoperability,
* but also to not store it for the same reason.
*/
LEX_TOK_BOM,
/*
* Any control character that is not newline or blank will be
* emitted as single character token here. This token is discussed
* in several comments below. For strings and comments, also
* blank control characters will be emitted since they are usually
* not desired unexpectd.
*/
LEX_TOK_CTRL,
LEX_TOK_STRING_CTRL,
LEX_TOK_COMMENT_CTRL,
/*
* Any printable ASCII character that is not otherwise consumed will
* be issued as a single length symbol token. Further discussion
* below. The symbol and CTRL tokens ensure that the entire input
* stream is covered by tokens. If utf-8 identifies have not been
* flagged, utf-8 leading characters may also end up here, and so
* my utf-8 characters in general, that are not viewed as valid
* identifiers (depending on configuration).
*/
LEX_TOK_SYMBOL,
/*
* Variable length identifier starting with (_A-Za-z) by default and
* followed by zero or more (_A-Za-z0-9) characters. (_) can be
* flagged out. utf-8 can be flagged in. Be default any non-ASCII
* character (0x80 and above), is treated as part of an identifier
* for simplicity and speed, but this may be redefined. Any broken
* utf-8 is not sanitized, thus 0x80 would be a valid identifier
* token with utf-8 identifiers enabled, and otherwise it would be a
* symbol token.
*
* The ID does a magic trick: It maps the lexeme to a very simple
* and fast 32 bit hash code called a tag. The tag is emitted with
* the id token and can be used for fast keyword lookup. The
* hash tag is:
*
* (length)(first char)(second char)(last char)
*
* where length is ASCII '0' .. '9' where any length overflow is an
* arbitrary value, but such that the length is never longer than
* the lexeme. The last char is the last char regardless of length.
* For short identifiers, the second char may be the first char
* duplicated, and the last char may be first char.
*
* This code is very simple to write by hand: "5whe" means while,
* and can be used in a case switch before a strcmp with "while".
* Conflicts are possible, but then several keywords are tested like
* any other hash conflict. This keyword lookup is user driven, but
* can follow example code quite straightforward.
*
* The lex_emit_id macro can be implemented to provide the above
* lookup and inject a keyword token instead. By convention such
* tokens have negative values to avoid conflicts with lexer
* generated tokens.
*
* The ID also has a special role in prefixes and suffixes: C string
* literals like (L"hello") and numeric literals like (42f) are
* lexed as two tokens, one of which is an ID. The parser must
* process this and observe absence of whitespace where such syntax
* is relevant.
*
* While not specific to ID, the emitter macroes can be designed to
* keep track of start of lines and end of whitespace and attach
* state flags to each token (at line start, after whitespace). The
* whitespace tokens can then be dropped. This might help parsing
* things like suffixes efficiently.
*/
LEX_TOK_ID,
/*
* C-int :: pos-dec-digit dec-digit *
* Julia-int ::= dec-digit+
*
* pos-dec-digit ::= '1'..'9'
* dec-digit ::= '0'..'9'
*
* Floating point numbers take precedence when possible so 00.10 is
* always a deciaml floating point value when decimal floats are
* enabled.
*
* The C-int is automatically enabled if C-octals are enabled, and
* disabled otherwise. There is no specific Julia-int type - we just
* use the terminology to represent integers with leading zeroes.
*
* Julia style integers accept leading zeroes. C style integers with
* leading zeroes are consumed as C style octal numbers, so 0019 is
* parsed as either 0019(Julia-int), or 001(C-octal), 9(C-int).
*
* Single digit '0' maps to octal when C-octals are enabled and to
* Julia-int otherwise. (Yes, integers are not that simple, it
* seems).
*
* Both C and Julia octal numbers (see octal token) can be active
* simultaneously. This can be used to control leading zero
* behavior, even if C-octal numbers are not part of the grammar
* being parsed. For example, a language might use 0o777 octal
* numbers and disallow 0777 integers. Enabling C-octals makes this
* easy to detect (but should accept octal 0).
*
* There is no destinction between the styles in the int token, but
* leading zeroes are easily detected in the lexeme.
*
* Constant suffixes like 1L are treated as 1(INT), and L(ID). The
* same goes for other numeric values.
*
* Parser should check for leading zeroes and decide if it is valid,
* a warning, or an error (it is in JSON). This also goes for float.
*
* Numericals, not limited to INT, may appear shorter than they are
* due to buffer splits. Special recovery is required, but will only
* happen just before EOS or EOB tokens (i.e. buffer split events).
*/
LEX_TOK_INT,
/*
* float ::= (int ['.' dec-digits*] dec-exponent)
* | ([int] '.' dec-digits* [dec-exponent])
* dec-exponents ::= ('e' | 'E') ['+' | '-'] dec-digits*
* dec-digits ::= '0'..'9'
* int ::= dec-digits*
*
* Consumes a superset of C float representation without suffix.
* Some invalid tokens such as 0.E are accepted. Valid tokens such
* as 00.10 take precedence over octal numbers even if it is a
* prefix, and the same is obviously true with respect to decimal
* integers.
*
* JSON does not allow leading zeroes, and also not leading '.'.
* This can easily be checked in the lexeme.
*
* The octal notation affecting integer leading zeroes is not
* relevant to floats because floats take precedence over octal and
* decimal int when containing '.', 'e' or 'E'.
*/
LEX_TOK_FLOAT,
/*
* binary ::= (0b | 0B) ('0' | '1')*
*
* 0b100 or just 0b, parser must check that digits are present,
* otherwise it may be interpreted as zero, just like octal zero
* in C.
*
* Like 0X hex, 0B can be flagged out because Julia v0.3 does not
* support uppercase 0B.
*/
LEX_TOK_BINARY,
/*
* C-octal ::= 0 octal-digit*
* octal-digits ::= '0'..'7'
*
* Julia-octal ::= 0o octal-digits*
* octal-digits ::= '0'..'7'
*
* 0777 for C style octal numbers, or 0o777 for Julia syntax. Julia
* v.0.3 does not allow uppercase 0O777, it would mean 0 * O777.
*
* When enabled, decimal floating points take precedence: 00.10 is
* parsed as 00.10(decimal float), as per C standard.
*
* NOTE: It is possible for both styles to be active simultaneously.
* This may be relevant in order to control handling of leading
* zeroes in decimal integers.
*
* If C-octal numbers are flagged out, leading zeroes are mapped to
* integers and the numerical value may change. Julia behaves this
* way. Nothing prevents support of both C and Julia octal numbers,
* but leading zeroes will then be interpreted the C way - it is not
* recommended to do this.
*/
LEX_TOK_OCTAL,
/*
* hex ::= hex-int
* hex-digits ::= 'a'..'f'| 'A'..'f' | '0'..'9'
* hex-int ::= (0x | 0X) hex_digts*
*
* where hex_digits are customizable (e.g. all lower case), and hex
* prefix 0x can be flagged to be lower case only (as in Julia).
*
* If hex floats are enabled, they take precedence:
* 0x1.0(hex-float), if not, 0x1.0 will parse as: 0x1(hex) followed
* by .0(decimal float).
*
* The lead prefix 0x may by flagged to be lower case only because
* this is required by Julia v0.3 where 0X means 0 * X. Julia
* accepts uppercase in the remaining hex digits (and exponent for
* floats). This could possibly change in future versions.
*
* The zero length sequence (0x | 0X) is accepted and left to the
* parser since the lexer emits a token for everything it sees.
* Conceptually it may be interpreted as zero, equivalent to 0 being
* both octal prefix and numeric 0 in C style octal representation.
* Or it may be an error.
*/
LEX_TOK_HEX,
/*
* hex_float ::= hex-int ['.' hex_digit*] hex-exponent
* hex-exponent ::= ('p' | 'P') ['+' | '-'] decimal-digit*
* decimal-digit ::= '0'..'9'
*
* A superset of IEEE-754-2008 Hexadecimal Floating Point notation.
*
* We require the exponent to be present, but does not ensure the
* value is otherwise complete, e.g. 0x1p+ would be accepted. The p
* is needed because otherwise 0x1.f could be accepted, and f is a
* float suffix in C, and juxtapostion factor (0x1. * f) in Julia,
* at least, that is one possible interpretation.
*
* The exponent can be flagged optional in which case 0x1.f will be
* consumed as a single hex float toke as a single hex float token.
* This may either simply be accepted in some grammars, or used to
* provide an error message. If the exponent is required, 0x1.f will
* be lexed as three tokens:
*
* <'0x1'(hex int), '.'(op), 'f'(id)>.
*
* Thus it may be a good idea to allow the exponent to be optional
* anyway and issue an error message or warning if the p is absent
* later in the parsing stage.
*
* Note that, as per IEEE-754, the exponent is a decimal power of
* two. In other words, the number of bits to shift the
* (hexa)decimal point. Also note that it is p and not e because e
* is a hex digit.
*/
LEX_TOK_HEX_FLOAT,
/*
* blank ::= ('\t' | '\x20')+
*
* Longest run in buffer holding only '\t' and '\x20' (space).
*
* buffer splits may generate adjacent blanks depending on recovery
* processing. (The same goes for other line oriented runs such as
* string parts and comment parts).
*/
LEX_TOK_BLANK,
/* newline ::= '\r' | '\n' | '\r\n' | '\n\r'
*
* Will always appear, also inside strings and comments. Can be used
* to track line starts and counts reliably as only one newline is
* issued at a time, and it is issued everywhere, also in strings
* and comments.
*
* May be preceeded by string escape token inside strings. This can
* be interpreted as line continuation within strings specifically,
* as is the case in Python and Javascript (and in C via
* pre-processor).
*
* The LEX_TOK_STRING_NEWLINE is emitted inside strings so the ordinary
* newline may be ignored in comments and other non-string content.
*/
LEX_TOK_NEWLINE,
LEX_TOK_STRING_NEWLINE,
/*
* string ::= string_start
* (string_part | string_escape |
* string_ctrl | string_newline)*
* (string_end | string_unterminated)
*
* There are several optional string styles. They all start with
* this token. The length and content provided details. Python
* may start with """ or ''' and this token will then have length
* 3 and three quotes as lexeme content. If the lexer exits before
* string end token, the returned lexer mode will remember the
* state and can be used for reentry - this also goes for comments.
*
* Strings can only contain part, escape, newline, and control
* tokens, and either string unterminated or string end token
* at last.
*/
LEX_TOK_STRING_BEGIN,
/* Longest run without control characters, without (\), without
* newline, and without the relevant end delimiter. The run may be
* shortened due to buffer splits. The part may, as an exception,
* begin with an end delimiter character or a (\) if it was
* preceeded by a string escape token. The escape character is
* always (\). Strings that use "" or '' as escape will be treated
* as start and end of separate strings. Strings that do not supoort
* (\) should just treat escape as a part of the string.
*/
LEX_TOK_STRING_PART,
/*
* This is always a single character token (\) and only happens
* inside strings. See also string part token.
*/
LEX_TOK_STRING_ESCAPE,
/* This token is similar to string start. It may be absent at buffer
* splits, but will then an unterminated string token will be used
* just before the split event token.
*
* */
LEX_TOK_STRING_END,
/*
* This is emitted before the buffer ends, or before unescaped
* newlines for line oriented string types (the usual strings).
* At buffer splits, recovery should clean it up. The returned
* mode allow parsing to continue in a new buffer with a slight
* content overlap.
*
* If string like ("hello, world!") in C, reaches end of line, it
* may be continued" ("hello, \)newline(world!"). If this line
* continuation is flagged out, this will lead to string
* unterminated, even if not at end of buffer. For block strings
* like """hello""", this only happens at end of buffer.
*/
LEX_TOK_STRING_UNTERMINATED,
/*
*
* comment ::= comment_start
* (comment_part | ctrl | newline)*
* (comment_end | comment_unterminated)
*
*
* Comments work like strings in most respects. They emit parts, and
* control characters, but not escape characters, and cannot be
* continued at end of line. Block comments are like python block
* strings (''').
*
* Julia supports nested comments (#= ... #= =# =#). In this case
* a new start token can be emitted before an end token. If the
* parser exits due to buffer split, the mode has the nesting level
* encoded so it can resumed in a new buffer.
*
* Line comments will have their end token just before newline, or
* unterminated comment just before buffer split token (EOB or EOS).
* (\) characters are consumed by the comment part tokens and do not
* affect the end of any comment.
*
* Comment begin may include extra characters when a doc comment is
* recognized. The emitter flags this. End comments are unaffected.
*/
LEX_TOK_COMMENT_BEGIN,
LEX_TOK_COMMENT_PART,
LEX_TOK_COMMENT_END,
LEX_TOK_COMMENT_UNTERMINATED,
/*
* Issued before ABORT token if nesting level is above a predefined
* level. This is to protect against malicious and misguided
* content, otherwise the nesting level counter could wrap and
* generate a different interpretation, which could be bad. The
* parser would probably do similar things with nested tokens.
*/
LEX_TOK_COMMENT_DEEPLY_NESTED,
/* Operators are all recognized single character symbols, or up to
* four characters. The token value is the ASCII codes shifted 8
* bits per extra character, by default, but the emitter macros
* can redefine this. Values below 32 are reserved token types as
* discussed above.
*
* What exactly represents an operator depends on what the lexer has
* enabled.
*
* Printable ASCII symbols that are NOT recognized, are emitted as
* the SYMBOL token and is always length 1. The value can be derived
* from the lexeme, but not the token itself. This may be perfectly
* fine for the parser, or it may be used to indicate an error.
* There are no illegal characters per se.
*
* Non-printable ASCII characters that are not covered by newline or
* blank, are emitted as CTRL tokens. These act the same as the
* symbol token and may be used to indicate error, or to handle form
* feed and other whitespace not handled by default. Unlike symbol,
* however, CTRL also appear in strings and comments since they are
* generally not allowed and this makes it easy to capture (there is
* virtually no performance overhead in providing this service
* unless attempting to parse a binary format).
*/
/* Don't bleed into this range. */
LEX_TOK_OPERATOR_BASE = 32,
/*
* Operators use ASCII range.
* Compound operators use range 0x80 to 0x7fff
* and possibly above for triple sequences.
* Custom keywords are normally negative but can be mapped
* to any other.
*
* The layout is designed for efficient table lookup.
* Compound operators might benefit from remapping down to a smaller
* range for compact lookup tables, but it depends on the parser.
*/
};
/*
* Custom keyword token range is negative, and well below -99..0 where
* special codes are reserved.
*/
#ifndef LEX_TOK_KW_BASE
#define LEX_TOK_KW_BASE -1000
#endif
#ifndef LEX_TOK_KW_NOT_FOUND
#define LEX_TOK_KW_NOT_FOUND LEX_TOK_ID
#endif
#ifdef LEX_DEBUG
#include <stdio.h>
#include <string.h>
static const char *lex_describe_token(long token)
{
switch(token) {
case LEX_TOK_BOM: return "BOM marker";
case LEX_TOK_EOF: return "EOF";
case LEX_TOK_EOS: return "buffer zero terminated";
case LEX_TOK_EOB: return "buffer exhausted";
case LEX_TOK_ABORT: return "abort";
case LEX_TOK_CTRL: return "control";
case LEX_TOK_STRING_CTRL: return "string control";
case LEX_TOK_COMMENT_CTRL: return "comment control";
case LEX_TOK_SYMBOL: return "symbol";
case LEX_TOK_ID: return "identifier";
case LEX_TOK_INT: return "integer";
case LEX_TOK_FLOAT: return "float";
case LEX_TOK_BINARY: return "binary";
case LEX_TOK_OCTAL: return "octal";
case LEX_TOK_HEX: return "hex";
case LEX_TOK_HEX_FLOAT: return "hex float";
case LEX_TOK_BLANK: return "blank";
case LEX_TOK_NEWLINE: return "newline";
case LEX_TOK_STRING_NEWLINE: return "string newline";
case LEX_TOK_STRING_BEGIN: return "string begin";
case LEX_TOK_STRING_PART: return "string part";
case LEX_TOK_STRING_END: return "string end";
case LEX_TOK_STRING_ESCAPE: return "string escape";
case LEX_TOK_STRING_UNTERMINATED: return "unterminated string";
case LEX_TOK_COMMENT_BEGIN: return "comment begin";
case LEX_TOK_COMMENT_PART: return "comment part";
case LEX_TOK_COMMENT_END: return "comment end";
case LEX_TOK_COMMENT_UNTERMINATED: return "unterminated comment";
case LEX_TOK_COMMENT_DEEPLY_NESTED: return "deeply nested comment";
default:
if (token < LEX_TOK_EOF) {
return "keyword";
}
if (token < 32) {
return "undefined";
}
if (token < 0x100L) {
return "operator";
}
if (token < 0x10000L) {
return "compound operator";
}
if (token < 0x1000000L) {
return "tricompound operator";
}
if (token < 0x7f0000000L) {
return "quadcompound operator";
}
return "reserved";
}
}
static void lex_fprint_token(FILE *fp,
long token,
const char *first, const char *last,
int line, int pos)
{
char buf[10];
const char *lexeme = first;
int len = (int)(last - first);
switch (token) {
case LEX_TOK_EOS:
case LEX_TOK_CTRL:
sprintf(buf, "^%02x", (int)*first);
lexeme = buf;
len = strlen(buf);
break;
default:
break;
}
fprintf(fp, "%04d:%03d %s (0x%lx): `%.*s`\n",
line, pos, lex_describe_token(token), token, len, lexeme);
}
#define lex_print_token(token, first, last, line, pos) \
lex_fprint_token(stdout, token, first, last, line, pos)
#else /* LEX_DEBUG */
#define lex_describe_token(token) "debug not available"
#define lex_fprint_token(fp, token, first, last, line, pos) ((void)0)
#define lex_print_token(token, first, last, line, pos) ((void)0)
#endif /* LEX_DEBUG */
#endif /* LEX_TOKENS_H */