path: root/external/lex/tokens.h

#ifndef LEX_TOKENS_H
#define LEX_TOKENS_H

/* Define LEX_DEBUG to enable token printing and describing functions. */


enum {

    /*
     * EOF is not emitted by lexer, but may be used by driver after
     * last buffer is processed.
     */
    LEX_TOK_EOF = 0,

    /*
     * Either EOB or EOS is emitted as the last token before exit,
     * or also ABORT in some lexers. Unterminated string or comment
     * will be emitted immediately one of these when relevant.
     *
     * It may be useful to redefine lex_emit_eos and lex_emit_eob to
     * produce LEX_TOK_EOF or error directly for simple string lexing.
     */
    LEX_TOK_EOB = 1,
    LEX_TOK_EOS = 2,

    /*
     * ABORT can be used for early exit by some lexers while other
     * lexers may choose to run to buffer end regardless of input (with
     * the exception of deeply nested comments).
     */
    LEX_TOK_ABORT = 3,

    /*
     * Byte order marker. Only happen if lexer was started in bom mode
     * and the input stream contains a leading bom marker.
     * The token can only be the first token in the stream. Utf-8 is the
     * only supported bom, but the lexeme may be checked in case other
     * boms are added later. Normally it is routed to lex_emit_other
     * along with comments so it just ignores the bom if present. It is
     * generally recommended to consume utf-8 bom for interoperability,
     * but also to not store it for the same reason.
     */
    LEX_TOK_BOM,

    /*
     * Any control character that is not newline or blank will be
     * emitted as single character token here. This token is discussed
     * in several comments below. For strings and comments, also
     * blank control characters will be emitted since they are usually
     * not desired unexpectd.
     */
    LEX_TOK_CTRL,
    LEX_TOK_STRING_CTRL,
    LEX_TOK_COMMENT_CTRL,

    /*
     * Any printable ASCII character that is not otherwise consumed will
     * be issued as a single length symbol token. Further discussion
     * below. The symbol and CTRL tokens ensure that the entire input
     * stream is covered by tokens. If utf-8 identifies have not been
     * flagged, utf-8 leading characters may also end up here, and so
     * my utf-8 characters in general, that are not viewed as valid
     * identifiers (depending on configuration).
     */
    LEX_TOK_SYMBOL,

    /*
     * Variable length identifier starting with (_A-Za-z) by default and
     * followed by zero or more (_A-Za-z0-9) characters. (_) can be
     * flagged out. utf-8 can be flagged in. Be default any non-ASCII
     * character (0x80 and above), is treated as part of an identifier
     * for simplicity and speed, but this may be redefined. Any broken
     * utf-8 is not sanitized, thus 0x80 would be a valid identifier
     * token with utf-8 identifiers enabled, and otherwise it would be a
     * symbol token.
     *
     * The ID does a magic trick: It maps the lexeme to a very simple
     * and fast 32 bit hash code called a tag. The tag is emitted with
     * the id token and can be used for fast keyword lookup. The
     * hash tag is:
     *
     *     (length)(first char)(second char)(last char)
     *
     * where length is ASCII '0' .. '9' where any length overflow is an
     * arbitrary value, but such that the length is never longer than
     * the lexeme. The last char is the last char regardless of length.
     * For short identifiers, the second char may be the first char
     * duplicated, and the last char may be first char.
     *
     * This code is very simple to write by hand: "5whe" means while,
     * and can be used in a case switch before a strcmp with "while".
     * Conflicts are possible, but then several keywords are tested like
     * any other hash conflict. This keyword lookup is user driven, but
     * can follow example code quite straightforward.
     *
     * The lex_emit_id macro can be implemented to provide the above
     * lookup and inject a keyword token instead. By convention such
     * tokens have negative values to avoid conflicts with lexer
     * generated tokens.
     *
     * The ID also has a special role in prefixes and suffixes: C string
     * literals like (L"hello") and numeric literals like (42f) are
     * lexed as two tokens, one of which is an ID. The parser must
     * process this and observe absence of whitespace where such syntax
     * is relevant.
     *
     * While not specific to ID, the emitter macroes can be designed to
     * keep track of start of lines and end of whitespace and attach
     * state flags to each token (at line start, after whitespace). The
     * whitespace tokens can then be dropped. This might help parsing
     * things like suffixes efficiently.
     */
    LEX_TOK_ID,

    /*
     * C-int :: pos-dec-digit dec-digit *
     * Julia-int ::= dec-digit+
     *
     *    pos-dec-digit ::= '1'..'9'
     *    dec-digit ::= '0'..'9'
     *
     * Floating point numbers take precedence when possible so 00.10 is
     * always a deciaml floating point value when decimal floats are
     * enabled.
     *
     * The C-int is automatically enabled if C-octals are enabled, and
     * disabled otherwise. There is no specific Julia-int type - we just
     * use the terminology to represent integers with leading zeroes.
     *
     * Julia style integers accept leading zeroes. C style integers with
     * leading zeroes are consumed as C style octal numbers, so 0019 is
     * parsed as either 0019(Julia-int), or 001(C-octal), 9(C-int).
     *
     * Single digit '0' maps to octal when C-octals are enabled and to
     * Julia-int otherwise. (Yes, integers are not that simple, it
     * seems).
     *
     * Both C and Julia octal numbers (see octal token) can be active
     * simultaneously. This can be used to control leading zero
     * behavior, even if C-octal numbers are not part of the grammar
     * being parsed. For example, a language might use 0o777 octal
     * numbers and disallow 0777 integers. Enabling C-octals makes this
     * easy to detect (but should accept octal 0).
     *
     * There is no destinction between the styles in the int token, but
     * leading zeroes are easily detected in the lexeme.
     *
     * Constant suffixes like 1L are treated as 1(INT), and L(ID). The
     * same goes for other numeric values.
     *
     * Parser should check for leading zeroes and decide if it is valid,
     * a warning, or an error (it is in JSON). This also goes for float.
     *
     * Numericals, not limited to INT, may appear shorter than they are
     * due to buffer splits. Special recovery is required, but will only
     * happen just before EOS or EOB tokens (i.e. buffer split events).
     */
    LEX_TOK_INT,

    /*
     * float ::= (int ['.' dec-digits*] dec-exponent)
     *          | ([int] '.' dec-digits* [dec-exponent])
     *      dec-exponents ::= ('e' | 'E') ['+' | '-'] dec-digits*
     *      dec-digits ::= '0'..'9'
     *      int ::= dec-digits*
     *
     * Consumes a superset of C float representation without suffix.
     * Some invalid tokens such as 0.E are accepted. Valid tokens such
     * as 00.10 take precedence over octal numbers even if it is a
     * prefix, and the same is obviously true with respect to decimal
     * integers.
     *
     * JSON does not allow leading zeroes, and also not leading '.'.
     * This can easily be checked in the lexeme.
     *
     * The octal notation affecting integer leading zeroes is not
     * relevant to floats because floats take precedence over octal and
     * decimal int when containing '.', 'e' or 'E'.
     */
    LEX_TOK_FLOAT,

    /*
     * binary ::= (0b | 0B) ('0' | '1')*
     *
     * 0b100 or just 0b, parser must check that digits are present,
     * otherwise it may be interpreted as zero, just like octal zero
     * in C.
     *
     * Like 0X hex, 0B can be flagged out because Julia v0.3 does not
     * support uppercase 0B.
     */
    LEX_TOK_BINARY,

    /*
     * C-octal ::= 0 octal-digit*
     *     octal-digits ::= '0'..'7'
     *
     * Julia-octal ::= 0o octal-digits*
     *     octal-digits ::= '0'..'7'
     *
     * 0777 for C style octal numbers, or 0o777 for Julia syntax. Julia
     * v.0.3 does not allow uppercase 0O777, it would mean 0 * O777.
     *
     * When enabled, decimal floating points take precedence: 00.10 is
     * parsed as 00.10(decimal float), as per C standard.
     *
     * NOTE: It is possible for both styles to be active simultaneously.
     * This may be relevant in order to control handling of leading
     * zeroes in decimal integers.
     *
     * If C-octal numbers are flagged out, leading zeroes are mapped to
     * integers and the numerical value may change.  Julia behaves this
     * way. Nothing prevents support of both C and Julia octal numbers,
     * but leading zeroes will then be interpreted the C way - it is not
     * recommended to do this.
     */
    LEX_TOK_OCTAL,

    /*
     * hex ::= hex-int
     *     hex-digits ::= 'a'..'f'| 'A'..'f' | '0'..'9'
     *     hex-int ::= (0x | 0X) hex_digts*
     *
     * where hex_digits are customizable (e.g. all lower case), and hex
     * prefix 0x can be flagged to be lower case only (as in Julia).
     *
     * If hex floats are enabled, they take precedence:
     * 0x1.0(hex-float), if not, 0x1.0 will parse as: 0x1(hex) followed
     * by .0(decimal float).
     *
     * The lead prefix 0x may by flagged to be lower case only because
     * this is required by Julia v0.3 where 0X means 0 * X. Julia
     * accepts uppercase in the remaining hex digits (and exponent for
     * floats). This could possibly change in future versions.
     *
     * The zero length sequence (0x | 0X) is accepted and left to the
     * parser since the lexer emits a token for everything it sees.
     * Conceptually it may be interpreted as zero, equivalent to 0 being
     * both octal prefix and numeric 0 in C style octal representation.
     * Or it may be an error.
     */
    LEX_TOK_HEX,

    /*
     * hex_float ::= hex-int ['.' hex_digit*] hex-exponent
     *     hex-exponent ::= ('p' | 'P') ['+' | '-'] decimal-digit*
     *     decimal-digit ::= '0'..'9'
     *
     * A superset of IEEE-754-2008 Hexadecimal Floating Point notation.
     *
     * We require the exponent to be present, but does not ensure the
     * value is otherwise complete, e.g. 0x1p+ would be accepted. The p
     * is needed because otherwise 0x1.f could be accepted, and f is a
     * float suffix in C, and juxtapostion factor (0x1. * f) in Julia,
     * at least, that is one possible interpretation.
     *
     * The exponent can be flagged optional in which case 0x1.f will be
     * consumed as a single hex float toke as a single hex float token.
     * This may either simply be accepted in some grammars, or used to
     * provide an error message. If the exponent is required, 0x1.f will
     * be lexed as three tokens:
     *
     *     <'0x1'(hex int), '.'(op), 'f'(id)>.
     *
     * Thus it may be a good idea to allow the exponent to be optional
     * anyway and issue an error message or warning if the p is absent
     * later in the parsing stage.
     *
     * Note that, as per IEEE-754, the exponent is a decimal power of
     * two. In other words, the number of bits to shift the
     * (hexa)decimal point. Also note that it is p and not e because e
     * is a hex digit.
     */
    LEX_TOK_HEX_FLOAT,

    /*
     * blank ::= ('\t' | '\x20')+
     *
     * Longest run in buffer holding only '\t' and '\x20' (space).
     *
     * buffer splits may generate adjacent blanks depending on recovery
     * processing. (The same goes for other line oriented runs such as
     * string parts and comment parts).
     */
    LEX_TOK_BLANK,

    /* newline ::= '\r' | '\n' | '\r\n' | '\n\r'
     *
     * Will always appear, also inside strings and comments. Can be used
     * to track line starts and counts reliably as only one newline is
     * issued at a time, and it is issued everywhere, also in strings
     * and comments.
     *
     * May be preceeded by string escape token inside strings. This can
     * be interpreted as line continuation within strings specifically,
     * as is the case in Python and Javascript (and in C via
     * pre-processor).
     *
     * The LEX_TOK_STRING_NEWLINE is emitted inside strings so the ordinary
     * newline may be ignored in comments and other non-string content.
     */
    LEX_TOK_NEWLINE,
    LEX_TOK_STRING_NEWLINE,

    /*
     * string ::= string_start
     *            (string_part | string_escape |
     *                 string_ctrl | string_newline)*
     *            (string_end | string_unterminated)
     *
     * There are several optional string styles. They all start with
     * this token. The length and content provided details. Python
     * may start with """ or ''' and this token will then have length
     * 3 and three quotes as lexeme content. If the lexer exits before
     * string end token, the returned lexer mode will remember the
     * state and can be used for reentry - this also goes for comments.
     *
     * Strings can only contain part, escape, newline, and control
     * tokens, and either string unterminated or string end token
     * at last.
     */
    LEX_TOK_STRING_BEGIN,

    /* Longest run without control characters, without (\), without
     * newline, and without the relevant end delimiter. The run may be
     * shortened due to buffer splits. The part may, as an exception,
     * begin with an end delimiter character or a (\) if it was
     * preceeded by a string escape token. The escape character is
     * always (\). Strings that use "" or '' as escape will be treated
     * as start and end of separate strings. Strings that do not supoort
     * (\) should just treat escape as a part of the string.
     */
    LEX_TOK_STRING_PART,

    /*
     * This is always a single character token (\) and only happens
     * inside strings. See also string part token.
     */
    LEX_TOK_STRING_ESCAPE,

    /* This token is similar to string start. It may be absent at buffer
     * splits, but will then an unterminated string token will be used
     * just before the split event token.
     *
     * */
    LEX_TOK_STRING_END,

    /*
     * This is emitted before the buffer ends, or before unescaped
     * newlines for line oriented string types (the usual strings).
     * At buffer splits, recovery should clean it up. The returned
     * mode allow parsing to continue in a new buffer with a slight
     * content overlap.
     *
     * If string like ("hello, world!") in C, reaches end of line, it
     * may be continued" ("hello, \)newline(world!"). If this line
     * continuation is flagged out, this will lead to string
     * unterminated, even if not at end of buffer. For block strings
     * like """hello""", this only happens at end of buffer.
     */
    LEX_TOK_STRING_UNTERMINATED,

    /*
     *
     *     comment ::= comment_start
     *     (comment_part | ctrl | newline)*
     *     (comment_end | comment_unterminated)
     *
     *
     * Comments work like strings in most respects. They emit parts, and
     * control characters, but not escape characters, and cannot be
     * continued at end of line. Block comments are like python block
     * strings (''').
     *
     * Julia supports nested comments (#= ... #= =# =#). In this case
     * a new start token can be emitted before an end token. If the
     * parser exits due to buffer split, the mode has the nesting level
     * encoded so it can resumed in a new buffer.
     *
     * Line comments will have their end token just before newline, or
     * unterminated comment just before buffer split token (EOB or EOS).
     * (\) characters are consumed by the comment part tokens and do not
     * affect the end of any comment.
     *
     * Comment begin may include extra characters when a doc comment is
     * recognized. The emitter flags this. End comments are unaffected.
     */
    LEX_TOK_COMMENT_BEGIN,
    LEX_TOK_COMMENT_PART,
    LEX_TOK_COMMENT_END,
    LEX_TOK_COMMENT_UNTERMINATED,

    /*
     * Issued before ABORT token if nesting level is above a predefined
     * level. This is to protect against malicious and misguided
     * content, otherwise the nesting level counter could wrap and
     * generate a different interpretation, which could be bad. The
     * parser would probably do similar things with nested tokens.
     */
    LEX_TOK_COMMENT_DEEPLY_NESTED,


    /* Operators are all recognized single character symbols, or up to
     * four characters. The token value is the ASCII codes shifted 8
     * bits per extra character, by default, but the emitter macros
     * can redefine this. Values below 32 are reserved token types as
     * discussed above.
     *
     * What exactly represents an operator depends on what the lexer has
     * enabled.
     *
     * Printable ASCII symbols that are NOT recognized, are emitted as
     * the SYMBOL token and is always length 1. The value can be derived
     * from the lexeme, but not the token itself. This may be perfectly
     * fine for the parser, or it may be used to indicate an error.
     * There are no illegal characters per se.
     *
     * Non-printable ASCII characters that are not covered by newline or
     * blank, are emitted as CTRL tokens. These act the same as the
     * symbol token and may be used to indicate error, or to handle form
     * feed and other whitespace not handled by default. Unlike symbol,
     * however, CTRL also appear in strings and comments since they are
     * generally not allowed and this makes it easy to capture (there is
     * virtually no performance overhead in providing this service
     * unless attempting to parse a binary format).
     */

    /* Don't bleed into this range. */
    LEX_TOK_OPERATOR_BASE = 32,


    /*
     * Operators use ASCII range.
     * Compound operators use range 0x80 to 0x7fff
     * and possibly above for triple sequences.
     * Custom keywords are normally negative but can be mapped
     * to any other.
     *
     * The layout is designed for efficient table lookup.
     * Compound operators might benefit from remapping down to a smaller
     * range for compact lookup tables, but it depends on the parser.
     */
};

/*
 * Custom keyword token range is negative, and well below -99..0 where
 * special codes are reserved.
 */
#ifndef LEX_TOK_KW_BASE
#define LEX_TOK_KW_BASE -1000
#endif

#ifndef LEX_TOK_KW_NOT_FOUND
#define LEX_TOK_KW_NOT_FOUND LEX_TOK_ID
#endif


#ifdef LEX_DEBUG

#include <stdio.h>
#include <string.h>

static const char *lex_describe_token(long token)
{
    switch(token) {
    case LEX_TOK_BOM: return "BOM marker";
    case LEX_TOK_EOF: return "EOF";
    case LEX_TOK_EOS: return "buffer zero terminated";
    case LEX_TOK_EOB: return "buffer exhausted";
    case LEX_TOK_ABORT: return "abort";
    case LEX_TOK_CTRL: return "control";
    case LEX_TOK_STRING_CTRL: return "string control";
    case LEX_TOK_COMMENT_CTRL: return "comment control";
    case LEX_TOK_SYMBOL: return "symbol";
    case LEX_TOK_ID: return "identifier";
    case LEX_TOK_INT: return "integer";
    case LEX_TOK_FLOAT: return "float";
    case LEX_TOK_BINARY: return "binary";
    case LEX_TOK_OCTAL: return "octal";
    case LEX_TOK_HEX: return "hex";
    case LEX_TOK_HEX_FLOAT: return "hex float";
    case LEX_TOK_BLANK: return "blank";
    case LEX_TOK_NEWLINE: return "newline";
    case LEX_TOK_STRING_NEWLINE: return "string newline";
    case LEX_TOK_STRING_BEGIN: return "string begin";
    case LEX_TOK_STRING_PART: return "string part";
    case LEX_TOK_STRING_END: return "string end";
    case LEX_TOK_STRING_ESCAPE: return "string escape";
    case LEX_TOK_STRING_UNTERMINATED: return "unterminated string";
    case LEX_TOK_COMMENT_BEGIN: return "comment begin";
    case LEX_TOK_COMMENT_PART: return "comment part";
    case LEX_TOK_COMMENT_END: return "comment end";
    case LEX_TOK_COMMENT_UNTERMINATED: return "unterminated comment";
    case LEX_TOK_COMMENT_DEEPLY_NESTED: return "deeply nested comment";

    default:
        if (token < LEX_TOK_EOF) {
            return "keyword";
        }
        if (token < 32) {
            return "undefined";
        }
        if (token < 0x100L) {
            return "operator";
        }
        if (token < 0x10000L) {
            return "compound operator";
        }
        if (token < 0x1000000L) {
            return "tricompound operator";
        }
        if (token < 0x7f0000000L) {
            return "quadcompound operator";
        }
        return "reserved";
    }
}

static void lex_fprint_token(FILE *fp,
        long token,
        const char *first, const char *last,
        int line, int pos)
{
    char buf[10];
    const char *lexeme = first;
    int len = (int)(last - first);
    switch (token) {
    case LEX_TOK_EOS:
    case LEX_TOK_CTRL:
        sprintf(buf, "^%02x", (int)*first);
        lexeme = buf;
        len = strlen(buf);
        break;
    default:
        break;
    }
    fprintf(fp, "%04d:%03d %s (0x%lx): `%.*s`\n",
            line, pos, lex_describe_token(token), token, len, lexeme);
}

#define lex_print_token(token, first, last, line, pos)                  \
        lex_fprint_token(stdout, token, first, last, line, pos)

#else /* LEX_DEBUG */

#define lex_describe_token(token) "debug not available"
#define lex_fprint_token(fp, token, first, last, line, pos) ((void)0)
#define lex_print_token(token, first, last, line, pos) ((void)0)

#endif /* LEX_DEBUG */


#endif /* LEX_TOKENS_H */