path: root/compiler/tokenizer.h

/*

  tokenizer.h -- Tokenizer/lexer (breaks the source in tokens).

  Copyright © 2011-2016 Samuel Lidén Borell <samuel@kodafritt.se>

  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
  in the Software without restriction, including without limitation the rights
  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  copies of the Software, and to permit persons to whom the Software is
  furnished to do so, subject to the following conditions:

  The above copyright notice and this permission notice shall be included in
  all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  THE SOFTWARE.

*/

#ifndef LRL_TOKENIZER_H
#define LRL_TOKENIZER_H

#include "context.h"

#define lrl_is_paren(type) ((type) >= LRL_Sym_LParen && (type) <= LRL_Sym_RCurly)
#define lrl_is_start_paren(type) ((type) & 1)
#define lrl_paren_start_type(type) ((type)-1)

#define LRL_FirstQual LRL_KW_Const
#define LRL_LastQual LRL_KW_Var
#define LRL_NumQualifiers (LRL_LastQual-LRL_FirstQual+1)

#define LRL_FirstOp LRL_Op_Plus
#define LRL_LastOp LRL_Op_ShiftRAssign
#define LRL_NumOps (LRL_LastOp-LRL_FirstOp+1)

typedef enum {
    LRL_TT_EOF = 0,         /* end of file */
    LRL_TT_Error = -1,      /* invalid token */
    LRL_TT_Incomplete = -2, /* unexpected EOF */
    
    /* Symbols */
    LRL_Sym_LParen = 1,
    LRL_Sym_RParen,
    LRL_Sym_LSquare,
    LRL_Sym_RSquare,
    LRL_Sym_LCurly,
    LRL_Sym_RCurly,
    LRL_Sym_Semicolon,
    LRL_Sym_Comma,
    LRL_Sym_NamespaceSep, /*  :  */
    LRL_Sym_ArrayIndex,
    LRL_Sym_FlexiPointer, /*  +> which allows arithmetic */
    LRL_Sym_RawPointer,   /*  *> which can be null but also be assigned to ^ */
    LRL_Sym_RawFlexiPointer, /*  +*>  */
    LRL_Sym_CVarArg,      /*  ...* which is not checked for incorrect usage  */
    /* Operators - Arithmetic */
    LRL_Op_Plus,    /*  +    */
    LRL_Op_Minus,   /*  -    */
    LRL_Op_Times,   /*  *    */
    LRL_Op_Divide,  /*  /    */
    LRL_Op_Modulo,  /*  mod  */
    /* Operators - Bitwise */
    LRL_Op_ShiftL,  /*  <<   */
    LRL_Op_ShiftR,  /*  >>   */
    LRL_Op_Compl,   /*  compl */
    LRL_Op_BitAnd,  /*  bitand */
    LRL_Op_BitOr,   /*  bitor */
    LRL_Op_BitXor,  /*  bitxor */
    /* Operators - Special */
    LRL_Op_Member,  /*  .    */
    LRL_Op_FunctionMember, /*  ->  */
    LRL_Op_Deref,   /*  ^   (or ' maybe?)   */
    LRL_Op_AddrOf,  /*  @    */
    LRL_Op_OptionalValue, /*  ?  */
    LRL_Op_MakeOpt, /* makeopt x */
    LRL_Op_EnumBase,/* enumbase x */
    LRL_Op_SizeOf,  /* sizeof(x) */
    LRL_Op_MinSizeOf,/* minsizeof(x) */
    LRL_Op_OffsetOf,/* offsetof(x) */
    LRL_Op_AlignOf, /* alignof(x) */
    LRL_Op_Then,    /*   (a then x else y)  */
    /* Operators - Boolean */
    LRL_Op_LNot,  /*  not  */
    LRL_Op_LAnd,  /*  and  */
    LRL_Op_LOr,   /*  or   */
    LRL_Op_LXor,  /*  xor  */
    /* Operators - Comparison */
    LRL_Op_Equal,
    LRL_Op_NotEqual,
    LRL_Op_Less,
    LRL_Op_LessEqual,
    LRL_Op_Greater,
    LRL_Op_GreaterEqual,
    /* Operators - Assignment */
    LRL_Op_Assign,       /*  =  */
    LRL_Op_PlusAssign,   /*  +=  */
    LRL_Op_MinusAssign,  /*  -=  */
    LRL_Op_TimesAssign,  /*  *=  */
    LRL_Op_DivideAssign, /*  /=  */
    LRL_Op_ShiftLAssign, /*  <<=  */
    LRL_Op_ShiftRAssign, /*  >>=  */
    /* Keywords - Types */
    LRL_KW_Typedef,
    LRL_KW_Struct, /* reserved */
    LRL_KW_Union,
    LRL_KW_Enum,
    LRL_KW_Bits,
    LRL_KW_NoReturn,
    LRL_KW_Private,
    LRL_KW_Incomplete,
    LRL_KW_Any,
    /* Keywords - Type qualifiers */
    LRL_KW_Const,
    LRL_KW_Gc,
    LRL_KW_Mine,
    LRL_KW_Own,
    LRL_KW_Shared,
    LRL_KW_Var,
    /* Keywords - Type constraints (not implemented. reserved words) */
    LRL_KW_Cond,
    LRL_KW_PreCond,
    LRL_KW_PostCond,
    /* Keywords - Definition qualifiers */
    LRL_KW_Alias,
    LRL_KW_AlignAs,
    LRL_KW_Deprecated,
    LRL_KW_Import,
    LRL_KW_Export,
    LRL_KW_Linkname,
    LRL_KW_Local,
    LRL_KW_DeclOnly,
    /* Keywords - Control statements */
    LRL_KW_If,
    LRL_KW_Else,
    LRL_KW_Elif, /* reserved */
    LRL_KW_While,
    LRL_KW_Do,
    LRL_KW_For,
    LRL_KW_LoopEnd,
    LRL_KW_LoopEmpty,
    LRL_KW_Switch,
    LRL_KW_Case,
    LRL_KW_Default,
    LRL_KW_Goto,
    LRL_KW_SkipTo,
    LRL_KW_RepeatFrom,
    LRL_KW_Label,
    LRL_KW_Continue,
    LRL_KW_Break,
    LRL_KW_Flow,
    LRL_KW_Leave,
    LRL_KW_Return,
    LRL_KW_Unreachable,
    LRL_KW_Uses,
    LRL_KW_As,
    LRL_KW_TypeAssert,
    LRL_KW_Assert,
    /* Keywords - Namespace, interop */
    LRL_KW_Namespace,
    LRL_KW_Interop,
    /* Reserved keywords (not in use yet) */
    LRL_KW_At,
    LRL_KW_In,
    LRL_KW_Is,
    LRL_KW_Of,
    LRL_KW_Unused,
    LRL_KW_Using,
    LRL_KW_With, /* used in switch-case statements */
    LRL_KW_Yield,
    /* Values: Identifiers, numbers, strings... */
    LRL_TT_Ident,
    LRL_TT_Integer,
    LRL_TT_Real,
    LRL_TT_String,
    LRL_TT_Undefined,
    LRL_TT_None,
    LRL_TT_NaN, /* not a number */
    LRL_TT_Inf, /* infinity */
    LRL_KW_Here,
    
    /* Character classes - Not used in the token stream */
    LRL_Char_Whitespace,
    
    LRL_TT_LastType
} LRLTokenType;

/* Case label defines to catch a group of node types */
#define LRL_case_tt_symbols_except_ops \
    case LRL_Sym_LParen: \
    case LRL_Sym_RParen: \
    case LRL_Sym_LSquare: \
    case LRL_Sym_RSquare: \
    case LRL_Sym_LCurly: \
    case LRL_Sym_RCurly: \
    case LRL_Sym_Semicolon: \
    case LRL_Sym_Comma: \
    case LRL_Sym_NamespaceSep: \
    case LRL_Sym_FlexiPointer: \
    case LRL_Sym_RawPointer: \
    case LRL_Sym_RawFlexiPointer: \
    case LRL_Sym_CVarArg:
#define LRL_case_tt_symbols \
    LRL_case_tt_symbols_except_ops \
    case LRL_Sym_ArrayIndex:
#define LRL_case_tt_unops_except_types \
    case LRL_Op_AddrOf: \
    case LRL_Op_EnumBase: \
    case LRL_Op_MakeOpt: \
    case LRL_Op_Compl: \
    case LRL_Op_LNot: \
    case LRL_Op_SizeOf: \
    case LRL_Op_MinSizeOf: \
    case LRL_Op_OffsetOf: \
    case LRL_Op_AlignOf:
#define LRL_case_tt_unops \
    LRL_case_tt_unops_except_types \
    case LRL_Op_Deref: \
    case LRL_Op_OptionalValue:
#define LRL_case_tt_binops_except_equality \
    case LRL_Op_Times: \
    case LRL_Op_Divide: \
    case LRL_Op_Modulo: \
    case LRL_Op_ShiftL: \
    case LRL_Op_ShiftR: \
    case LRL_Op_BitAnd: \
    case LRL_Op_BitXor: \
    case LRL_Op_BitOr: \
    case LRL_Op_LAnd: \
    case LRL_Op_LOr: \
    case LRL_Op_LXor: \
    case LRL_Op_Assign: \
    case LRL_Op_PlusAssign: \
    case LRL_Op_MinusAssign: \
    case LRL_Op_TimesAssign: \
    case LRL_Op_DivideAssign: \
    case LRL_Op_ShiftLAssign: \
    case LRL_Op_ShiftRAssign:
#define LRL_case_tt_equality_binops \
    case LRL_Op_Equal: \
    case LRL_Op_NotEqual: \
    case LRL_Op_Less: \
    case LRL_Op_LessEqual: \
    case LRL_Op_Greater: \
    case LRL_Op_GreaterEqual:
#define LRL_case_tt_binops \
    LRL_case_tt_binops_except_equality \
    LRL_case_tt_equality_binops
#define LRL_case_tt_unops_binops \
    case LRL_Op_Plus: \
    case LRL_Op_Minus:
#define LRL_case_tt_otherops_except_syms_controls \
    case LRL_Op_FunctionMember: \
    case LRL_Op_Member: \
    case LRL_Op_Then:
#define LRL_case_tt_otherops_except_syms \
    LRL_case_tt_otherops_except_syms_controls \
    case LRL_KW_Else: \
    case LRL_KW_As: \
    case LRL_KW_TypeAssert:
#define LRL_case_tt_otherops \
    LRL_case_tt_otherops_except_syms \
    case LRL_Sym_ArrayIndex:
#define LRL_case_tt_ops_except_syms_types_controls \
    LRL_case_tt_unops_except_types \
    LRL_case_tt_binops \
    LRL_case_tt_unops_binops \
    LRL_case_tt_otherops_except_syms_controls
#define LRL_case_tt_ops_except_syms \
    LRL_case_tt_unops \
    LRL_case_tt_binops \
    LRL_case_tt_unops_binops \
    LRL_case_tt_otherops_except_syms
#define LRL_case_tt_ops \
    LRL_case_tt_unops \
    LRL_case_tt_binops \
    LRL_case_tt_unops_binops \
    LRL_case_tt_otherops
#define LRL_case_tt_types \
    case LRL_KW_Typedef: \
    case LRL_KW_Union: \
    case LRL_KW_Enum: \
    case LRL_KW_Bits: \
    case LRL_KW_NoReturn: \
    case LRL_KW_Private: \
    case LRL_KW_Incomplete: \
    case LRL_KW_Any:
#define LRL_case_tt_typequals \
    case LRL_KW_Const: \
    case LRL_KW_Gc: \
    case LRL_KW_Mine: \
    case LRL_KW_Own: \
    case LRL_KW_Shared: \
    case LRL_KW_Var:
#define LRL_case_tt_defquals \
    case LRL_KW_Alias: \
    case LRL_KW_AlignAs: \
    case LRL_KW_Deprecated: \
    case LRL_KW_Import: \
    case LRL_KW_Export: \
    case LRL_KW_Linkname: \
    case LRL_KW_Local: \
    case LRL_KW_DeclOnly:
#define LRL_case_tt_control_except_ops \
    case LRL_KW_If: \
    case LRL_KW_While: \
    case LRL_KW_Do: \
    case LRL_KW_For: \
    case LRL_KW_LoopEnd: \
    case LRL_KW_LoopEmpty: \
    case LRL_KW_Switch: \
    case LRL_KW_Case: \
    case LRL_KW_Default: \
    case LRL_KW_Goto: \
    case LRL_KW_SkipTo: \
    case LRL_KW_RepeatFrom: \
    case LRL_KW_Label: \
    case LRL_KW_Continue: \
    case LRL_KW_Break: \
    case LRL_KW_Flow: \
    case LRL_KW_Leave: \
    case LRL_KW_Return: \
    case LRL_KW_Unreachable: \
    case LRL_KW_Assert: \
    case LRL_KW_Uses:
#define LRL_case_tt_control \
    LRL_case_tt_control_except_ops \
    case LRL_KW_Else: \
    case LRL_KW_As: \
    case LRL_KW_TypeAssert:
#define LRL_case_tt_namespace \
    case LRL_KW_Namespace: \
    case LRL_KW_Interop:
#define LRL_case_tt_values \
    case LRL_TT_Ident: \
    case LRL_TT_Integer: \
    case LRL_TT_Real: \
    case LRL_TT_String: \
    case LRL_TT_Undefined: \
    case LRL_TT_None: \
    case LRL_TT_NaN: \
    case LRL_TT_Inf: \
    case LRL_KW_Here:
#define LRL_case_tt_specials \
    case LRL_TT_EOF: \
    case LRL_TT_Error: \
    case LRL_TT_Incomplete: \
    case LRL_KW_Struct: \
    case LRL_KW_Cond: \
    case LRL_KW_PreCond: \
    case LRL_KW_PostCond: \
    case LRL_KW_Elif: \
    case LRL_KW_At: \
    case LRL_KW_In: \
    case LRL_KW_Is: \
    case LRL_KW_Of: \
    case LRL_KW_Unused: \
    case LRL_KW_Using: \
    case LRL_KW_With: \
    case LRL_KW_Yield: \
    case LRL_Char_Whitespace: \
    case LRL_TT_LastType:

#define LRL_case_except_tt_values \
    LRL_case_tt_symbols \
    LRL_case_tt_ops_except_syms \
    LRL_case_tt_types \
    LRL_case_tt_typequals \
    LRL_case_tt_defquals \
    LRL_case_tt_control_except_ops \
    LRL_case_tt_namespace \
    LRL_case_tt_specials
#define LRL_case_except_tt_ops \
    LRL_case_tt_symbols_except_ops \
    LRL_case_tt_types \
    LRL_case_tt_typequals \
    LRL_case_tt_defquals \
    LRL_case_tt_control_except_ops \
    LRL_case_tt_namespace \
    LRL_case_tt_values \
    LRL_case_tt_specials
#define LRL_case_except_tt_unops \
    LRL_case_except_tt_ops \
    LRL_case_tt_binops \
    LRL_case_tt_otherops
#define LRL_case_except_tt_binops \
    LRL_case_except_tt_ops \
    LRL_case_tt_unops \
    LRL_case_tt_otherops
#define LRL_case_except_tt_equality_binops \
    LRL_case_except_tt_binops \
    LRL_case_tt_unops_binops \
    LRL_case_tt_binops_except_equality

/* typedefed in context.h */
struct LRLToken {
    LRLTokenType type;         /* type of token */
    LRLCodeLocation loc;       /* location in the source code */
};


size_t lrl_tokenize(LRLCtx *ctx, LRLToken **tokenList, const char *source);


#endif