/* Unit tests of token.c Copyright © 2021-2024 Samuel Lidén Borell Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include "../token.c" #define SLULPARSE #include "parsecommon.h" #include "unittest.h" #include "alltests.h" #include "testcommon.h" #include /** Runs before each test case */ static void before_test(void) { set_mhparse(0); /* Set mode. */ } /** Runs after each test case */ static void after_test(void) { verify_errors(); } static void test_tok_empty(void) { testcommon_empty(); } static void test_tok_newline_single(void) { testcommon_newline_single(); } static void test_tok_newline_splitted(void) { testcommon_newline_splitted(); } static void test_tok_newline_eof1(void) { testcommon_newline_eof1(); } static void test_tok_newline_eof2(void) { testcommon_newline_eof2(); } static void test_tok_null(void) { testcommon_null(); } static void test_tok_utf8_2min(void) { testcommon_utf8_2min(); } static void test_tok_utf8_2max(void) { testcommon_utf8_2max(); } static void test_tok_utf8_2split(void) { testcommon_utf8_2split(); } static void test_tok_utf8_2overlong(void) { testcommon_utf8_2overlong(); } static void test_tok_utf8_2eof(void) { testcommon_utf8_2eof(); } static void test_tok_utf8_2badcont(void) { testcommon_utf8_2badcont(); } static void test_tok_utf8_3min(void) { testcommon_utf8_3min(); } static void test_tok_utf8_3max(void) { testcommon_utf8_3max(); } static void test_tok_utf8_3split(void) { testcommon_utf8_3split(); } static void test_tok_utf8_3overlong(void) { testcommon_utf8_3overlong(); } static void test_tok_utf8_3eof2(void) { testcommon_utf8_3eof2(); } static void test_tok_utf8_3nullchar3(void) { testcommon_utf8_3nullchar3(); } static void test_tok_utf8_4min(void) { testcommon_utf8_4min(); } static void test_tok_utf8_4max(void) { testcommon_utf8_4max(); } static void test_tok_utf8_4split(void) { testcommon_utf8_4split(); } static void test_tok_utf8_4overlong(void) { testcommon_utf8_4overlong(); } static void test_tok_utf8_4overflow(void) { testcommon_utf8_4overflow(); } static void test_tok_utf8_4eof2(void) { testcommon_utf8_4eof2(); } static void test_tok_utf8_4eof3(void) { testcommon_utf8_4eof3(); } static void test_tok_utf8_4nullchar3(void) { testcommon_utf8_4nullchar3(); } static void test_tok_utf8_4nullchar4(void) { testcommon_utf8_4nullchar4(); } static void test_tok_utf8_forbidden(void) { testcommon_utf8_forbidden(); } static void test_tok_comment(void) { testcommon_comment(); } static void test_tok_comment_badchar(void) { testcommon_comment_badchar(); } static void test_tok_multiline_comment(void) { testcommon_multiline_comment(); } static void test_tok_multiline_comment_badstart(void) { testcommon_multiline_comment_badstart(); } static void test_tok_keyword_lowerident(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("func abc\n"); TEST_ONE(buff, 9, 1, CSLUL_T_KW_Func, 1, 5, 0, TDone); ASSERT_TOKEN_VALUE("func", 1, 1); TEST_CONT(CSLUL_T_LowerIdent, 1, 9, 0, TDone); ASSERT_TOKEN_VALUE("abc", 1, 6); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_keyword_upperident(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("type Abc\n"); TEST_ONE(buff, 9, 1, CSLUL_T_KW_Type, 1, 5, 0, TDone); ASSERT_TOKEN_VALUE("type", 1, 1); TEST_CONT(CSLUL_T_UpperIdent, 1, 9, 0, TDone); ASSERT_TOKEN_VALUE("Abc", 1, 6); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_keyword_splitted(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("fun"); TEST_ONE(buff, 3, 0, CSLUL_T_NEEDDATA, 1, 4, 0, TInIdent); buff[0] = 'c'; buff[1] = ' '; TEST_ONE(buff, 2, 0, CSLUL_T_KW_Func, 1, 5, 0, TDone); ASSERT_TOKEN_VALUE("func", 1, 1); tfree(buff); free_ctx(ctx); } static void test_tok_gototarget(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("target:following\n"); TEST_ONE(buff, 17, 1, CSLUL_T_GotoTarget, 1, 8, 0, TDone); ASSERT_TOKEN_VALUE("target", 1, 1); TEST_CONT(CSLUL_T_LowerIdent, 1, 17, 0, TDone); ASSERT_TOKEN_VALUE("following", 1, 8); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_version(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("1.23~abc1)\n"); /* ('_' is actually forbidden) */ ctx->parser.slul.version_line = 1; TEST_ONE(buff, 11, 1, CSLUL_T_Version, 1, 10, 0, TDone); ASSERT_TOKEN_VALUE("1.23~abc1", 1, 1); tsoftassert(!ctx->parser.slul.version_line); TEST_CONT(CSLUL_T_RParen, 1, 11, 0, TDone); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_since_version(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("since 1.2\n"); TEST_ONE(buff, 10, 1, CSLUL_T_KW_Since, 1, 6, 0, TDone); tsoftassert(ctx->parser.slul.version_line); TEST_CONT(CSLUL_T_Version, 1, 10, 0, TDone); ASSERT_TOKEN_VALUE("1.2", 1, 7); tsoftassert(!ctx->parser.slul.version_line); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_identtoolong(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char buff[102]; memset(buff, 'x', 100); buff[0] = 'a'; buff[99] = 'b'; buff[100] = 'c'; buff[101] = '\n'; expect_error(CSLUL_E_IDENTTOOLONG, 1, 101); TEST_ONE(buff, 102, 1, CSLUL_T_LowerIdent, 1, 102, 0, TDone); tsoftassert(ctx->toklen == 100); tsoftassert(!strncmp(ctx->tokval, buff, 100)); verify_errors(); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); free_ctx(ctx); } /** * Tests an identifier that is splitted across buffers BEFORE the * maximum length of 100 bytes is reached */ static void test_tok_identtoolong_splitted1(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char buff[90]; memset(buff, 'x', 87); buff[0] = 'a'; buff[87] = 'b'; buff[88] = 'c'; buff[89] = 'd'; ctx->toktmp[100] = 'z'; TEST_ONE(buff, 90, 0, CSLUL_T_NEEDDATA, 1, 91, 0, TInIdent); expect_error(CSLUL_E_IDENTTOOLONG, 1, 101); TEST_ONE(buff, 90, 0, CSLUL_T_NEEDDATA, 1, 181, 0, TInIdent); verify_errors(); buff[89] = '\n'; TEST_ONE(buff, 90, 1, CSLUL_T_LowerIdent, 1, 270, 0, TDone); tsoftassert(ctx->tokval != buff); tsoftassert(ctx->toktmp[100] == 'z'); tsoftassert(ctx->toklen == 100); buff[89] = 'd'; tsoftassert(!strncmp(ctx->tokval, buff, 90)); tsoftassert(ctx->toktmp[90] == 'a'); tsoftassert(ctx->toktmp[99] == 'x'); buff[89] = '\n'; tsoftassert(ctx->buffer == &buff[89]); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); free_ctx(ctx); } /** * Tests an identifier that is splitted across buffers AFTER the * maximum length of 100 bytes is reached */ static void test_tok_identtoolong_splitted2(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char buff[300]; memset(buff, 'x', 299); buff[0] = 'a'; buff[98] = 'b'; buff[99] = 'c'; buff[100] = 'd'; buff[299] = 'e'; ctx->toktmp[100] = 'z'; expect_error(CSLUL_E_IDENTTOOLONG, 1, 101); TEST_ONE(buff, 300, 0, CSLUL_T_NEEDDATA, 1, 301, 0, TInIdent); verify_errors(); tsoftassert(ctx->tokval == buff); tassert(ctx->toklen == 100); buff[99] = 'y'; TEST_ONE(buff, 300, 0, CSLUL_T_NEEDDATA, 1, 601, 0, TInIdent); buff[299] = '\n'; TEST_ONE(buff, 300, 1, CSLUL_T_LowerIdent, 1, 900, 0, TDone); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); free_ctx(ctx); } static void test_tok_operator_1char(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("()[]{}+-*/<>,.?:;=\n"); TEST_ONE(buff, 19, 1, CSLUL_T_LParen, 1, 2, 0, TDone); TEST_CONT(CSLUL_T_RParen, 1, 3, 0, TDone); TEST_CONT(CSLUL_T_LSquare, 1, 4, 0, TDone); TEST_CONT(CSLUL_T_RSquare, 1, 5, 0, TDone); TEST_CONT(CSLUL_T_LCurly, 1, 6, 0, TDone); TEST_CONT(CSLUL_T_RCurly, 1, 7, 0, TDone); TEST_CONT(CSLUL_T_Plus, 1, 8, 0, TDone); TEST_CONT(CSLUL_T_Minus, 1, 9, 0, TDone); TEST_CONT(CSLUL_T_Asterisk, 1, 10, 0, TDone); TEST_CONT(CSLUL_T_Slash, 1, 11, 0, TDone); TEST_CONT(CSLUL_T_Less, 1, 12, 0, TDone); TEST_CONT(CSLUL_T_Greater, 1, 13, 0, TDone); TEST_CONT(CSLUL_T_Comma, 1, 14, 0, TDone); TEST_CONT(CSLUL_T_Dot, 1, 15, 0, TDone); TEST_CONT(CSLUL_T_Question, 1, 16, 0, TDone); TEST_CONT(CSLUL_T_Colon, 1, 17, 0, TDone); TEST_CONT(CSLUL_T_Semicolon,1, 18, 0, TDone); TEST_CONT(CSLUL_T_Assign, 1, 19, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_operator_2char(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("+=-=*=/=<===>=!=->\n"); TEST_ONE(buff, 19, 1, CSLUL_T_PlusAssign, 1, 3, 0, TDone); TEST_CONT(CSLUL_T_MinusAssign, 1, 5, 0, TDone); TEST_CONT(CSLUL_T_MultiplyAssign, 1, 7, 0, TDone); TEST_CONT(CSLUL_T_DivideAssign, 1, 9, 0, TDone); TEST_CONT(CSLUL_T_LessEqual, 1, 11, 0, TDone); TEST_CONT(CSLUL_T_Equal, 1, 13, 0, TDone); TEST_CONT(CSLUL_T_GreaterEqual, 1, 15, 0, TDone); TEST_CONT(CSLUL_T_NotEqual, 1, 17, 0, TDone); TEST_CONT(CSLUL_T_RArrow, 1, 19, 0, TDone); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_operator_2char_splitted(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); /* test string: +=== buffer #: 1223 token 1: += token 2: == */ char *buff1 = cut0("+"); char *buff2 = cut0("=="); TEST_ONE(buff1, 1, 0, NEEDDATA, 1, 2, 0, TInOperator); /* + */ tsoftassert(ctx->parser.slul.two_char_first == CSLUL_T_Plus); TEST_ONE(buff2, 2, 0, CSLUL_T_PlusAssign, 1, 3, 0, TDone); /* == */ TEST_CONT(NEEDDATA, 1, 4, 0, TInOperator); tsoftassert(ctx->parser.slul.two_char_first == CSLUL_T_Assign); buff1[0] = '='; TEST_ONE(buff1, 1, 1, CSLUL_T_Equal, 1, 5, 0, TDone); /* = */ expect_error(CSLUL_E_NOEOFNEWLINE, 1, 5); TEST_CONT(CSLUL_T_EOF, 1, 5, 0, TDone); tfree(buff2); tfree(buff1); free_ctx(ctx); } static void test_tok_operator_2char_eof(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff1 = cut0("==="); TEST_ONE(buff1, 3, 0, CSLUL_T_Equal, 1, 3, 0, TDone); /* == */ TEST_CONT(NEEDDATA, 1, 4, 0, TInOperator); tsoftassert(ctx->parser.slul.two_char_first == CSLUL_T_Assign); /* = */ expect_error(CSLUL_E_NOEOFNEWLINE, 1, 4); /* reported early */ TEST_ONE(&buff1[3], 0, 1, CSLUL_T_Assign, 1, 4, 0, TDone); verify_errors(); TEST_CONT(CSLUL_T_EOF, 1, 4, 0, TDone); tfree(buff1); free_ctx(ctx); } static void test_tok_string_empty(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\"\n"); TEST_ONE(buff, 3, 1, CSLUL_T_String, 1, 3, 0, TDone); ASSERT_RAW_VALUE("", 1, 1); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_simple(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"abc\"\n"); TEST_ONE(buff, 6, 1, CSLUL_T_String, 1, 6, 0, TDone); ASSERT_RAW_VALUE("abc", 1, 1); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_controlchar(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "\"\0\x1F\x7F\"\n"; const char *value; size_t length; TEST_ERROR3(buff, sizeof(buff)-1, 1, CSLUL_T_String, 1, 6, 0, TDone, CSLUL_E_INVALIDCHAR, 1, 2, CSLUL_E_INVALIDCHAR, 1, 3, CSLUL_E_INVALIDCHAR, 1, 4); cslul_ll_current_value(ctx, &value, &length); tassert(length == 3); tsoftassert(!memcmp(value, buff+1, 3)); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); free_ctx(ctx); } static void test_tok_string_utf8(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); /* "åäö" */ char *buff = cut0("\"\xc3\xa5\xc3\xa4\xc3\xb6\"\n"); TEST_ONE(buff, 9, 1, CSLUL_T_String, 1, 6, 3, TDone); ASSERT_RAW_VALUE("\xc3\xa5\xc3\xa4\xc3\xb6", 1, 1); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_maxsize(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = tmalloc(514); const char *value; size_t length; buff[0] = '"'; buff[513] = '"'; memset(buff+1, 'x', 512); TEST_ONE(buff, 514, 0, CSLUL_T_String, 1, 515, 0, TDone); cslul_ll_current_value(ctx, &value, &length); tassert(length == 512); tsoftassert(!memcmp(value, buff+1, 512)); TEST_CONT(NEEDDATA, 1, 515, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_toolong(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = tmalloc(515); const char *value; size_t length; buff[0] = '"'; buff[514] = '"'; memset(buff+1, 'x', 513); TEST_ERROR(buff, 515, 0, CSLUL_T_String, 1, 516, 0, TDone, CSLUL_E_STRINGTOOLONG, 1, 515); cslul_ll_current_value(ctx, &value, &length); tassert(length == 512); tsoftassert(!memcmp(value, buff+1, 512)); TEST_CONT(NEEDDATA, 1, 516, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_toolong2(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = tmalloc(800); const char *value; size_t length; buff[0] = '"'; buff[799] = '"'; memset(buff+1, 'x', 798); TEST_ERROR(buff, 800, 0, CSLUL_T_String, 1, 801, 0, TDone, CSLUL_E_STRINGTOOLONG, 1, 800); cslul_ll_current_value(ctx, &value, &length); tassert(length == 512); tsoftassert(!memcmp(value, buff+1, 512)); TEST_CONT(NEEDDATA, 1, 801, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_unterminated1(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"abc"); TEST_ERROR2(buff, 4, 1, CSLUL_T_EOF, 1, 5, 0, TInString, CSLUL_E_UNEXPECTEDEOF, 1, 5, CSLUL_E_NOEOFNEWLINE, 1, 5); tfree(buff); free_ctx(ctx); } static void test_tok_string_unterminated2(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"abc\n"); TEST_ERROR(buff, 5, 1, CSLUL_T_String, 1, 5, 0, TDone, CSLUL_E_UNTERMINATEDSTRING, 1, 5); tfree(buff); free_ctx(ctx); } static void test_tok_string_unterminated3(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"ab\\x1f\n"); TEST_ERROR(buff, 8, 1, CSLUL_T_String, 1, 8, 0, TDone, CSLUL_E_UNTERMINATEDSTRING, 1, 8); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_single(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\\0\\b\\t\\n\\r\\\"\\\\\"\n"); const char *value; size_t length; TEST_ONE(buff, 17, 1, CSLUL_T_String, 1, 17, 0, TDone); cslul_ll_current_value(ctx, &value, &length); tassert(length == 7); tsoftassert(!memcmp(value, "\0\b\t\n\r\"\\", 7)); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_hexbyte(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\\x0\\x9\\xa\\xA\\xf\\xF\\x01\\x90\\xA5\\xFF\\xff\"\n"); const char *value; size_t length; TEST_ONE(buff, 41, 1, CSLUL_T_String, 1, 41, 0, TDone); cslul_ll_current_value(ctx, &value, &length); tassert(length == 11); tsoftassert(!memcmp(value, "\0\x9\xa\xa\xf\xf\x1\x90\xa5\xff\xff", 11)); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_unicode(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\\u7FF\\u800\\uFFFF\\u10000\\u10FFFF\"\n"); TEST_ONE(buff, 34, 1, CSLUL_T_String, 1, 34, 0, TDone); ASSERT_RAW_VALUE("\xDF\xBF" "\xE0\xA0\x80" /* 7FF, 800 */ "\xEF\xBF\xBF" "\xF0\x90\x80\x80" /* FFFF, 10000 */ "\xF4\x8F\xBF\xBF" /* 10FFFF */, 1, 1); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_toolong(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\\x123\\xABC\\u0025CFA\"\n"); TEST_ERROR3(buff, 22, 1, CSLUL_T_String, 1, 22, 0, TDone, CSLUL_E_ESCAPETOOLONG, 1, 6, CSLUL_E_ESCAPETOOLONG, 1, 11, CSLUL_E_ESCAPETOOLONG, 1, 20); ASSERT_RAW_VALUE("\x12" "3\xAB" "C\xE2\x97\x8F" "A", 1, 1); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_badunicode(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\\uD800\\uDFFF\\u110000x\"\n"); TEST_ERROR3(buff, 24, 1, CSLUL_T_String, 1, 24, 0, TDone, CSLUL_E_BADUNICODEESCAPE, 1, 7, CSLUL_E_BADUNICODEESCAPE, 1, 13, CSLUL_E_BADUNICODEESCAPE, 1, 21); ASSERT_RAW_VALUE("x", 1, 1); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_mixed(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"ab\\bc\\xf9\"\n"); TEST_ONE(buff, 12, 1, CSLUL_T_String, 1, 12, 0, TDone); ASSERT_RAW_VALUE("ab\bc\xf9", 1, 1); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_missing(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\\x\"\n"); const char *value; size_t length; TEST_ERROR(buff, 5, 1, CSLUL_T_String, 1, 5, 0, TDone, CSLUL_E_MISSINGESCAPE, 1, 4); cslul_ll_current_value(ctx, &value, &length); tassert(length == 1); tsoftassert(*value == '\0'); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_scripts(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\\C;\xD0\x93"); TEST_ONE(buff, 6, 0, CSLUL_T_NEEDDATA, 1, 6, 1, TInString); tsoftassert(ctx->allowed_scripts == SCRIPT_CYRILLIC); tassert(ctx->tmplen == 2); tsoftassert(ctx->toktmp[0] == '\xD0'); tsoftassert(ctx->toktmp[1] == '\x93'); tfree(buff); buff = cut0("\\LGSO;"); TEST_ONE(buff, 6, 0, CSLUL_T_NEEDDATA, 1, 12, 0, TInString); tsoftassert(ctx->allowed_scripts == (SCRIPT_LATIN| SCRIPT_GREEK|SCRIPT_SPECIALS|SCRIPT_OTHER)); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_scripts_bad(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\\C\xD0\x93"); TEST_ERROR(buff, 5, 0, CSLUL_T_NEEDDATA, 1, 5, 1, TInString, CSLUL_E_SCRIPTESCAPEBAD, 1, 4); tsoftassert(ctx->allowed_scripts == SCRIPT_CYRILLIC); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_scripts_unknown(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"\\CZ;\xD0\x93"); TEST_ERROR(buff, 7, 0, CSLUL_T_NEEDDATA, 1, 7, 1, TInString, CSLUL_E_SCRIPTESCAPEUNKNOWN, 1, 4); tsoftassert(ctx->allowed_scripts == SCRIPT_CYRILLIC); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_splittedeof1(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"a\\"); TEST_ONE(buff, 3, 0, NEEDDATA, 1, 4, 0, TEscapeStart); TEST_ERROR(buff, 0, 1, CSLUL_T_EOF, 1, 4, 0, TEscapeStart, CSLUL_E_UNEXPECTEDEOF, 1, 4); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_splittedeof2(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"a\\x"); TEST_ONE(buff, 4, 0, NEEDDATA, 1, 5, 0, TEscapeHex); TEST_ERROR(buff, 0, 1, CSLUL_T_EOF, 1, 5, 0, TEscapeHex, CSLUL_E_UNEXPECTEDEOF, 1, 5); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_splittedeof3(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"a\\u"); TEST_ONE(buff, 4, 0, NEEDDATA, 1, 5, 0, TEscapeUnicode); TEST_ERROR(buff, 0, 1, CSLUL_T_EOF, 1, 5, 0, TEscapeUnicode, CSLUL_E_UNEXPECTEDEOF, 1, 5); tfree(buff); free_ctx(ctx); } static void test_tok_string_escape_splittedeof4(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("\"a\\u0"); TEST_ONE(buff, 5, 0, NEEDDATA, 1, 6, 0, TEscapeUnicode); TEST_ERROR(buff, 0, 1, CSLUL_T_EOF, 1, 6, 0, TEscapeUnicode, CSLUL_E_UNEXPECTEDEOF, 1, 6); tfree(buff); free_ctx(ctx); } static void test_tok_string_script_allow_one(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char str[] = "\"" /* Latin | Latin extended A-B | Latin extended C */ "AaZz" "\xc3\x80" "\xc9\x8f" "\xe2\xb1\xa0" "\xe2\xb1\xbf" /* Cyrillic | Extended | Ext-C | Ext-A (superscripts) | Ext-B */ "\\C;" "\xD0\x80" "\xD4\xAF" "\xE1\xB2\x80" "\xE1\xB2\x8F" "\xE2\xB7\xA0" "\xE2\xB7\xBF" "\xEA\x99\x80" "\xEA\x9A\x9F" /* Greek */ "\\G;" "\xCD\xB0" "\xCF\xBF" /* Specials */ "\\S;" "\xE4\x80\xA4" /* Other */ "\\O;" "\xE2\xB2\x80" /* Allow latin again */ "\\L;Az" "\xE2\xB1\xBF" "\""; TEST_ONE(str, sizeof(str)-1, 0, CSLUL_T_String, 1, 41, 28, TDone); free_ctx(ctx); } static void test_tok_string_script_allow_combo(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char str[] = "\"" "\\CGO;" "\xF0\x9E\x8B\x81" "\xCE\x91" "\xEA\x99\x80" "\""; TEST_ONE(str, sizeof(str)-1, 0, CSLUL_T_String, 1, 11, 6, TDone); free_ctx(ctx); } static void test_tok_string_script_allow_all(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char str[] = "\"" "\\A;" "A" "\xF0\x9E\x8B\x81" "\xCE\x91" "\xEA\x99\x80" "\xE4\x80\xA4" "\""; TEST_ONE(str, sizeof(str)-1, 0, CSLUL_T_String, 1, 11, 8, TDone); free_ctx(ctx); } static void test_tok_string_script_default_disallowed(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char str[] = "\"" "\xF0\x9E\x8B\x81" /* Other */ "\xCE\x91" /* Greek */ "\xEA\x99\x80" /* Cyrillic */ "\xE4\x80\xA4" /* Chinese (not confusable, not RTL) */ "\xD8\xA7" /* Arabic */ "\xE2\x81\x84" /* Specials */ "\xCE\x92" /* <-- same error should not be reported twice in the same string */ "\""; expect_error(CSLUL_E_SCRIPTSPECIALS, 1, 7); expect_error(CSLUL_E_SCRIPTRTL, 1, 6); expect_error(CSLUL_E_SCRIPTCYRILLIC, 1, 4); expect_error(CSLUL_E_SCRIPTGREEK, 1, 3); expect_error(CSLUL_E_SCRIPTOTHER, 1, 2); TEST_ONE(str, sizeof(str)-1, 0, CSLUL_T_String, 1, 10, 12, TDone); free_ctx(ctx); } static void test_tok_string_script_explicit_disallowed(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char str[] = "\"" "\\C;" "\xF0\x9E\x8B\x81" /* Other */ "\xCE\x91" /* Greek */ "\xE4\x80\xA4" /* Chinese (not confusable, not RTL) */ "\xD8\xA7" /* Arabic */ "\xE2\x81\x84" /* Specials */ "A" /* Latin */ "\xCE\x92" /* <-- same error should not be reported twice in the same string */ "\\L;" "\xEA\x99\x80" /* Cyrillic */ "\""; expect_error(CSLUL_E_SCRIPTCYRILLIC, 1, 15); /*expect_error(CSLUL_E_SCRIPTLATIN, 1, 10);*/ /* FIXME disallowing of A-Z */ expect_error(CSLUL_E_SCRIPTSPECIALS, 1, 9); expect_error(CSLUL_E_SCRIPTRTL, 1, 8); expect_error(CSLUL_E_SCRIPTGREEK, 1, 6); expect_error(CSLUL_E_SCRIPTOTHER, 1, 5); TEST_ONE(str, sizeof(str)-1, 0, CSLUL_T_String, 1, 17, 12, TDone); free_ctx(ctx); } static void test_tok_string_splitted(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = tmalloc(1); buff[0] = '"'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 2, 0, TInString); buff[0] = 'a'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 3, 0, TInString); buff[0] = '\\'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 4, 0, TEscapeStart); buff[0] = 'x'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 5, 0, TEscapeHex); tsoftassert(ctx->parser.slul.escapelen == 0); tsoftassert(ctx->parser.slul.escape == 0); buff[0] = '5'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 6, 0, TEscapeHex); tsoftassert(ctx->parser.slul.escapelen == 1); tsoftassert(ctx->parser.slul.escape == 0x5); buff[0] = 'A'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 7, 0, TEscapeHex); tsoftassert(ctx->parser.slul.escapelen == 2); tsoftassert(ctx->parser.slul.escape == 0x5A); buff[0] = 'g'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 8, 0, TInString); buff[0] = '\\'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 9, 0, TEscapeStart); buff[0] = 'u'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 10, 0, TEscapeUnicode); buff[0] = 'C'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 11, 0, TEscapeUnicode); buff[0] = '5'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 12, 0, TEscapeUnicode); buff[0] = '"'; TEST_ONE(buff, 1, 0, CSLUL_T_String, 1, 13, 0, TDone); ASSERT_RAW_VALUE("a\x5Ag\xC3\x85", 1, 1); TEST_CONT(NEEDDATA, 1, 13, 0, TDone); tfree(buff); free_ctx(ctx); } #define CONTINUE_NUMBER(line, col, expected_number) do { \ TEST_CONT(CSLUL_T_Integer, (line), (col), 0, TDone); \ tsoftassert(ctx->parser.slul.number == (expected_number)); \ } while (0) static void test_tok_number_hex(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "0x0\n" "0x1\n" "0x9\n" "0x10\n" "0x_12\n" "0x2_3\n" "0x1A\n" "0x1F\n" "0x1a\n" "0x1f\n" "0xa_B\n" "0xcd_\n" "0xFFF\n" "0xFFFF\n" "0xFFFFFFFF\n" "0xFFFF_FFFF_FFFF_FFFF\n" "0x00_00FFFF_FFFF_FFFF_FFFF\n"; TEST_ONE(&buff[0], sizeof(buff)-1, 1, CSLUL_T_Integer, 1, 4, 0, TDone); tsoftassert(ctx->parser.slul.number == 0x0); CONTINUE_NUMBER(2, 4, 0x1); CONTINUE_NUMBER(3, 4, 0x9); CONTINUE_NUMBER(4, 5, 0x10); CONTINUE_NUMBER(5, 6, 0x12); CONTINUE_NUMBER(6, 6, 0x23); CONTINUE_NUMBER(7, 5, 0x1a); CONTINUE_NUMBER(8, 5, 0x1f); CONTINUE_NUMBER(9, 5, 0x1a); CONTINUE_NUMBER(10, 5, 0x1f); CONTINUE_NUMBER(11, 6, 0xab); CONTINUE_NUMBER(12, 6, 0xcd); CONTINUE_NUMBER(13, 6, 0xfff); CONTINUE_NUMBER(14, 7, 0xffffU); CONTINUE_NUMBER(15, 11, UINT32_MAX); CONTINUE_NUMBER(16, 22, UINT64_MAX); CONTINUE_NUMBER(17, 27, UINT64_MAX); TEST_CONT(CSLUL_T_EOF, 18, 1, 0, TDone); free_ctx(ctx); } #define CONT_ANYNUM_ERR(type, line, col, expected, err, errline, errcol) do { \ expect_error(err, errline, errcol); \ TEST_CONT(type, (line), (col), 0, TDone); \ verify_errors(); \ tsoftassert(ctx->parser.slul.number == (expected)); \ } while (0) #define CONT_NUMBER_ERR(line, col, expected, err, errline, errcol) \ CONT_ANYNUM_ERR(CSLUL_T_Integer, line, col, expected, err, errline, errcol) static void test_tok_number_hex_error(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "0x\n" "0x_\n" "0x1g\n" "0xg1\n" "0x1.2\n" "0x10000_0000_0000_0000\n"; TEST_ERROR(&buff[0], sizeof(buff)-1, 1, CSLUL_T_Integer, 1, 3, 0, TDone, CSLUL_E_NODIGITS, 1, 3); tsoftassert(ctx->parser.slul.number == 0x0); CONT_NUMBER_ERR(2, 4, 0x0, CSLUL_E_NODIGITS, 2, 4); CONT_NUMBER_ERR(3, 4, 0x1, CSLUL_E_BADNUMBERCHAR, 3, 4); TEST_CONT(CSLUL_T_LowerIdent, 3, 5, 0, TDone); /* "g" */ CONT_NUMBER_ERR(4, 3, 0x0, CSLUL_E_BADNUMBERCHAR, 4, 3); TEST_CONT(CSLUL_T_LowerIdent, 4, 5, 0, TDone); /* "g1" */ CONT_NUMBER_ERR(5, 6, 0x1, CSLUL_E_BADNUMBERCHAR, 5, 4); expect_error(CSLUL_E_NUMBERTOOLARGE, 6, 22); TEST_CONT(CSLUL_T_Integer, 6, 23, 0, TDone); verify_errors(); TEST_CONT(CSLUL_T_EOF, 7, 1, 0, TDone); free_ctx(ctx); } static void test_tok_number_bin(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "0b0\n" "0b1\n" "0b01\n" "0b10\n" "0b_10\n" "0b0_1\n" "0b11_\n" "0b1010_1010_1100_1100_1111\n" "0b1111111111111111111111111111111111111111111111111111111111111111\n" "0b00_00011111111_11111111_11111111_11111111_11111111_11111111_11111111_11111111_\n"; TEST_ONE(&buff[0], sizeof(buff)-1, 1, CSLUL_T_Integer, 1, 4, 0, TDone); tsoftassert(ctx->parser.slul.number == 0x0); CONTINUE_NUMBER(2, 4, 0x1); CONTINUE_NUMBER(3, 5, 0x1); CONTINUE_NUMBER(4, 5, 0x2); CONTINUE_NUMBER(5, 6, 0x2); CONTINUE_NUMBER(6, 6, 0x1); CONTINUE_NUMBER(7, 6, 0x3); CONTINUE_NUMBER(8, 27, 0xaaccf); CONTINUE_NUMBER(9, 67, UINT64_MAX); CONTINUE_NUMBER(10, 81, UINT64_MAX); TEST_CONT(CSLUL_T_EOF, 11, 1, 0, TDone); free_ctx(ctx); } static void test_tok_number_bin_error(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "0b\n" "0b_\n" "0b12\n" "0b21\n" "0b1.2\n" "0b10000000000000000000000000000000000000000000000000000000000000000\n"; TEST_ERROR(&buff[0], sizeof(buff)-1, 1, CSLUL_T_Integer, 1, 3, 0, TDone, CSLUL_E_NODIGITS, 1, 3); tsoftassert(ctx->parser.slul.number == 0x0); CONT_NUMBER_ERR(2, 4, 0x0, CSLUL_E_NODIGITS, 2, 4); CONT_NUMBER_ERR(3, 5, 0x1, CSLUL_E_BADNUMBERCHAR, 3, 4); CONT_NUMBER_ERR(4, 5, 0x0, CSLUL_E_BADNUMBERCHAR, 4, 3); CONT_NUMBER_ERR(5, 6, 0x1, CSLUL_E_BADNUMBERCHAR, 5, 4); expect_error(CSLUL_E_NUMBERTOOLARGE, 6, 67); TEST_CONT(CSLUL_T_Integer, 6, 68, 0, TDone); verify_errors(); TEST_CONT(CSLUL_T_EOF, 7, 1, 0, TDone); free_ctx(ctx); } static void test_tok_number_dec(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "0\n" "1\n" "9\n" "10\n" "1_2\n" "23_\n" "4294967295\n" "18446744073709551614\n" "18446744073709551615\n" "184__46744073709551_615_\n"; TEST_ONE(&buff[0], sizeof(buff)-1, 1, CSLUL_T_Integer, 1, 2, 0, TDone); tsoftassert(ctx->parser.slul.number == 0); CONTINUE_NUMBER(2, 2, 1); CONTINUE_NUMBER(3, 2, 9); CONTINUE_NUMBER(4, 3, 10); CONTINUE_NUMBER(5, 4, 12); CONTINUE_NUMBER(6, 4, 23); CONTINUE_NUMBER(7, 11, UINT32_MAX); CONTINUE_NUMBER(8, 21, UINT64_MAX-1); CONTINUE_NUMBER(9, 21, UINT64_MAX); CONTINUE_NUMBER(10, 25, UINT64_MAX); TEST_CONT(CSLUL_T_EOF, 11, 1, 0, TDone); free_ctx(ctx); } static void test_tok_number_dec_error(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "00\n" "01\n" "0_1\n" "0a\n" "1x1\n" "18446744073709551616\n" "18446744073709551619\n" "18446744073709551620\n"; TEST_ERROR(&buff[0], sizeof(buff)-1, 1, CSLUL_T_Integer, 1, 3, 0, TDone, CSLUL_E_LEADINGZERO, 1, 2); tsoftassert(ctx->parser.slul.number == 0x0); CONT_NUMBER_ERR(2, 3, 0, CSLUL_E_LEADINGZERO, 2, 2); CONT_NUMBER_ERR(3, 4, 0, CSLUL_E_LEADINGZERO, 3, 2); CONT_NUMBER_ERR(4, 3, 0, CSLUL_E_BADNUMBERTYPE, 4, 2); CONT_NUMBER_ERR(5, 2, 1, CSLUL_E_BADNUMBERCHAR, 5, 2); TEST_CONT(CSLUL_T_LowerIdent, 5, 4, 0, TDone); /* "x1" */ expect_error(CSLUL_E_NUMBERTOOLARGE, 6, 20); TEST_CONT(CSLUL_T_Integer, 6, 21, 0, TDone); verify_errors(); expect_error(CSLUL_E_NUMBERTOOLARGE, 7, 20); TEST_CONT(CSLUL_T_Integer, 7, 21, 0, TDone); verify_errors(); expect_error(CSLUL_E_NUMBERTOOLARGE, 8, 20); TEST_CONT(CSLUL_T_Integer, 8, 21, 0, TDone); verify_errors(); TEST_CONT(CSLUL_T_EOF, 9, 1, 0, TDone); free_ctx(ctx); } static void test_tok_number_dec_zeroeof(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); TEST_ONE("0", 1, 1, CSLUL_T_Integer, 1, 2, 0, TDone); tsoftassert(ctx->parser.slul.number == 0); expect_error(CSLUL_E_NOEOFNEWLINE, 1, 2); TEST_CONT(CSLUL_T_EOF, 1, 2, 0, TDone); free_ctx(ctx); } #define CONTINUE_FLOAT(line, col, num, exp) do { \ TEST_CONT(CSLUL_T_Float, (line), (col), 0, TDone); \ tsoftassert(ctx->parser.slul.number == (num)); \ if (!tsoftassert(ctx->parser.slul.exponent == (exp)) && !quiet) \ tprintf(" was %d\n", (int)ctx->parser.slul.exponent); \ } while (0) static void test_tok_number_float(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "0.0\n" "0.00\n" "1.2\n" "0.23\n" "1_2.34\n" "5.6_\n" "78e3\n" "23.4e3\n" "23.456789e3\n" "0.123456789e10\n" "0.00000000000000456789\n" "0.000000000_00000456789e4\n" "0.000000000_00000456789e25\n" "123e-3\n" "0.123e-321\n" "0.567e1000\n"; TEST_ONE(&buff[0], sizeof(buff)-1, 1, CSLUL_T_Float, 1, 4, 0, TDone); tsoftassert(ctx->parser.slul.number == 0); tsoftassert(ctx->parser.slul.exponent == 0); /* Line Col Num Exp */ CONTINUE_FLOAT(2, 5, 0, 0); CONTINUE_FLOAT(3, 4, 12, -1); CONTINUE_FLOAT(4, 5, 23, -2); CONTINUE_FLOAT(5, 7, 1234, -2); CONTINUE_FLOAT(6, 5, 56, -1); CONTINUE_FLOAT(7, 5, 78, 3); CONTINUE_FLOAT(8, 7, 234, 2); CONTINUE_FLOAT(9, 12, 23456789, -3); CONTINUE_FLOAT(10, 15, 123456789, 1); CONTINUE_FLOAT(11, 23, 456789, -20); CONTINUE_FLOAT(12, 26, 456789, -16); CONTINUE_FLOAT(13, 27, 456789, 5); CONTINUE_FLOAT(14, 7, 123, -3); CONTINUE_FLOAT(15, 11, 123, -324); CONTINUE_FLOAT(16, 11, 567, 997); TEST_CONT(CSLUL_T_EOF, 17, 1, 0, TDone); free_ctx(ctx); } static void test_tok_number_float_error(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "0.\n" "1.\n" "1..2\n" "1.2.3\n" "0e\n" "1e\n" "0ee\n" "1ee\n" "2e++1\n" "3e--1\n" "1e9999999999999999\n" "1e-9999999999999999\n" "4e1e\n" "5e0.1\n" "6.e3\n" "7.x\n" "8ex\n" /* .1 is disallowed also, but will be detected in the parsing phase */ ; TEST_ERROR(&buff[0], sizeof(buff)-1, 1, CSLUL_T_Integer, 1, 3, 0, TDone, CSLUL_E_NOFRACDIGITS, 1, 3); tsoftassert(ctx->parser.slul.number == 0x0); CONT_NUMBER_ERR(2, 3, 1, CSLUL_E_NOFRACDIGITS, 2, 3); CONT_NUMBER_ERR(3, 5, 1, CSLUL_E_DOUBLEDECPOINT, 3, 3); CONT_NUMBER_ERR(4, 6, 12, CSLUL_E_DOUBLEDECPOINT, 4, 4); CONT_NUMBER_ERR(5, 3, 0, CSLUL_E_BADNUMBERTYPE, 5, 2); CONT_NUMBER_ERR(6, 3, 1, CSLUL_E_NOEXPDIGITS, 6, 3); CONT_NUMBER_ERR(7, 4, 0, CSLUL_E_BADNUMBERTYPE, 7, 2); CONT_NUMBER_ERR(8, 4, 1, CSLUL_E_NOEXPDIGITS, 8, 3); CONT_NUMBER_ERR(9, 4, 2, CSLUL_E_NOEXPDIGITS, 9, 4); TEST_CONT(CSLUL_T_Plus, 9, 5, 0, TDone); TEST_CONT(CSLUL_T_Integer, 9, 6, 0, TDone); CONT_NUMBER_ERR(10, 4, 3, CSLUL_E_NOEXPDIGITS, 10, 4); TEST_CONT(CSLUL_T_Minus, 10, 5, 0, TDone); TEST_CONT(CSLUL_T_Integer, 10, 6, 0, TDone); CONT_NUMBER_ERR(11, 19, 1, CSLUL_E_EXPONENTTOOLARGE, 11, 7); CONT_NUMBER_ERR(12, 20, 1, CSLUL_E_EXPONENTTOOLARGE, 12, 8); CONT_NUMBER_ERR(13, 5, 4, CSLUL_E_BADNUMBERCHAR, 13, 4); CONT_NUMBER_ERR(14, 6, 5, CSLUL_E_BADNUMBERCHAR, 14, 4); CONT_NUMBER_ERR(15, 5, 6, CSLUL_E_NOFRACDIGITS, 15, 3); CONT_NUMBER_ERR(16, 3, 7, CSLUL_E_BADNUMBERCHAR, 16, 3); TEST_CONT(CSLUL_T_LowerIdent, 16, 4, 0, TDone); CONT_NUMBER_ERR(17, 3, 8, CSLUL_E_NOEXPDIGITS, 17, 3); TEST_CONT(CSLUL_T_LowerIdent, 17, 4, 0, TDone); TEST_CONT(CSLUL_T_EOF, 18, 1, 0, TDone); free_ctx(ctx); } static void test_tok_number_float_splitted(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = tmalloc(1); buff[0] = '0'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 2, 0, TZeroPrefixed); buff[0] = '.'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 3, 0, TNumberDec); buff[0] = '2'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 4, 0, TNumberDec); buff[0] = 'e'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 5, 0, TNumberExpSign); buff[0] = '-'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 6, 0, TNumberExp); buff[0] = '3'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 7, 0, TNumberExp); buff[0] = '\n'; TEST_ONE(buff, 1, 0, CSLUL_T_Float, 1, 7, 0, TDone); tsoftassert(ctx->parser.slul.number == 2); tsoftassert(ctx->parser.slul.exponent == -4); TEST_CONT(NEEDDATA, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_number_hex_splitted(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = tmalloc(1); buff[0] = '0'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 2, 0, TZeroPrefixed); buff[0] = 'x'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 3, 0, TNumberHex); buff[0] = 'F'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 4, 0, TNumberHex); buff[0] = '_'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 5, 0, TNumberHex); buff[0] = '9'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 6, 0, TNumberHex); buff[0] = '\n'; TEST_ONE(buff, 1, 0, CSLUL_T_Integer, 1, 6, 0, TDone); tsoftassert(ctx->parser.slul.numdigits == 2); tsoftassert(ctx->parser.slul.number == 0xf9); TEST_CONT(NEEDDATA, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_number_bin_splitted(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = tmalloc(1); buff[0] = '0'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 2, 0, TZeroPrefixed); buff[0] = 'b'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 3, 0, TNumberBin); buff[0] = '1'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 4, 0, TNumberBin); buff[0] = '_'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 5, 0, TNumberBin); buff[0] = '0'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 6, 0, TNumberBin); buff[0] = '\n'; TEST_ONE(buff, 1, 0, CSLUL_T_Integer, 1, 6, 0, TDone); tsoftassert(ctx->parser.slul.numdigits == 2); tsoftassert(ctx->parser.slul.number == 0x2); TEST_CONT(NEEDDATA, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_number_bin_error_splitted(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = tmalloc(1); buff[0] = '0'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 2, 0, TZeroPrefixed); buff[0] = 'b'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 3, 0, TNumberBin); buff[0] = '1'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 4, 0, TNumberBin); buff[0] = '2'; TEST_ERROR(buff, 1, 0, NEEDDATA, 1, 5, 0, TNumberSkip, CSLUL_E_BADNUMBERCHAR, 1, 4); buff[0] = '0'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 6, 0, TNumberSkip); buff[0] = '\n'; TEST_ONE(buff, 1, 0, CSLUL_T_Integer, 1, 6, 0, TDone); tsoftassert(ctx->parser.slul.numdigits == INVALID_NUMBER); tsoftassert(ctx->parser.slul.number == 0x1); TEST_CONT(NEEDDATA, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_number_dec_splitted(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = tmalloc(1); buff[0] = '9'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 2, 0, TNumberDec); buff[0] = '_'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 3, 0, TNumberDec); buff[0] = '8'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 4, 0, TNumberDec); buff[0] = '_'; TEST_ONE(buff, 1, 0, NEEDDATA, 1, 5, 0, TNumberDec); buff[0] = '\n'; TEST_ONE(buff, 1, 0, CSLUL_T_Integer, 1, 5, 0, TDone); tsoftassert(ctx->parser.slul.numdigits == 2); tsoftassert(ctx->parser.slul.number == 98); TEST_CONT(NEEDDATA, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_number_kwend(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); char *buff = cut0("2not\n"); TEST_ERROR(buff, 5, 1, CSLUL_T_Integer, 1, 2, 0, TDone, CSLUL_E_BADNUMBERCHAR, 1, 2); tsoftassert(ctx->parser.slul.number == 2); TEST_CONT(CSLUL_T_KW_Not, 1, 5, 0, TDone); TEST_CONT(CSLUL_T_EOF, 2, 1, 0, TDone); tfree(buff); free_ctx(ctx); } static void test_tok_trailingwhitespace(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = "type \n"; TEST_ONE(&buff[0], sizeof(buff)-1, 1, CSLUL_T_KW_Type, 1, 5, 0, TDone); expect_error(CSLUL_E_TRAILINGSPACE, 1, 6); TEST_CONT(EOF, 2, 1, 0, TDone); free_ctx(ctx); } static void test_tok_indentedwhitespace(void) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); static const char buff[] = " \n"; static const char buff2[] = " \n"; TEST_ERROR(&buff[0], sizeof(buff)-1, 0, NEEDDATA, 2, 1, 0, TDone, CSLUL_E_INDENTEDBLANKLINE, 1, 2); TEST_ERROR(&buff2[0], sizeof(buff2)-1, 1, CSLUL_T_EOF, 3, 1, 0, TDone, CSLUL_E_INDENTEDBLANKLINE, 2, 3); free_ctx(ctx); } static void check_tokenize_ident(enum IdentParseMode mode) { struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); const char *start, *end; int status; cslul_ll_set_current_filename(ctx, ""); ctx->last_buffer = 0; /* Full identifier */ ctx->toklen = 0; ctx->tmplen = 0; start = "short "; end = start+6; status = 0; sourceline = __LINE__; tsoftassert(tokenize_ident(ctx, start, end, &status, mode) == start+5); tsoftassert(ctx->toklen == 5); tsoftassert(ctx->tmplen == 0); tsoftassert(ctx->tokval == start); tsoftassert(!memcmp(ctx->tokval, "short", 5)); tsoftassert(status == 1); /* Truncated identifier */ ctx->toklen = 0; ctx->tmplen = 0; start = "te"; end = start+2; status = 0; sourceline = __LINE__; tsoftassert(tokenize_ident(ctx, start, end, &status, mode) == start+2); tsoftassert(ctx->toklen == 2); tsoftassert(ctx->tmplen == 2); tsoftassert(!memcmp(ctx->toktmp, "te", 2)); tsoftassert(status == 0); /* Continued identifer */ ctx->toklen = 2; ctx->tmplen = 2; start = "st"; end = start+2; status = 0; sourceline = __LINE__; tsoftassert(tokenize_ident(ctx, start, end, &status, mode) == start+2); tsoftassert(ctx->toklen == 4); tsoftassert(ctx->tmplen == 4); tsoftassert(!memcmp(ctx->toktmp, "test", 4)); tsoftassert(status == 0); /* Ended identifier */ ctx->toklen = 4; ctx->tmplen = 4; start = " "; end = start+1; status = 0; sourceline = __LINE__; tsoftassert(tokenize_ident(ctx, start, end, &status, mode) == start); tsoftassert(ctx->toklen == 4); tsoftassert(ctx->tmplen == 4); tsoftassert(ctx->tokval == ctx->toktmp); tsoftassert(!memcmp(ctx->tokval, "test", 4)); tsoftassert(status == 1); free_ctx(ctx); } static void test_tokenize_ident(void) { if (char_by_char) return; check_tokenize_ident(ParseIdent); } static void test_tokenize_value(void) { if (char_by_char) return; check_tokenize_ident(ParseAttrValue); } static void check_tokenize_ident_long(enum IdentParseMode mode) { /* FIXME There are some bugs with identifiers that are exactly 100 characters long, that causes a "identifier too long" error in some cases. For now, we simply test with 99 character identifiers. */ #define BUG100CH - 1 struct CSlul *ctx = create_ctx(CSLUL_P_IMPL); const char *start, *end; int status; char buff[101 BUG100CH]; cslul_ll_set_current_filename(ctx, ""); ctx->last_buffer = 0; /* Full identifier */ ctx->toklen = 0; ctx->tmplen = 0; memset(buff, 'a', 100 BUG100CH); buff[100 BUG100CH] = ' '; end = &buff[101 BUG100CH]; status = 0; sourceline = __LINE__; tsoftassert(tokenize_ident(ctx, buff, end, &status, mode) == end-1); tsoftassert(ctx->toklen == 100 BUG100CH); tsoftassert(ctx->tmplen == 0); tsoftassert(ctx->tokval == buff); tsoftassert(status == 1); /* Truncated identifier */ ctx->toklen = 0; ctx->tmplen = 0; memset(buff, 'a', 100 BUG100CH); end = &buff[100 BUG100CH]; status = 0; sourceline = __LINE__; tsoftassert(tokenize_ident(ctx, buff, end, &status, mode) == end); tsoftassert(ctx->toklen == 100 BUG100CH); tsoftassert(ctx->tmplen == 100 BUG100CH); tsoftassert(!memcmp(ctx->toktmp, buff, 100 BUG100CH)); tsoftassert(status == 0); /* Continued identifer */ ctx->toklen = 90; ctx->tmplen = 90; memset(ctx->toktmp, 'a', 100); memset(buff, 'b', 100 BUG100CH); end = &buff[100 BUG100CH]; status = 0; ctx->line = 1; ctx->startcolumn = 91; ctx->linestart = buff; ctx->mbtrailerbytes = 0; ctx->bufferend = end; expect_error(CSLUL_E_IDENTTOOLONG, 1, 101); sourceline = __LINE__; tsoftassert(tokenize_ident(ctx, buff, end, &status, mode) == end); tsoftassert(ctx->toklen == 100); tsoftassert(ctx->tmplen == 100); tsoftassert(ctx->toktmp[0] == 'a'); tsoftassert(ctx->toktmp[89] == 'a'); tsoftassert(ctx->toktmp[90] == 'b'); tsoftassert(ctx->toktmp[99] == 'b'); tsoftassert(status == 0); verify_errors(); /* Ended identifier */ ctx->toklen = 100 BUG100CH; ctx->tmplen = 100 BUG100CH; start = " "; end = start+1; status = 0; sourceline = __LINE__; tsoftassert(tokenize_ident(ctx, start, end, &status, mode) == start); tsoftassert(ctx->toklen == 100 BUG100CH); tsoftassert(ctx->tmplen == 100 BUG100CH); tsoftassert(ctx->tokval == ctx->toktmp); tsoftassert(status == 1); free_ctx(ctx); } static void test_tokenize_ident_long(void) { if (char_by_char) return; check_tokenize_ident_long(ParseIdent); } static void test_tokenize_value_long(void) { if (char_by_char) return; check_tokenize_ident_long(ParseAttrValue); } const TestFunctionInfo tests_token[] = { TEST_BEFORE(before_test) TEST_AFTER(after_test) TEST_INFO(test_tok_empty) TEST_INFO(test_tok_newline_single) TEST_INFO(test_tok_newline_splitted) TEST_INFO(test_tok_newline_eof1) TEST_INFO(test_tok_newline_eof2) TEST_INFO(test_tok_null) TEST_INFO(test_tok_utf8_2min) TEST_INFO(test_tok_utf8_2max) TEST_INFO(test_tok_utf8_2split) TEST_INFO(test_tok_utf8_2overlong) TEST_INFO(test_tok_utf8_2eof) TEST_INFO(test_tok_utf8_2badcont) TEST_INFO(test_tok_utf8_3min) TEST_INFO(test_tok_utf8_3max) TEST_INFO(test_tok_utf8_3split) TEST_INFO(test_tok_utf8_3overlong) TEST_INFO(test_tok_utf8_3eof2) TEST_INFO(test_tok_utf8_3nullchar3) TEST_INFO(test_tok_utf8_4min) TEST_INFO(test_tok_utf8_4max) TEST_INFO(test_tok_utf8_4split) TEST_INFO(test_tok_utf8_4overlong) TEST_INFO(test_tok_utf8_4overflow) TEST_INFO(test_tok_utf8_4eof2) TEST_INFO(test_tok_utf8_4eof3) TEST_INFO(test_tok_utf8_4nullchar3) TEST_INFO(test_tok_utf8_4nullchar4) TEST_INFO(test_tok_utf8_forbidden) TEST_INFO(test_tok_comment) TEST_INFO(test_tok_comment_badchar) TEST_INFO(test_tok_multiline_comment) TEST_INFO(test_tok_multiline_comment_badstart) TEST_INFO(test_tok_keyword_lowerident) TEST_INFO(test_tok_keyword_upperident) TEST_INFO(test_tok_keyword_splitted) TEST_INFO(test_tok_gototarget) TEST_INFO(test_tok_version) TEST_INFO(test_tok_since_version) TEST_INFO(test_tok_identtoolong) TEST_INFO(test_tok_identtoolong_splitted1) TEST_INFO(test_tok_identtoolong_splitted2) TEST_INFO(test_tok_operator_1char) TEST_INFO(test_tok_operator_2char) TEST_INFO(test_tok_operator_2char_splitted) TEST_INFO(test_tok_operator_2char_eof) TEST_INFO(test_tok_string_empty) TEST_INFO(test_tok_string_simple) TEST_INFO(test_tok_string_controlchar) TEST_INFO(test_tok_string_utf8) TEST_INFO(test_tok_string_maxsize) TEST_INFO(test_tok_string_toolong) TEST_INFO(test_tok_string_toolong2) TEST_INFO(test_tok_string_unterminated1) TEST_INFO(test_tok_string_unterminated2) TEST_INFO(test_tok_string_unterminated3) TEST_INFO(test_tok_string_escape_single) TEST_INFO(test_tok_string_escape_hexbyte) TEST_INFO(test_tok_string_escape_unicode) TEST_INFO(test_tok_string_escape_toolong) TEST_INFO(test_tok_string_escape_badunicode) TEST_INFO(test_tok_string_escape_mixed) TEST_INFO(test_tok_string_escape_missing) TEST_INFO(test_tok_string_escape_scripts) TEST_INFO(test_tok_string_escape_scripts_bad) TEST_INFO(test_tok_string_escape_scripts_unknown) TEST_INFO(test_tok_string_escape_splittedeof1) TEST_INFO(test_tok_string_escape_splittedeof2) TEST_INFO(test_tok_string_escape_splittedeof3) TEST_INFO(test_tok_string_escape_splittedeof4) TEST_INFO(test_tok_string_script_allow_one) TEST_INFO(test_tok_string_script_allow_combo) TEST_INFO(test_tok_string_script_allow_all) TEST_INFO(test_tok_string_script_default_disallowed) TEST_INFO(test_tok_string_script_explicit_disallowed) TEST_INFO(test_tok_string_splitted) TEST_INFO(test_tok_number_hex) TEST_INFO(test_tok_number_hex_error) TEST_INFO(test_tok_number_hex_splitted) TEST_INFO(test_tok_number_bin) TEST_INFO(test_tok_number_bin_error) TEST_INFO(test_tok_number_bin_splitted) TEST_INFO(test_tok_number_bin_error_splitted) TEST_INFO(test_tok_number_dec) TEST_INFO(test_tok_number_dec_error) TEST_INFO(test_tok_number_dec_splitted) TEST_INFO(test_tok_number_dec_zeroeof) TEST_INFO(test_tok_number_float) TEST_INFO(test_tok_number_float_error) TEST_INFO(test_tok_number_float_splitted) TEST_INFO(test_tok_number_kwend) TEST_INFO(test_tok_trailingwhitespace) TEST_INFO(test_tok_indentedwhitespace) TEST_INFO(test_tokenize_ident) TEST_INFO(test_tokenize_value) TEST_INFO(test_tokenize_ident_long) TEST_INFO(test_tokenize_value_long) TEST_END };