/*

  Common helper functions for parser/mhparser unit tests

  Copyright © 2021-2024 Samuel Lidén Borell <samuel@kodafritt.se>

  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
  in the Software without restriction, including without limitation the rights
  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  copies of the Software, and to permit persons to whom the Software is
  furnished to do so, subject to the following conditions:

  The above copyright notice and this permission notice shall be included in
  all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  THE SOFTWARE.

*/

#include "parsecommon.h"
#include "../internal.h"
#include "../ast.h"
#include "../hash.h"
#include "unittest.h"
#include "testcommon.h"
#include <assert.h>
#include <stdlib.h>
#include <string.h>

struct ExpectedError expected[MAX_EXPECTED_ERRS];
int num_expected = 0;
int sourceline;
static struct CSlulState lasterror;
static const char *expected_filename;

/* These are for selecting between module header and SLUL parsing */
static int mhparse;
static enum MHTokenState badchar_state;
static enum CSlulPhase phase;

void set_mhparse(int value)
{
    mhparse = value;
    phase = mhparse ? CSLUL_P_MODULEHEADER : CSLUL_P_IMPL;
    badchar_state = mhparse ? MHTInComment : (enum MHTokenState)TDone;
}

static void msghandler(const struct CSlulState *state)
{
    char s[200];
    struct ExpectedError *experror = NULL;
    lasterror = *state;
    /*tprintf("[line %d] msghandler, code: %d, %s:%d.%d: %-40s\n",
        sourceline, state->errorcode, state->locs[0].filename,
        state->locs[0].line, state->locs[0].column,
        cslul_lookup_message(state->errorcode));*/
    if (num_expected != 0) {
        experror = &expected[num_expected-1];
        if (state->errorcode == experror->errcode &&
            state->locs[0].line == experror->line &&
            state->locs[0].column == experror->column &&
            (!experror->filename ||
                !strcmp(experror->filename, state->locs[0].filename))) {
            /* Expected error */
            num_expected--;
            return;
        }
    }
    sprintf(s, "Unexpected error %d was reported at %s "
            "line %d column %d in buffer.", (int)state->errorcode,
            state->locs[0].filename ? state->locs[0].filename : "",
            state->locs[0].line, state->locs[0].column);
    tsoftfail_("<unknown>", sourceline, s, 0);
    if (!quiet) {
        tprintf("- error description: %-40s\n",
                cslul_lookup_message(state->errorcode));
        if (experror) {
            tprintf("- expected error %d, %s%sline %d, column %d\n",
                    (int)experror->errcode,
                    experror->filename ? experror->filename : "",
                    experror->filename ? " " : "",
                    experror->line, experror->column);
        }
    }
}

struct CSlul *create_ctx(enum CSlulPhase initial_phase)
{
    return create_ctx_cfg(initial_phase, NULL);
}

/* TODO might need a flag for controlling the presence of \api_def's
   (for unstable/unversioned libraries) */
struct CSlul *create_ctx_cfg(enum CSlulPhase initial_phase,
                             struct CSlulConfig *cfg)
{
    static const char app_header[] =
        "\\slul 0.0.0\n"
        "\\name test\n"
        "\\version 1.3\n";
    static const char lib_header[] =
        "\\slul 0.0.0\n"
        "\\name test\n"
        "\\type library\n"
        "\\version 1.3\n"
        "\\api_def 1.0\n"
        "\\api_def 1.1\n"
        "\\api_def 1.0.1\n"
        "\\api_def 1.0.2\n"
        "\\api_def 1.2\n"
        "\\api_def 1.0.3\n"
        "\\api_def 1.3\n";
    struct CSlul *ctx;
    if (!cfg) {
        cfg = tcfg(cslul_config_create(NULL));
        tassert(cfg != NULL);
    }
    cslul_config_set_message_handler(cfg, msghandler);
    ctx = tctx(cslul_create(cfg));
    tassert(ctx != NULL);
    if (initial_phase) {
        if (initial_phase == CSLUL_P_IMPL || initial_phase == CSLUL_P_IFACE) {
            /* Must enter moduleheader phase first */
            tassert(cslul_ll_start_phase(ctx, CSLUL_P_MODULEHEADER));
            cslul_ll_set_current_filename(ctx, "<unittest setup>");
            if (initial_phase == CSLUL_P_IMPL) {
                cslul_ll_set_input_buffer(ctx, app_header,
                                          sizeof(app_header)-1, 0);
            } else {
                cslul_ll_set_input_buffer(ctx, lib_header,
                                          sizeof(lib_header)-1, 0);
            }
            cslul_ll_parse(ctx);
            tassert(!ctx->has_fatal_errors);
            determine_effective_langver(ctx, 0);
            if (initial_phase == CSLUL_P_IMPL) {
                tassert(cslul_ll_start_phase(ctx, CSLUL_P_IFACE));
            }
        }
        tassert(cslul_ll_start_phase(ctx, initial_phase));
        cslul_ll_set_current_filename(ctx, "<unittest>");
        ctx->tokline = 1;
        ctx->tokcolumn = 1;
    }
    /* We're not testing the runtime. Skip checks for SlulApp.main */
    ctx->has_app_main = 1;
    return ctx;
}

void free_ctx(struct CSlul *ctx)
{
    struct CSlulConfig *cfg = (struct CSlulConfig *)ctx->cfg;
    cslul_free(ctx);
    cslul_config_free(cfg);
}

void errors_in(const char *filename)
{
    expected_filename = filename;
}

void expect_error(enum CSlulErrorCode errorcode, int line, int column)
{
    assert(num_expected < MAX_EXPECTED_ERRS);
    expected[num_expected].errcode = errorcode;
    expected[num_expected].line = line;
    expected[num_expected].column = column;
    expected[num_expected].filename = expected_filename;
    num_expected++;
}

/**
 * Feeds a given string to the parser, optionally 1 character at a time
 * if char_by_char is set
 */
void test_source(struct CSlul *ctx, const char *s, size_t length, int is_last)
{
    int happy_path = !num_expected;
    if (ctx->has_fatal_errors) {
        tfail("test_source called after a fatal error occurred");
    }
    if (char_by_char) {
        const char *c = (s);
        size_t left = length;
        if (!left) {
            cslul_ll_set_input_buffer(ctx, c, 0, (is_last));
            cslul_ll_parse(ctx);
        } else {
            while (left-- && !cslul_ll_has_fatal_errors(ctx)) {
                cslul_ll_set_input_buffer(ctx, c, 1, is_last && !left);
                cslul_ll_parse(ctx);
                c++;
            }
        }
    } else {
        cslul_ll_set_input_buffer(ctx, (s), length, is_last);
        cslul_ll_parse(ctx);
    }
    tassert(!(ctx->has_fatal_errors && happy_path)); /* Abort in OoM-test */
    verify_errors();
}

void verify_errors(void)
{
    if (num_expected) {
        if (!quiet) {
            tprintf("Missing %d errors [line %d]\n", num_expected,
                    sourceline);
        }
        tsoftfail("Expected an error");
        num_expected = 0;
    }
    expected_filename = NULL;
}

/**
 * Duplicates a string without including the null terminator.
 * This makes it possible to detect 1 byte-out-of-bound reads
 * with tools like Valgrind.
 */
char *cut0(const char *s)
{
    size_t len = strlen(s);
    char *dup = tmalloc(len);
    memcpy(dup, s, len);
    return dup;
}

int nexttoken(struct CSlul *ctx)
{
    if (mhparse) return cslul_ll_next_mh_token(ctx);
    else return cslul_ll_next_slul_token(ctx);
}

struct TreeNode *lookup(struct CSlul *ctx, struct TreeNode *root,
                        const char *name)
{
    size_t len = strlen(name);
    const char *c;
    HashCode h = 0;
    for (c = name; *c; c++) {
        h = HASH(h, *c);
    }
    return tree_search(ctx, root, h, len, name);
}

struct TypeDecl *lookup_type(struct CSlul *ctx, const char *name)
{
    struct TopLevelType *tltype = (struct TopLevelType *)lookup(
            ctx, ctx->tl.types_root, name);
    if (!tltype) return NULL;
    tsoftassert(tltype->sinceversions == NULL);
    return &tltype->decl;
}

struct TypeDecl *lookup_type_ver(struct CSlul *ctx, const char *name,
                                 const char *version)
{
    struct ApiDef *ver = (struct ApiDef *)lookup(
            ctx, ctx->module.apidefs_root, version);
    struct TopLevelType *tltype = (struct TopLevelType *)lookup(
            ctx, ctx->tl.types_root, name);
    if (!tltype) return NULL;
    tassert(tltype->sinceversions != NULL);
    if (tsoftassert(ver != NULL)) {
        tsoftassert(tltype->sinceversions->next == NULL);
        tsoftassert(tltype->sinceversions->version == ver);
    }
    return &tltype->decl;
}


struct IdentDecl *lookup_toplevel(struct CSlul *ctx, const char *name)
{
    struct TopLevelIdent *tlident = (struct TopLevelIdent *)lookup(
            ctx, ctx->tl.idents_root, name);
    if (!tlident) return NULL;
    tsoftassert(tlident->sinceversions == NULL);
    return &tlident->decl;
}

struct IdentDecl *lookup_toplevel_ver(struct CSlul *ctx, const char *name,
                                     const char *version)
{
    struct ApiDef *ver = (struct ApiDef *)lookup(
            ctx, ctx->module.apidefs_root, version);
    struct TopLevelIdent *tlident = (struct TopLevelIdent *)lookup(
            ctx, ctx->tl.idents_root, name);
    if (!tlident) return NULL;
    tassert(tlident->sinceversions != NULL);
    if (tsoftassert(ver != NULL)) {
        tsoftassert(tlident->sinceversions->next == NULL);
        tsoftassert(tlident->sinceversions->version == ver);
    }
    return &tlident->decl;
}


/** Tests parsing an empty buffer */
void testcommon_empty(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = tmalloc(1);
    TEST_ONE(&buff[1], 0, 0, NEEDDATA,      1, 1, 0, MHTDone);
    TEST_ONE(&buff[1], 0, 1, CSLUL_MHT_EOF, 1, 1, 0, MHTDone);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests parsing a buffer with a single newline */
void testcommon_newline_single(void)
{
    char *buff = cut0("\n");
    struct CSlul *ctx = create_ctx(phase);
    TEST_ONE(buff, 1, 0, NEEDDATA,      2, 1, 0, MHTDone);
    TEST_ONE(buff, 1, 1, CSLUL_MHT_EOF, 3, 1, 0, MHTDone);
    free_ctx(ctx);
    ctx = create_ctx(phase);
    buff[0] = '\r';
    TEST_ONE(buff, 1, 0, NEEDDATA,      1, 2, 0, mhparse ? MHTInNewline : TInNewline);
    expect_error(CSLUL_E_CRNEWLINE, 2, 1);
    expect_error(CSLUL_E_CRNEWLINE, 1, 1); /* from the TEST_ONE above */
    TEST_ONE(buff, 1, 1, CSLUL_MHT_EOF, 2, 2, 0, mhparse ? MHTInNewline : TInNewline);
    free_ctx(ctx);
    tfree(buff);
    buff = cut0("\r\n");
    ctx = create_ctx(phase);
    TEST_ONE(buff, 2, 0, NEEDDATA,      2, 1, 0, MHTDone);
    TEST_ONE(buff, 2, 1, CSLUL_MHT_EOF, 3, 1, 0, MHTDone);
    free_ctx(ctx);
    tfree(buff);
}

/** Tests a DOS/Windows newline that spans across two buffers */
void testcommon_newline_splitted(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\r");
    TEST_ONE(buff, 1, 0, NEEDDATA, 1, 2, 0, mhparse ? MHTInNewline : TInNewline);
    buff[0] = '\n';
    TEST_ONE(buff, 1, 0, NEEDDATA, 2, 1, 0, MHTDone);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests a newline with an EOF (with 1 byte buffer at EOF) */
void testcommon_newline_eof1(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\r");
    TEST_ONE(buff, 1, 0, NEEDDATA, 1, 2, 0, mhparse ? MHTInNewline : TInNewline);
    expect_error(CSLUL_E_CRNEWLINE, 2, 1);
    expect_error(CSLUL_E_CRNEWLINE, 1, 1);
    TEST_ONE(buff, 1, 1, EOF, 2, 2, 0, mhparse ? MHTInNewline : TInNewline);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests a newline with an EOF (with 0 byte buffer at EOF) */
void testcommon_newline_eof2(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\r");
    TEST_ONE(buff, 1, 0, NEEDDATA, 1, 2, 0, mhparse ? MHTInNewline : TInNewline);
    expect_error(CSLUL_E_CRNEWLINE, 1, 1);
    TEST_ONE(&buff[1], 0, 1, EOF, 1, 2, 0, mhparse ? MHTInNewline : TInNewline);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests forbidden control character */
void testcommon_null(void)
{
    struct CSlul *ctx = create_ctx(phase);
    const char buff[1] = { 0x00 };
    TEST_ERROR(buff, 1, 0, NEEDDATA, 1, 2, 0, badchar_state,
               CSLUL_E_INVALIDCHAR, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tassert(lasterror.locs[0].length == 4);
    tsoftassert(!strncmp("\\x00", lasterror.locs[0].text, 4));
    free_ctx(ctx);
}

/** Tests two-byte UTF-8 codepoints */
void testcommon_utf8_2min(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xc2\x80");
    TEST_ERROR2(buff, 2, 0, NEEDDATA, 1, 2, 1, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_DISALLOWEDUNICODE, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests two-byte UTF-8 codepoints */
void testcommon_utf8_2max(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xdf\xbf");
    TEST_ERROR(buff, 2, 0, NEEDDATA, 1, 2, 1, badchar_state,
               CSLUL_E_INVALIDCHAR, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests a two-byte UTF-8 codepoint splitted over two buffers */
void testcommon_utf8_2split(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xc3\xa5");
    TEST_ERROR(buff, 1, 0, NEEDDATA, 1, 2, 0, badchar_state,
               CSLUL_E_INVALIDCHAR, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_BYTE2);
    /*expect_error(CSLUL_E_INVALIDCHAR, 1, 3);*/
    TEST_ONE(&buff[1], 1, 0, NEEDDATA, 1, 2, 1, badchar_state);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests overlong two-byte UTF-8 codepoints */
void testcommon_utf8_2overlong(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xc1\xbf"); /* parsed as 2 characters */
    TEST_ERROR3(buff, 2, 0, NEEDDATA, 1, 3, 0, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_BADUTF8, 1, 2);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests EOF in two-byte UTF-8 codepoints */
void testcommon_utf8_2eof(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xc2");
    TEST_ERROR3(buff, 1, 1, CSLUL_MHT_EOF, 1, 2, 0, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_NOEOFNEWLINE, 1, 2);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests bad continuation in two-byte UTF-8 codepoints */
void testcommon_utf8_2badcont(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xc2#");
    TEST_ERROR2(buff, 2, 0, NEEDDATA, 1, 3, 0,
                /* MHTInComment is used for error recovery */
                mhparse ? MHTInComment : TInMaybeMLCommentStart1,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 2);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests three-byte UTF-8 codepoints */
void testcommon_utf8_3min(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xe0\xa0\x80");
    TEST_ERROR(buff, 3, 0, NEEDDATA, 1, 2, 2, badchar_state,
               CSLUL_E_INVALIDCHAR, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests three-byte UTF-8 codepoints */
void testcommon_utf8_3max(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xef\xbf\xb0");
    TEST_ERROR(buff, 3, 0, NEEDDATA, 1, 2, 2, badchar_state,
               CSLUL_E_INVALIDCHAR, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests a three-byte UTF-8 codepoint splitted over three buffers */
void testcommon_utf8_3split(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xe2\x97\x8f");
    TEST_ERROR(buff, 1, 0, NEEDDATA, 1, 2, 0, badchar_state,
               CSLUL_E_INVALIDCHAR, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_BYTE2);
    TEST_ONE(&buff[1], 1, 0, NEEDDATA, 1, 2, 1, badchar_state);
    tsoftassert(ctx->utf8state == UTF8ST_MB3_BYTE3);
    TEST_ONE(&buff[2], 1, 0, NEEDDATA, 1, 2, 1, badchar_state);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests overlong three-byte UTF-8 codepoints */
void testcommon_utf8_3overlong(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xe0\x9f\x80"); /* parsed as 3 characters */
    TEST_ERROR4(buff, 3, 0, NEEDDATA, 1, 4, 0, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_BADUTF8, 1, 2,
                CSLUL_E_BADUTF8, 1, 3);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests EOF in three-byte UTF-8 codepoints */
void testcommon_utf8_3eof2(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xe2\x97");
    TEST_ERROR3(buff, 2, 1, CSLUL_MHT_EOF, 1, 2, 1, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_NOEOFNEWLINE, 1, 2); /* 1,3 would be more correct */
    tsoftassert(ctx->utf8state == UTF8ST_MB3_BYTE3);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests null character in a three-byte codepoint */
void testcommon_utf8_3nullchar3(void)
{
    struct CSlul *ctx = create_ctx(phase);
    static const char buff[] = "\xed\xa0"; /* + \x00 at end */
    TEST_ERROR3(buff, 3, 0, NEEDDATA, 1, 4, 0, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_INVALIDCHAR, 1, 3);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    free_ctx(ctx);
}

/** Tests four-byte UTF-8 codepoints */
void testcommon_utf8_4min(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xf0\x90\x80\x80");
    TEST_ERROR(buff, 4, 0, NEEDDATA, 1, 2, 3, badchar_state,
               CSLUL_E_INVALIDCHAR, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests four-byte UTF-8 codepoints */
void testcommon_utf8_4max(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xf4\x8f\xbf\xbf");
    TEST_ERROR2(buff, 4, 0, NEEDDATA, 1, 2, 3, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_DISALLOWEDUNICODE, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests a four-byte UTF-8 codepoint splitted over four buffers */
void testcommon_utf8_4split(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xf4\x8f\xbf\xbf");
    TEST_ERROR(buff, 1, 0, NEEDDATA, 1, 2, 0, badchar_state,
               CSLUL_E_INVALIDCHAR, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_BYTE2);
    TEST_ONE(&buff[1], 1, 0, NEEDDATA, 1, 2, 1, badchar_state);
    tsoftassert(ctx->utf8state == UTF8ST_MB4_BYTE3);
    TEST_ONE(&buff[2], 1, 0, NEEDDATA, 1, 2, 1, badchar_state);
    tsoftassert(ctx->utf8state == UTF8ST_MB4_BYTE4);
    expect_error(CSLUL_E_DISALLOWEDUNICODE, 1, 1);
    TEST_ONE(&buff[3], 1, 0, NEEDDATA, 1, 2, 1, badchar_state);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests overlong four-byte UTF-8 codepoint */
void testcommon_utf8_4overlong(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xf0\x8f\xbf\xbf"); /* parsed as 4 characters */
    TEST_ERROR5(buff, 4, 0, NEEDDATA, 1, 5, 0, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_BADUTF8, 1, 2,
                CSLUL_E_BADUTF8, 1, 3,
                CSLUL_E_BADUTF8, 1, 4);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests four-byte UTF-8 codepoint larger than maximum value */
void testcommon_utf8_4overflow(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xf4\x90\x80\x80");
    TEST_ERROR2(buff, 4, 0, NEEDDATA, 1, 2, 3, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_DISALLOWEDUNICODE, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    tfree(buff);
    free_ctx(ctx);
}

/** Tests EOF in four-byte UTF-8 codepoints after byte 2 */
void testcommon_utf8_4eof2(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xf4\x8f");
    TEST_ERROR3(buff, 2, 1, CSLUL_MHT_EOF, 1, 2, 1, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_NOEOFNEWLINE, 1, 2); /* 1,3 would be more correct */
    tfree(buff);
    free_ctx(ctx);
}

/** Tests EOF in four-byte UTF-8 codepoints after byte 3 */
void testcommon_utf8_4eof3(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("\xf4\x8f\xbf");
    TEST_ERROR3(buff, 3, 1, CSLUL_MHT_EOF, 1, 2, 2, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_NOEOFNEWLINE, 1, 2); /* 1,4 would be more correct */
    tfree(buff);
    free_ctx(ctx);
}

/** Tests null char in four-byte UTF-8 codepoints */
void testcommon_utf8_4nullchar3(void)
{
    struct CSlul *ctx = create_ctx(phase);
    static const char buff[] = "\xf4\x8f"; /* + 0x00 */
    TEST_ERROR3(buff, 3, 0, NEEDDATA, 1, 4, 0, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_INVALIDCHAR, 1, 3);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    free_ctx(ctx);
}

/** Tests null char in four-byte UTF-8 codepoints */
void testcommon_utf8_4nullchar4(void)
{
    struct CSlul *ctx = create_ctx(phase);
    static const char buff[] = "\xf4\x8f\xbf"; /* + 0x00 */
    TEST_ERROR3(buff, 4, 0, NEEDDATA, 1, 5, 0, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_BADUTF8, 1, 1,
                CSLUL_E_INVALIDCHAR, 1, 4);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    free_ctx(ctx);
}

static void goodchar(const char *s)
{
    struct CSlul *ctx = create_ctx(phase);
    int len = strlen(s);
    char *buff = cut0(s);
    TEST_ERROR(buff, len, 0, NEEDDATA, 1, 2, len-1, badchar_state,
               CSLUL_E_INVALIDCHAR, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    free_ctx(ctx);
    tfree(buff);
}

static void badchar(const char *s)
{
    struct CSlul *ctx = create_ctx(phase);
    int len = strlen(s);
    char *buff = cut0(s);
    TEST_ERROR2(buff, len, 0, NEEDDATA, 1, 2, len-1, badchar_state,
                CSLUL_E_INVALIDCHAR, 1, 1,
                CSLUL_E_DISALLOWEDUNICODE, 1, 1);
    tsoftassert(ctx->utf8state == UTF8ST_NONE);
    free_ctx(ctx);
    tfree(buff);
}

void testcommon_utf8_forbidden(void)
{
    goodchar("\x7f"); /* not UTF-8 */
    badchar("\xc2\x80"); /* 80 = control char */
    badchar("\xc2\x8f"); /* 8F = control char */
    goodchar("\xc2\xa0"); /* A0 = NBSP. Can be allowed via allowed_scripts */
    goodchar("\xc2\xa1"); /* A1 */
    goodchar("\xe2\x80\x8d"); /* 200D */
    goodchar("\xe2\x80\x90"); /* 2010 */
    goodchar("\xe2\x80\xa7"); /* 2027 */
    badchar("\xe2\x80\xa8");  /* 2028 = visible line separator */
    badchar("\xe2\x80\xa9");  /* 2029 = paragraph separator */
    badchar("\xe2\x80\xae");  /* 202E = RLO */
    goodchar("\xe2\x80\xaf"); /* 202F */
    goodchar("\xe2\x81\x9f"); /* 205F */
    badchar("\xe2\x81\xa0"); /* 2060 = word joiner */
    badchar("\xe2\x81\xaf"); /* 206F = nominal digit shapes */
    goodchar("\xe2\x81\xb0"); /* 2070 */
    goodchar("\xed\x9f\xbf"); /* D7FF */
    badchar("\xed\xa0\x80");  /* D800 = high surrogate */
    badchar("\xed\xbf\xbf");  /* DFFF = low surrogate */
    goodchar("\xee\x80\x80"); /* E000 */
    badchar("\xef\xb7\x90"); /* FDD0... = non-characters */
    badchar("\xef\xb7\xaf"); /* ...FDEF = non-characters */
    badchar("\xef\xbb\xbf"); /* FEFF = byte order mark */
    badchar("\xef\xbf\xbe"); /* FFFE = non-character */
    badchar("\xef\xbf\xbf"); /* FFFF = non-character */
    badchar("\xf4\x8f\xbf\xbe"); /* ??FFFE = non-character (10FFFE here) */
    badchar("\xf4\x8f\xbf\xbf"); /* ??FFFF = non-character (10FFFF here) */
}

void testcommon_comment(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("# ");
    TEST_ONE(buff, 2, 0, NEEDDATA, 1, 3, 0, mhparse ? MHTInComment : TInComment);
    buff[0] = '\xc3';
    buff[1] = '\xa5';
    TEST_ONE(buff, 2, 0, NEEDDATA, 1, 4, 1, mhparse ? MHTInComment : TInComment);
    tfree(buff);
    buff = cut0("\n");
    TEST_ONE(buff, 1, 0, NEEDDATA, 2, 1, 0, MHTDone);
    tfree(buff);
    free_ctx(ctx);
}

void testcommon_comment_badchar(void)
{
    struct CSlul *ctx = create_ctx(phase);
    static const char buff[] = "# \0 x \x1f y \x7f";
    TEST_ERROR3(buff, sizeof(buff)-1, 0, NEEDDATA, 1, 12, 0, mhparse ? MHTInComment : TInComment,
                CSLUL_E_INVALIDCHAR, 1, 3,
                CSLUL_E_INVALIDCHAR, 1, 7,
                CSLUL_E_INVALIDCHAR, 1, 11);
    free_ctx(ctx);
}

void testcommon_multiline_comment(void)
{
    struct CSlul *ctx = create_ctx(phase);
    char *buff = cut0("#{{\n");
    unsigned expectedstate = mhparse ?
            MHTInMaybeMLCommentEnd0 : TInMaybeMLCommentEnd0;
    TEST_ONE(buff, 4, 0, NEEDDATA, 2, 1, 0, expectedstate);
    tassert(ctx->in_multiline_comment == 1);
    TEST_ONE(buff, 4, 0, NEEDDATA, 3, 1, 0, expectedstate);
    tassert(ctx->in_multiline_comment == 2);
    memcpy(buff, "#}}", 3);
    TEST_ONE(buff, 4, 0, NEEDDATA, 4, 1, 0, expectedstate);
    tassert(ctx->in_multiline_comment == 1);
    TEST_ONE(buff, 4, 0, NEEDDATA, 5, 1, 0, mhparse ? MHTDone : TDone);
    tassert(ctx->in_multiline_comment == 0);
    tfree(buff);
    free_ctx(ctx);
}

void testcommon_multiline_comment_badstart(void)
{
    struct CSlul *ctx = create_ctx(phase);
    static const char buff[] = " #{{";
    expect_error(CSLUL_E_MLCOMMENTNOTLINESTART, 1, 2);
    if (mhparse) {
        expect_error(CSLUL_E_INDENTEDATTR, 1, 1);
    }
    TEST_ONE(buff, sizeof(buff)-1, 0, NEEDDATA, 1, 5, 0, mhparse ? MHTInComment : TInComment);
    tassert(ctx->in_multiline_comment == 0);
    free_ctx(ctx);
}