path: root/src-backend/codegen/aarch64.c

/*

  aarch64.c -- Code generator for Aarch64

  Copyright © 2022-2024 Samuel Lidén Borell <samuel@kodafritt.se>

  Permission is hereby granted, free of charge, to any person obtaining a copy
  of this software and associated documentation files (the "Software"), to deal
  in the Software without restriction, including without limitation the rights
  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  copies of the Software, and to permit persons to whom the Software is
  furnished to do so, subject to the following conditions:

  The above copyright notice and this permission notice shall be included in
  all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  THE SOFTWARE.

*/

#include "codegen_undef.h"
#include "codegen_common.h"
#include "../outformat/outformat_common.h"

#include <assert.h>
#include <string.h>

#define STACKALIGN 16
#define LINKREG_BYTES 8
#define PREALLOC_INSN IPREALLOC(8)

static unsigned assign_reg_by_lane(const struct FuncIR *f, unsigned lane);
static int emit_load_insn(struct CgCtx *cg, unsigned reg,
                          unsigned basereg, uint32 offset,
                          unsigned loadtype);
static int emit_store_insn(struct CgCtx *cg, unsigned reg,
                           unsigned basereg, uint32 offset,
                           unsigned storetype);


static int aarch64_has_endianness(enum CsbeEndianness endianness)
{
    return endianness == CSBEEN_LITTLE_ENDIAN ||
           endianness == CSBEEN_BIG_ENDIAN;
}

static int aarch64_has_feature_state(enum CsbeCpu cpu, int feature, int state)
{
    (void)cpu;
    (void)state;
    switch (feature) {
    case CSBEF_COMMON_HARDFLOAT:
        return 1; /* allow FP to be disabled for e.g. kernel-mode software */
    }
    return 0;
}

#define RS_INTEGER 0 /* registers R0-30 (also known as Wn/Xn) */
#define RS_FLOAT 1
#define NUM_REGSETS 2
static const unsigned char aarch64_regset[NUM_TYPEKINDS] = {
    /* CSBET_BOOL = */ RS_INTEGER,
    /* CSBET_INT = */ RS_INTEGER,
    /* CSBET_UINT = */ RS_INTEGER,
    /* CSBET_LONG = */ RS_INTEGER,
    /* CSBET_ULONG = */ RS_INTEGER,
    /* CSBET_I8 = */ RS_INTEGER,
    /* CSBET_U8 = */ RS_INTEGER,
    /* CSBET_I16 = */ RS_INTEGER,
    /* CSBET_U16 = */ RS_INTEGER,
    /* CSBET_I32 = */ RS_INTEGER,
    /* CSBET_U32 = */ RS_INTEGER,
    /* CSBET_I64 = */ RS_INTEGER,
    /* CSBET_U64 = */ RS_INTEGER,
    /* CSBET_F32 = */ RS_FLOAT,
    /* CSBET_F64 = */ RS_FLOAT,
    /* CSBET_SSIZE = */ RS_INTEGER,
    /* CSBET_USIZE = */ RS_INTEGER,
    /* CSBET_DPTR = */ RS_INTEGER,
    /* CSBET_DPTR_ALIASED = */ RS_INTEGER,
    /* CSBET_DPTR_THREADED = */ RS_INTEGER,
    /* CSBET_FPTR = */ RS_INTEGER
};

#define SIMPLE          0
#define SIMPLE_INPUTS   1
#define COMPLEX         2

#define OPTY_IMM  0x0
#define OPTY_REG  0x1
#define OPTY_W    0x0
#define OPTY_X    0x2
struct A64OpInfo {
    unsigned complexity : 2; /**< Not a simple rd=ra[+rb][+imm] style op.
                              is_complex=2: inputs are simple but op is not */
    unsigned is_unop    : 1; /**< Unary operation */
    unsigned is_binop   : 1; /**< Binary operation */
    unsigned is_trap    : 1; /**< Can trap */
    unsigned is_regonly : 1; /**< Cannot use immediate */
    uint32 code[4]; /**< Opcodes (imm w, reg w, imm x, reg x) */
};

#define s SIMPLE
#define C COMPLEX /**< Not a simple rd=ra[+rb][+imm] style op */
#define I SIMPLE_INPUTS /**< Simple inputs, but operation is complex */
#define u 0
#define b 0
#define t 0
#define r 0
#define U 1 /**< Unary operation */
#define B 1 /**< Binary operation */
#define T 1 /**< Can trap */
#define R 1 /**< Register-only (can't take immediate value) */
static const struct A64OpInfo a64_opinfos[CSBEO_LAST] = {
    /* CSBEO_NOP */         { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_JUMP */        { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_CONDJUMP */    { C, U, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_COMPAREJUMP */ { C, u, B, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_TRAP */        { C, u, b, T, r, { 0, 0, 0, 0 } },
    /* CSBEO_CONDTRAP */    { C, U, b, T, r, { 0, 0, 0, 0 } },
    /* CSBEO_RETURN_VOID */ { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_RETURN_ARG */  { C, U, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_CALL_START */  { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_CALL_ARG */    { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_CALL_FUNCDEF */{ C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_CALL_FUNCVAR */{ C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_CALL_GET_RETURN */{ C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_CALL_DISCARD_RETURN */{ C, u, b, t, r, { 0, 0, 0, 0 } },
    /* XXX how to handle large moves? or require a CSBEO_COPY in that case? */
    /* CSBEO_MOVE */        { s, U, b, t, r, { 0x52800000, 0x2A0003E0, 0xD2800000, 0xAA0003E0 } }, /* MOVZ. ORR with rm=ZR. Large immeds are handled in load_operand() */
    /* CSBEO_MOVE_TRAP */   { s, U, b, T, r, { 0x52800000, 0x2A0003E0, 0xD2800000, 0xAA0003E0 } },
    /* CSBEO_NEG */         { s, U, b, t, R, { 0, 0x4B000000, 0, 0xCB000000 } }, /* SUB ZR, r */
    /* CSBEO_NEG_TRAP */    { s, U, b, T, R, { 0, 0x6B000000, 0, 0xEB000000 } }, /* SUBS ZR, r */
    /* CSBEO_ADD */         { s, u, B, t, r, { 0x11000000, 0x0B000000, 0x91000000, 0x8B000000 } }, /* ADD */
    /* CSBEO_ADD_TRAP */    { s, u, B, T, r, { 0x31000000, 0x2B000000, 0xB1000000, 0xAB000000 } }, /* ADDS */
    /* CSBEO_SUB */         { s, u, B, t, r, { 0x51000000, 0x4B000000, 0xD1000000, 0xCB000000 } }, /* SUB */
    /* CSBEO_SUB_TRAP */    { s, u, B, T, r, { 0x71000000, 0x6B000000, 0xF1000000, 0xEB000000 } }, /* SUBS */
    /* CSBEO_MUL */         { s, u, B, t, R, { 0, 0x1B007C00, 0, 0x9B007C00 } }, /* MADD with ZR. TODO optimize small multiplications */
    /* CSBEO_MUL_TRAP */    { s, u, B, T, R, { 0, 0x1B007C00, 0, 0x9B007C00 } }, /* TODO detect overflow */
    /* CSBEO_DIV */         { s, u, B, t, R, { 0, 0, 0, 0 } }, /* FIXME sdiv and udiv */
    /* CSBEO_DIV_TRAP */    { s, u, B, T, R, { 0, 0, 0, 0 } },
    /* CSBEO_MOD */         { I, u, B, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_MOD_TRAP */    { I, u, B, T, r, { 0, 0, 0, 0 } },
    /* XXX bitwise ops can trap during type conversions!
           (should type conversions trap at all?) */
    /* CSBEO_BAND */        { s, u, B, t, r, { 0, 0x0A000000, 0, 0x8A000000 } }, /* XXX immr and imms? / AND */
    /* CSBEO_BOR */         { s, u, B, t, r, { 0, 0x2A000000, 0, 0xAA000000 } }, /* TODO/ORR */
    /* CSBEO_BXOR */        { s, u, B, t, r, { 0, 0x4A000000, 0, 0xCA000000 } }, /* TODO/EOR */
    /* CSBEO_BNOT */        { s, U, b, t, R, { 0, 0, 0, 0 } }, /* TODO  =xor with 0xFFFF... */
    /* CSBEO_SHL */         { s, u, B, t, r, { 0, 0x1AC02000, 0, 0x9AC02000 } }, /* TODO immh and immb? / LSLV  */
    /* CSBEO_SHR */         { s, u, B, t, r, { 0, 0x1AC02400, 0, 0x9AC02400 } }, /* TODO/LSRV */
    /* CSBEO_SHRA */        { s, u, B, t, r, { 0, 0x1AC02800, 0, 0x9AC02800 } }, /* TODO/ASRV */
    /* CSBEO_LAND */        { I, u, B, t, R, { 0, 0x0A000000, 0, 0x8A000000 } },
    /* CSBEO_LOR */         { I, u, B, t, R, { 0, 0x2A000000, 0, 0xAA000000 } },
    /* CSBEO_LXOR */        { I, u, B, t, R, { 0, 0x4A000000, 0, 0xCA000000 } },
    /* CSBEO_LNOT */        { I, U, b, t, R, { 0, 0, 0, 0 } },
    /* CSBEO_EQ */          { s, u, B, t, r, { 0x51000000, 0x4B000000, 0xD1000000, 0xCB000000 } }, /* SUB (+CSET_INV NE) */
    /* CSBEO_NEQ */         { s, u, B, t, r, { 0x51000000, 0x4B000000, 0xD1000000, 0xCB000000 } }, /* SUB (+CSET_INV EQ) */
    /* CSBEO_LT */          { I, u, B, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_GT */          { I, u, B, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_LE */          { I, u, B, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_GE */          { I, u, B, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_DISCARD */     { C, U, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_CHOICE */      { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_SIZEOF */      { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_COPY */        { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_CLEAR */       { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_MOVETOPTR */   { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_DEREF */       { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_ADDRELEM */    { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_LOADELEM */    { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_ADDRLOCAL */   { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_ADDRGLOBAL */  { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_LOADGLOBAL */  { C, u, b, t, r, { 0, 0, 0, 0 } },
    /* CSBEO_ADDRFUNC */    { C, u, b, t, r, { 0, 0, 0, 0 } }
};
#undef s
#undef c
#undef u
#undef b
#undef t
#undef r
#undef C
#undef I
#undef U
#undef B
#undef T
#undef R

static const uint32 aarch64_elf_reloc_types[CG_NUM_RELOC_KINDS] = {
    /* CG_RK_FUNCCALL = R_AARCH64_CALL26 = */283,
    /* CG_RK_FUNCADDR = R_AARCH64_ TODO = */0,
    /* CG_RK_DATAADDR = R_AARCH64_ TODO = */0,
    /* CG_RK_DATALOAD = R_AARCH64_ TODO = */0,
    /* CG_RK_GOTPC = not used on aarch64 */0
};

static const unsigned short aarch64_typesizes[NUM_TYPEKINDS] = {
    /* CSBET_BOOL  */   1,
    /* CSBET_INT   */   4,
    /* CSBET_UINT  */   4,
    /* CSBET_LONG  */   8,
    /* CSBET_ULONG */   8,
    /* CSBET_I8    */   1,
    /* CSBET_U8    */   1,
    /* CSBET_I16   */   2,
    /* CSBET_U16   */   2,
    /* CSBET_I32   */   4,
    /* CSBET_U32   */   4,
    /* CSBET_I64   */   8,
    /* CSBET_U64   */   8,
    /* CSBET_F32   */   4,
    /* CSBET_F64   */   8,
    /* CSBET_SSIZE */   8,
    /* CSBET_USIZE */   8,
    /* CSBET_DPTR  */   8,
    /* CSBET_DPTR_ALIASED */ 8,
    /* CSBET_DPTR_THREADED */ 8,
    /* CSBET_FPTR  */ 8
};

static enum CsbeErr aarch64_initialize(struct CsbeConfig *cfg)
{
    struct ArchInfo *arch;

    csbe_config_arch_feature(cfg, CSBEF_COMMON_HARDFLOAT, 1);
    arch = &cfg->last_arch->archinfo;
    arch->wordbits = 64;
    arch->dptrbits = 64;
    arch->fptrbits = 64;
    arch->elf_relocs_have_addends = 1;
    arch->reg_size = 8;
    arch->tempreg_count = 7;   /*  x9-x15 */
    arch->savedreg_count = 11; /* x18-x28 */
    arch->paramreg_count = 8;  /*  x0- x7 */
    arch->elf_reloc_types = aarch64_elf_reloc_types;
    arch->is_textual = 0;
    memcpy(arch->regset, aarch64_regset, sizeof(arch->regset));
    arch->num_regsets = NUM_REGSETS;
    memcpy(arch->types_sizes, aarch64_typesizes, sizeof(arch->types_sizes));
    /* alignment==size for all types */
    memcpy(arch->types_align, aarch64_typesizes, sizeof(arch->types_align));
    return CSBEE_OK;
}


/* Registers (usually called w0,w1,.../x0,x1,...) */
#define R0   0 /**< First function parameter/return register */
#define R1   1
#define R2   2
#define R3   3
#define R4   4
#define R5   5
#define R6   6
#define R7   7 /**< Last function parameter/return register */
#define R8   8 /**< Indirect result */
#define R9   9 /**< First ordinary temporary/caller-saved register */
#define R16 16 /**< Intra-call temp 1 */
#define R17 17 /**< Intra-call temp 2 */
#define R18 18 /**< First callee-saved register */
#define R28 28 /**< Last callee-saved regiser */
#define FP  29 /**< Frame pointer register */
#define LR  30 /**< Link Register (holds the return address) */
#define SP  31 /**< Stack Pointer */
#define ZR  31 /**< Zero Register */

/* x8 is used for intermediate results (e.g. when loading a second operand,
   when the destination reg might already be in use) */
#define TEMPREG1 R8
/* x16-17 are linker/intra-call temporaries, but here they are also be used
   for intermediate results. NOTE: Indirect calls should not use x16-x17! */
#define TEMPREG2 R16
#define TEMPREG3 R17

/* Common instruction formats.
        rd = destination reg
        rn,rm,ra = inputs
        imm = immediate
        opc* = opcode bytes */
#define IEMIT_IMM12(rd,rn,imm,opc2,opc1) \
    IEMIT4((rd)|((rn)&0x7U)<<5, (rn)>>3|((imm)&0x3FU)<<2, ((imm)&0xFFFU)>>6|(opc2), (opc1))
#define IEMIT_IMM16(rd,imm,opc2,opc1) \
    IEMIT4((rd)|((imm)&0x7U)<<5, ((imm)&0x7F8U)>>3, ((imm)&0xF800U)>>11|(opc2), (opc1))
#define IEMIT_IMM26(imm,opc) \
    IEMIT4((imm)&0xFF, ((imm)&0xFF00U)>>8, ((imm)&0xFF0000U)>>16, ((imm)&0x3000000U)>>24|(opc))
#define IEMIT_IMM19(addr,extra,opc) \
    IEMIT4((extra)|((addr)&0x7U)<<5, ((addr)&0x7F8U)>>3, ((addr)&0x7F800U)>>11, (opc))
#define IEMIT_RRR(rd,rn,ra,opc3,rm,opc2,opc1) \
    IEMIT4((rd)|((rn)&0x7U)<<5, (rn)>>3|(ra)<<2|(opc3), (rm)|(opc2), (opc1))
#define IEMIT_RR(rd,rn,opc3,rm,opc2,opc1) \
    IEMIT4((rd)|((rn)&0x7U)<<5, (rn)>>3|(opc3), (rm)|(opc2), (opc1))
#define IEMIT_RR_FLAGS(rd,rn,flags,rm,opc2,opc1) \
    IEMIT4((rd)|((rn)&0x7U)<<5, (rn)>>3|(flags)<<2, (rm)|(opc2), (opc1))
#define IEMIT_R(rd,rn,opc3,opc2,opc1) \
    IEMIT4((rd)|((rn)&0x7U)<<5, (opc3), (opc2), (opc1))

#define IMM12_MAX ((1U<<12U) - 1)

#define EQ  0x0
#define NE  0x1
#define HS  0x2
#define LO  0x3
#define HI  0x8
#define LS  0x9
#define GE  0xA
#define LT  0xB
#define GT  0xC
#define LE  0xD

static const unsigned char branchtype2cond[] = {
    /* CSBEBT_Z */   EQ,
    /* CSBEBT_NZ */  NE,
    /* FIXME these are for signed numbers. there are cond-codes for unsigned also! (see ARM v8 i.s.o. page 16) */
    /* CSBEBT_LZ */  LT,
    /* CSBEBT_GEZ */ GE,
    /* CSBEBT_GZ */  GT,
    /* CSBEBT_LEZ */ LE
};
#define INVERSE_COND(c) ((c) ^ 1)

#define IS_32 0
#define IS_64 1
/*#define SHIFTFUNC_LSL 0
#define SHIFTFUNC_LSR 1
#define SHIFTFUNC_ASR 2
#define SHIFTFUNC_ROR 3*/

#define MOV_X(rd,rn)      IEMIT_RR(rd,rn,0x00,ZR,0x00,0xAA)
#define MOV_R_XSP(rd)     ADD_IMM_X(rd,SP,0)
#define MOV_IMM16(rd,imm,is64) \
    IEMIT_IMM16(rd,imm,0x80,(is64)?0xd2:0x52)
#define MOVN_IMM16(rd,imm,is64) IEMIT_IMM16(rd,imm,0x80,(is64)?0x92:0x12)
#define MOVK_IMM16(rd,imm,is64,scale) \
    IEMIT_IMM16(rd,imm,0x80|((scale)<<5),(is64)?0xF2:0x72)
#define MOVZ_IMM16(rd,imm,is64,scale) \
    IEMIT_IMM16(rd,imm,0x80|((scale)<<5),(is64)?0xD2:0x52)
/*#define ORR(rd,rn,sign,imm,is64) \
    IEMIT_IMM_R(rd,rn, XXX imm,XXX imm,(is64)?0xB2:0x32)*/
/*#define ORR_R(rd,rn,rm,shiftfunc,amount,is64) \
    IEMIT_RR(rd,rn,(amount)<<2,rm,(shiftfunc)<<6,0x2a|(is64)<<7)*/
#define ADD_IMM_X(rd,rn,imm) IEMIT_IMM12(rd,rn,imm,0x00,0x91)
#define SUB_IMM_X(rd,rn,imm) IEMIT_IMM12(rd,rn,imm,0x00,0xD1)
#define ADD_RR(rd,rn,rm,is64) IEMIT_RR(rd,rn,0x00,rm,0x00,0x0b|(is64)<<7)
/*#define SUB_RR(rd,rn,rm,is64) IEMIT_RR(rd,rn,0x00,rm,0x00,0x51|(is64)<<7)*/ /* FIXME shouldn't this be 0x4b? */
#define SUBS_RR(rd,rn,rm,is64) IEMIT_RR(rd,rn,0x00,rm,0x00,0x6b|(is64)<<7)
#define ADDS_IMM12(rd,rn,imm,is64) IEMIT_IMM12(rd,rn,imm,0x00,0x31|(is64)<<7)
#define SUBS_IMM12(rd,rn,imm,is64) IEMIT_IMM12(rd,rn,imm,0x00,0x71|(is64)<<7)
#define MADD(rd,rn,rm,ra,is64) IEMIT_RRR(rd,rn,ra,0,rm,0x0,0x1b|(is64)<<7)
#define STR_IMM_X(rt,rn,imm) IEMIT_IMM12(rt,rn,imm,0x00,0xF9)
#define LDR_IMM_X(rt,rn,imm) IEMIT_IMM12(rt,rn,imm,0x40,0xF9)
#define LDR_IMM_BY_TYPE(rt,rn,imm,loadtype) \
    IEMIT_IMM12(rt,rn,imm,((loadtype)&0xf)<<4, (loadtype)>>4)
#define LOADTYPE_B      0x394
#define LOADTYPE_B_S32  0x39c
#define LOADTYPE_B_S64  0x398
#define LOADTYPE_H      0x794
#define LOADTYPE_H_S32  0x79c
#define LOADTYPE_H_S64  0x798
#define LOADTYPE_W      0xb94
#define LOADTYPE_W_S64  0xb98
#define LOADTYPE_X      0xf94
#define STR_IMM_BY_TYPE(rt,rn,imm,loadtype) \
    IEMIT_IMM12(rt,rn,imm,0x00,(loadtype)&0xff)
#define STORETYPE_B      0x39
#define STORETYPE_H      0x79
#define STORETYPE_W      0xb9
#define STORETYPE_X      0xf9
#define STR_X(rt,rn) STR_IMM_X(rt,rn,0)
#define LDR_X(rt,rn) LDR_IMM_X(rt,rn,0)
/* rt = value(in/out), rn=base address, offs=offset */
#define STR_PRE_X(rt,rn,offs) \
    IEMIT4((rt)|((rn)&0x7U)<<5, (rn)>>3|0x0C|((offs)&0x7U)<<5, ((offs)&0x1FFU)>>4, 0xF8)
#define LDR_POST_X(rt,rn,offs) \
    IEMIT4((rt)|((rn)&0x7U)<<5, (rn)>>3|0x04|((offs)&0x7U)<<5, ((offs)&0x1FFU)>>4|0x40, 0xF8)
#define ADRP(rd,imm) IEMIT_IMM19((imm)>>2,(rd),0x90|(((imm)&3U)<<5))
/* RET = RET lr = 0xC0, 0x03, 0x5F, 0xD6 */
#define RET(rn)             IEMIT4(((rn)&0x7U)<<5, (rn)>>3, 0x5F, 0xD6)
#define BR(rn)              IEMIT4(((rn)&0x7U)<<5, (rn)>>3, 0x1F, 0xD6)
#define B(addr)             IEMIT_IMM26((addr)>>2, 0x14)
#define BL(addr)            IEMIT_IMM26((addr)>>2, 0x94)
#define CB(rn,addr,is64,nz) IEMIT_IMM19((addr)>>2, (rn), 0x34|(nz)|((is64)<<7))
#define CBZ(rn,addr,is64)   CB((rn),(addr),(is64),0)
#define CBNZ(rn,addr,is64)  CB((rn),(addr),(is64),1)
#define B_COND(cond,addr)   IEMIT_IMM19((addr)>>2, (cond), 0x54)
#define CMP_IMM12(rn,imm,is64) \
    SUBS_IMM12(ZR, (rn), (imm), (is64))
#define CMP_RR(rn,rm,is64) \
    SUBS_RR(ZR, (rn), (rm), (is64))
#define CSINC(rd,rn,rm,cond,is64) \
    IEMIT_RR_FLAGS((rd),(rn),(cond)<<2|0x1,(rm), 0x80, 0x1A|(is64)<<7)
#define CSET_INV(rd,cond,is64) \
    CSINC((rd), ZR, ZR, (cond), (is64))

#define ADDRSLOT_IMM26 1
#define ADDRSLOT_BCOND_IMM19 2

/** Fills in address offsets in forward jumps */
static void aarch64_process_addrslots(struct CgCtx *cg, struct CgAddrSlot *first)
{
    struct CgAddrSlot *as;
    uint32 pos = cg_get_position(cg);
    for (as = first; as; as = as->next) {
        uint32 delta = (pos>>2) - (as->location>>2);
        switch (as->addrtype) {
        case ADDRSLOT_IMM26:
            assert(delta <= 0x3ffffffU*4 ||
                   delta >= (uint32)(-(int32)0x4000000*4));
            cg_edit_bytes_le(cg, as, 0, delta&0x03FFFFFFU, 4);
            break;
        case ADDRSLOT_BCOND_IMM19:
            assert(delta <= 0x7ffffU*4 ||
                   delta >= (uint32)(-(int32)0x80000U*4));
            cg_edit_bytes_le(cg, as, 0, (delta&0x7FFFFU)<<5, 3);
            break;
        default:
            assert(0);
        }
    }
}

enum SaveRestoreMode {
    SAVE,
    RESTORE
};

/**
 * Saves/restores the registers { from_reg ..< to_reg }.
 * stack_index is used to avoid clobbering already saved regs.
 */
static int saverestore_regs(struct CgCtx *cg,
                            enum SaveRestoreMode mode,
                            unsigned from_reg, unsigned to_reg,
                            unsigned stack_index)
{
    unsigned stackoffs = (cg->regsave_stackoffset+stack_index) / 8; /*scaled*/
    unsigned reg = from_reg;
    while (reg < to_reg) {
        /* TODO save/restore float regs */
        /* TODO use stp/ldp (store pair / load pair) */
        assert(stackoffs > 0);
        stackoffs--;
        assert(8*stackoffs >= cg->regsave_stackend);
        if (mode == SAVE) {
            ZCHK(emit_store_insn(cg, reg, SP, stackoffs, STORETYPE_X));
        } else {
            ZCHK(emit_load_insn(cg, reg, SP, stackoffs, LOADTYPE_X));
        }
        reg++;
    }
    return 1;
}

/** Saves our incoming function params when calling another function */
static int saverestore_param_regs(struct CgCtx *cg, const struct FuncIR *f,
                                  enum SaveRestoreMode mode)
{
    unsigned num_funcparams = f->funcdef->num_params;
    return saverestore_regs(cg, mode, R0, R0 + MIN(num_funcparams,8), 0);
}

/** Saves existing values of callee-saved regs in the prologue */
static int saverestore_calleesaved(struct CgCtx *cg, const struct FuncIR *f,
                                   enum SaveRestoreMode mode)
{
    unsigned offset = (f->contains_likely_calls ?
                        MIN(f->funcdef->num_params, 8) : 0);
    return saverestore_regs(cg, mode, R18, R18 + cg->alloced_calleesaved, offset);
}

static int aarch64_prologue(struct CgCtx *cg, const struct FuncIR *f)
{
    /* TODO code alignment */
    assert(cg->func_stacksize < 4096); /* TODO above this size, each page has to be touched */
    /* Save link register to stack (if needed), and adjust stack */
    if (!f->contains_likely_calls || f->likely_no_return) {
        ZCHK(cg_stack_alloc_vars(cg, f));
        /* XXX LR might need to be saved later even if cg->func_stacksize == 0! */
        if (cg->func_stacksize) {
            cg_align_stacksize(cg, 0, STACKALIGN);
            SUB_IMM_X(SP, SP, cg->func_stacksize);
        }
    } else {
        cg_align_stacksize(cg, LINKREG_BYTES, STACKALIGN);
        ZCHK(cg_stack_alloc_vars(cg, f));
        cg_align_stacksize(cg, 0, STACKALIGN);
        if (cg->func_stacksize <= 255) { /* 9 bits, including 1 sign bit */
            STR_PRE_X(LR, SP, -cg->func_stacksize); /* str x30, [sp, #-SIZE]! */
        } else {
            SUB_IMM_X(SP, SP, cg->func_stacksize); /* sub sp, sp, #SIZE */
            STR_X(LR, SP);                         /* str x30, [sp] */
        }
    }
    saverestore_calleesaved(cg, f, SAVE);
    return 1;
  oom:
    return 0;
}

static int aarch64_epilogue(struct CgCtx *cg, const struct FuncIR *f)
{
    aarch64_process_addrslots(cg, cg->jumps_to_epilogue);
    saverestore_calleesaved(cg, f, RESTORE);
    if (!f->contains_likely_calls || f->likely_no_return) {
        if (cg->func_stacksize) {
            ADD_IMM_X(SP, SP, cg->func_stacksize);
        }
    } else if (cg->func_stacksize <= 255) { /* 9 bits, including 1 sign bit */
        LDR_POST_X(LR, SP, cg->func_stacksize); /* ldr x30, [sp], #SIZE */
    } else {
        LDR_X(LR, SP);                         /* ldr x30, [sp] */
        ADD_IMM_X(SP, SP, cg->func_stacksize); /* add sp, sp, #SIZE */
    }
    RET(LR);    /* ret */
    return 1;
  oom:
    return 0;
}

/** Controls the expected state after a load operation */
enum LoadMode {
    /** The value can be in either a register or immediate value. */
    REG_OR_IMM,
    /** The value MUST be loaded to a register. */
    REG_OR_IMM_TO_REG
};

#define NO_REG ((unsigned)-1)

enum A64Type {
    T_REG_W = 0, /* used for "is64" parameters */
    T_REG_X = 1, /* used for "is64" parameters */
    T_FLOATREG,
    T_LARGE
};

#define IS_REG_TYPE(type) ((type) == T_REG_W || (type) == T_REG_X)

static enum A64Type determine_type_simple(const enum CsbeTypeKind simple_type)
{
    switch (simple_type) {
    case CSBET_BOOL:
    case CSBET_INT:
    case CSBET_UINT:
    case CSBET_I8:
    case CSBET_U8:
    case CSBET_I16:
    case CSBET_U16:
    case CSBET_I32:
    case CSBET_U32:
        return T_REG_W;
    case CSBET_LONG:
    case CSBET_ULONG:
    case CSBET_I64:
    case CSBET_U64:
    case CSBET_SSIZE:
    case CSBET_USIZE:
    case CSBET_DPTR:
    case CSBET_DPTR_THREADED:
    case CSBET_DPTR_ALIASED:
    case CSBET_FPTR:
        return T_REG_X;
    case CSBET_F32:
    case CSBET_F64:
        return T_FLOATREG;
    default:
        assert(0);
        return T_REG_X;
    }
}

static enum A64Type determine_type(struct CgCtx *cg,
                                   const struct CsbeType *type)
{
    (void)cg;
    if (!type->is_struct && !type->is_array) {
        return determine_type_simple(type->simple_kind);
    } else {
        /* TODO small structs/arrays should be placed in registers
                (and must be, in case of function parameters) */
        return T_LARGE;
    }
}

static enum A64Type determine_var_type(struct CgCtx *cg,
                                       const struct FuncIR *f,
                                       unsigned var_id)
{
    assert(var_id < f->num_vars);
    return determine_type(cg, &f->vars[var_id].type);
}

static unsigned assign_reg(struct CgCtx *cg,
                           const struct FuncIR *f,
                           unsigned var_id)
{
    unsigned lane = f->vars[var_id].lane;
    if (cg->vars[var_id].has_stack_space) return NO_REG;
    return assign_reg_by_lane(f, lane);
}

static unsigned assign_reg_by_lane(const struct FuncIR *f, unsigned lane)
{
    /* TODO how are 128-bit structs passed by value? */
    /* TODO what should this return for large types such as structs and arrays?
            - NO_REG or a reg to use as a pointer?
            - small structs/arrays can should be stored in registers in many ABIs */
    /* XXX if large variables and word-sized variables are allocated the same varlane,
           then the large variables will block a register from being used! */
    if (lane >= CSBE_INTERN_VARLANE_FIRSTPARAM) {
        /* Function parameter. These use regs x0-x7 */
        /* TODO consider using callee-saved regs to store function params */
        lane -= CSBE_INTERN_VARLANE_FIRSTPARAM;
        assert(lane < f->funcdef->num_params);
        if (lane <= 7) return lane;
        else return NO_REG;
    }

    if (lane < MIN(f->first_callee_saved_varlane,7)) {
        /* Variables that aren't saved across calls use regs x9-x15 */
        return R9 + lane-1;
    }
    lane -= MIN(f->first_callee_saved_varlane-1,7);
    if (lane <= 11) {
        /* Regs x18-x28 are for caller-saved variables */
        return R18 + lane-1;
    } else {
        /* Fall back to stack */
        return NO_REG;
    }
}

/**
 * Determines which register to use for the result, or whether the
 * temporary register should be used (e.g. for storing to memory)
 */
static int prepare_destreg(struct CgCtx *cg,
                           const struct FuncIR *f,
                           const struct A64OpInfo *opinfo,
                           const struct Op *op, unsigned opnd_index,
                           enum A64Type type, unsigned *destreg_out)
{
    unsigned var_id = op->opnd[opnd_index+1];
    unsigned reg;
    assert(var_id < f->num_vars);

    /* TODO handling of floating point */
    (void)cg;
    (void)opinfo;
    (void)type;

    reg = assign_reg(cg, f, var_id);
    assert(reg != TEMPREG1);
    if (reg == NO_REG) {
        /* Store to stack in store_result() */
        reg = TEMPREG1;
    }

    *destreg_out = reg;
    return 1;
}

/** Emits a load operations with a given loadtype
    (byte, signed byte, word...)  */
static int emit_load_insn(struct CgCtx *cg, unsigned reg,
                          unsigned basereg, uint32 offset,
                          unsigned loadtype)
{
    assert(offset <= IMM12_MAX); /* TODO handle oversized offset */
    LDR_IMM_BY_TYPE(reg, basereg, offset, loadtype);
    return 1;
  oom:
    return 0;
}

static int emit_store_insn(struct CgCtx *cg, unsigned reg,
                           unsigned basereg, uint32 offset,
                           unsigned storetype)
{
    assert(offset <= IMM12_MAX); /* TODO handle oversized offset */
    STR_IMM_BY_TYPE(reg, basereg, offset, storetype);
    return 1;
  oom:
    return 0;
}

/**
 * Emits a load operation.
 *
 * \param cg        Code generator context.
 * \param reg       Result register
 * \param regtype   Register type (w or x)
 * \param basereg   Pointer base register (SP for stack load)
 * \param offset    Offset relative to base register
 * \param type      Type of the variable/field in memory.
 *
 * \return  1 if successful, 0 on out-of-memory error.
 */
static int emit_load(struct CgCtx *cg, unsigned reg, enum A64Type regtype,
                     unsigned basereg, uint32 offset, enum CsbeTypeKind type)
{
    unsigned loadtype = 0;
    const int is64 = (regtype == T_REG_X);
    /* Determine size and sign-extension */
    switch (type) {
    case CSBET_U8:
    case CSBET_BOOL:
        loadtype = LOADTYPE_B;
        break;
    case CSBET_I8:
        loadtype = is64 ? LOADTYPE_B_S64 : LOADTYPE_B_S32;
        break;
    case CSBET_U16:
        offset /= 2;
        loadtype = LOADTYPE_H;
        break;
    case CSBET_I16:
        offset /= 2;
        loadtype = is64 ? LOADTYPE_H_S64 : LOADTYPE_H_S32;
        break;
    case CSBET_U32:
    case CSBET_UINT:
        offset /= 4;
        loadtype = LOADTYPE_W;
        break;
    case CSBET_I32:
    case CSBET_INT:
        offset /= 4;
        /* no sign-extension needed for 32-bit types */
        loadtype = is64 ? LOADTYPE_W_S64 : LOADTYPE_W;
        break;
    case CSBET_U64:
    case CSBET_ULONG:
    case CSBET_USIZE:
    case CSBET_DPTR:
    case CSBET_DPTR_ALIASED:
    case CSBET_DPTR_THREADED:
    case CSBET_FPTR:
        offset /= 8;
        loadtype = LOADTYPE_X;
        break;
    case CSBET_I64:
    case CSBET_LONG:
    case CSBET_SSIZE:
        if (regtype == T_REG_X) {
            offset /= 8;
            /* no sign-extension needed for 64-bit types */
            loadtype = LOADTYPE_X;
        } else if (regtype == T_REG_W) {
            offset /= 4;
            loadtype = LOADTYPE_W;
        } else assert(0);
        break;
    case CSBET_F32:
        /* TODO */
        assert(0);
        break;
    case CSBET_F64:
        /* TODO */
        assert(0);
        break;
    }
    assert(loadtype);
    return emit_load_insn(cg, reg, basereg, offset, loadtype);
}

static int emit_store(struct CgCtx *cg, unsigned reg, enum A64Type regtype,
                      unsigned basereg, uint32 offset, enum CsbeTypeKind type)
{
    unsigned storetype = 0;

    (void)regtype; /* TODO probably needed for floating point */
    switch (type) {
    case CSBET_U8:
    case CSBET_I8:
    case CSBET_BOOL:
        storetype = STORETYPE_B;
        break;
    case CSBET_U16:
    case CSBET_I16:
        offset /= 2;
        storetype = STORETYPE_H;
        break;
    case CSBET_U32:
    case CSBET_UINT:
    case CSBET_I32:
    case CSBET_INT:
        offset /= 4;
        storetype = STORETYPE_W;
        break;
    case CSBET_U64:
    case CSBET_ULONG:
    case CSBET_USIZE:
    case CSBET_DPTR:
    case CSBET_DPTR_ALIASED:
    case CSBET_DPTR_THREADED:
    case CSBET_FPTR:
    case CSBET_I64:
    case CSBET_LONG:
    case CSBET_SSIZE:
        offset /= 8;
        storetype = STORETYPE_X;
        break;
    case CSBET_F32:
        /* TODO */
        assert(0);
        break;
    case CSBET_F64:
        /* TODO */
        assert(0);
        break;
    }
    assert(storetype);
    return emit_store_insn(cg, reg, basereg, offset, storetype);
}

/** Emits a move instruction between two registers */
static int emit_rr_mov(struct CgCtx *cg, unsigned destreg, unsigned sourcereg,
                   enum A64Type regtype)
{
    const struct A64OpInfo *mov = &a64_opinfos[CSBEO_MOVE];
    uint32 opcode = mov->code[OPTY_REG|regtype<<1];
    IEMIT_RR(destreg, ZR, opcode>>8, sourcereg, opcode>>16, opcode>>24);
    return 1;
  oom:
    return 0;
}

/** Emits a load of an immediate to a register */
static int emit_load_imm(struct CgCtx *cg, unsigned reg,
                         enum A64Type regtype, uint64 imm)
{
    int is64 = (regtype==T_REG_X);
    if (imm <= 65535L) {
        MOV_IMM16(reg, imm, is64);
    } else if ((imm & 0xFFFF) == 0 && (!is64 || imm <= 4294967295U)) {
        /* Only high order bits 31-16 set */
        MOVZ_IMM16(reg, imm>>16, is64, 1);
    } else if (imm >= (uint64)-65536L) {
        /* Signed "MOV immed" uses MOVN */
        MOVN_IMM16(reg, ~imm, is64);
    } else {
        /* 32/64-bit immediate (signed is handled as 64-bit).
           Many numbers have shorter forms than mov+movk... */
        MOV_IMM16(reg, imm, is64);
        MOVK_IMM16(reg, imm>>16, is64, 1);
        if (imm > 4294967295U && is64) {
            /* 64-bit immediate */
            MOVK_IMM16(reg, imm>>32, IS_64, 2);
            MOVK_IMM16(reg, imm>>48, IS_64, 3);
        }
    }
    return 1;
  oom:
    return 0;
}

/** Emits a computation of a stack address */
static int emit_stackaddr_load(struct CgCtx *cg, unsigned destreg,
                               uint64 offset)
{
    /* TODO large stack offsets */
    assert(offset <= 32767);
    ADD_IMM_X(destreg, SP, offset);
    return 1;
  oom:
    return 0;
}

/** Checks whether an immediate fits in a given instruction */
static int immed_fits(const struct Op *op, uint64 imm)
{
    enum CsbeOp o;
    /* XXX how should e.g. bitwise operations handle signed values? */
    /* All instructions can handle at least 0..=2047 (imm12 minus sign bit) */
    if (imm <= 2047) return 1;

    /* For some instructions there is some kind of negative inverse.
       But that only works if imm >= -2048. */
    if (imm < (uint64)-2048) return 0;

    o = op->op;
    return
        o == CSBEO_MOVE || o == CSBEO_MOVE_TRAP; /* inverse: MOVN */
    /* TODO ADD/SUB supports negative values by flipping ADD<->SUB */
}

/**
 * Loads an operand (from immediate value, register or stack) to some
 * register. Depending on the mode, small immediates will NOT be loaded
 * (so they can be used directly in the following instruction).
 *
 * \param cg        Codegen context.
 * \param f         Function body.
 * \param op        Current IR operation.
 * \param opnd_index Index of argument to load in IR operation.
 * \param opnd_large_index Index of 64-bit immediate.
 * \param type      Type of register (x, w)
 * \param reg_out   Output: The selected register.
 * \param mode      Whether to always load, even for immediates, and
 *                  how to select the output register.
 * \param destreg   Requested destination register if the operand is
 *                  loaded.
 *
 * \return  1 if successful, 0 on out-of-memory error.
 */
static int load_operand(struct CgCtx *cg,
                        const struct FuncIR *f,
                        const struct Op *op,
                        unsigned opnd_index, unsigned opnd_large_index,
                        enum A64Type type, unsigned *reg_out,
                        enum LoadMode mode, unsigned destreg)
{
    unsigned flag = op->opnd[opnd_index];
    unsigned reg;

    if (IS_VAR(flag)) {
        unsigned var_id = op->opnd[opnd_index+1];
        assert(var_id < f->num_vars);
        /* TODO handle floating point types */
        /* TODO handling of function parameters (correct reg, and save/load if needed) */
        reg = assign_reg(cg, f, var_id);
        if (reg == NO_REG) {
            /* Load from stack */
            const struct CgVar *cv;
            const struct Var *fv;
            reg = destreg;
            cv = &cg->vars[var_id];
            assert(cv->has_stack_space);
            /* FIXME this assertion can fail when we run out of regs.
                     it should really be in the outer level, and should be fixed. */
            assert(type != T_LARGE); /* can only load scalars here */
            assert(cv->size <= 8);
            fv = &f->vars[var_id];
            assert(!fv->type.is_struct && !fv->type.is_array);
            assert(cv->stackoffs <= cg->func_stacksize);
            ZCHK(emit_load(cg, reg, type,
                           SP, cv->stackoffs, fv->type.simple_kind));
        }
    } else {
        /* Immediate value */
        uint64 imm = op->opnd_large[opnd_large_index].u64;
        if (mode != REG_OR_IMM || !immed_fits(op, imm)) {
            /* Must put the immed in a reg */
            reg = destreg;
            ZCHK(emit_load_imm(cg, destreg, type, imm));
        } else {
            /* Immediate value, and it can be used directly */
            reg = NO_REG;
        }
    }
    assert(reg <= R28 || reg == NO_REG); /* never load to FP,SP,ZR */
    *reg_out = reg;
    return 1;
}

static int clamp_result(struct CgCtx *cg, const struct FuncIR *f,
                        struct IrOpIter *op_iter,
                        unsigned var_id, unsigned reg)
{
    struct CsbeType *type;
    assert(var_id < f->num_vars);
    type = &f->vars[var_id].type;
    if (!type->is_struct && !type->is_array &&
            type->simple_kind == CSBET_BOOL) {
        /* Clamp to [0,1] unless the next instr just checks the flag
           and then discards the variable */
        const struct Op *next = cg_lookahead(op_iter);
        if (!next || next->op != CSBEO_CONDJUMP ||
                    IS_NON_DISCARD(next->opnd[1+0]) ||
                    next->opnd[1+1] != var_id) {
            CMP_IMM12(reg, 0, IS_32);
            if (op_iter->op->op == CSBEO_EQ) {
                CSET_INV(reg, NE, IS_32);
            } else if (op_iter->op->op == CSBEO_NEQ) {
                CSET_INV(reg, EQ, IS_32);
            } else {
                CSET_INV(reg, EQ, IS_32);
            }
        }
    }
    return 1;
  oom:
    return 0;
}

static int store_result(struct CgCtx *cg,
                        const struct FuncIR *f, unsigned var_id,
                        enum A64Type type, unsigned reg)
{
    const struct CgVar *cv;
    const struct Var *fv;

    if (reg != TEMPREG1) return 1; /* Result is already stored */

    fv = &f->vars[var_id];
    cv = &cg->vars[var_id];
    /* Check that assign_reg and cg_stack_alloc_vars agree */
    assert(cv->has_stack_space);
    if (type != T_LARGE) {
        /* Scalar value (or small enough to fit in a register) */
        assert(cv->size <= 8);
        assert(!fv->type.is_struct && !fv->type.is_array); /* not implemented */
        assert(cv->stackoffs <= cg->func_stacksize);
        ZCHK(emit_store(cg, reg, type,
                        SP, cv->stackoffs, fv->type.simple_kind));
    } else {
        /* Struct/Array */
        /* TODO */
    }
    return 1;
}

static int aarch64_jump_to_epilogue(struct CgCtx *cg)
{
    PREALLOC_INSN; /* keep addrslot in sync with following instruction */
    ZCHK(cg_add_addrslot(cg, &cg->jumps_to_epilogue, ADDRSLOT_IMM26));
    B(0);
    return 1;
  oom:
    return 0;
}

static int aarch64_jump_to_ebb(struct CgCtx *cg, unsigned ebb_id)
{
    uint32 offs = cg->ebb_offsets[ebb_id];
    uint32 reladdr;
    if (offs == INVALID_OFFSET) {
        PREALLOC_INSN;
        ZCHK(cg_add_addrslot(cg, &cg->relative_jumps[ebb_id], ADDRSLOT_IMM26));
        reladdr = 0;
    } else {
        reladdr = offs - cg_get_position(cg);
        assert(reladdr <= 0x3ffffffU*4 ||
               reladdr >= (uint32)(-(int32)0x4000000*4));
    }
    B(reladdr);
    return 1;
  oom:
    return 0;
}

static int aarch64_call_internal(struct CgCtx *cg, unsigned func_id)
{
    uint32 offs = cg->func_offsets[func_id];
    uint32 reladdr;
    if (offs == INVALID_OFFSET) {
        PREALLOC_INSN;
        ZCHK(cg_add_addrslot(cg, &cg->internal_calls[func_id],
                             ADDRSLOT_IMM26));
        reladdr = 0;
    } else {
        reladdr = offs - cg_get_position(cg);
        assert(reladdr <= 0x3ffffffU*4 ||
               reladdr >= (uint32)(-(int32)0x4000000*4));
    }
    BL(reladdr);
    return 1;
  oom:
    return 0;
}

/* Unlike aarch64_branch_*(), this function does not actually emit the instr */
static int aarch64_add_condjump(struct CgCtx *cg, unsigned ebb_id,
                                uint32 *reladdr)
{
    uint32 offs = cg->ebb_offsets[ebb_id];
    if (offs == INVALID_OFFSET) {
        PREALLOC_INSN; /* keep addrslot in sync with following instruction */
        ZCHK(cg_add_addrslot(cg, &cg->relative_jumps[ebb_id],
                             ADDRSLOT_BCOND_IMM19));
        *reladdr = 0;
    } else {
        uint32 ra = offs - cg_get_position(cg);
        assert(ra <= 0x7ffffU*4 ||
               ra >= (uint32)(-(int32)0x80000U*4));
        *reladdr = ra;
    }
    return 1;
  oom:
    return 0;
}

/** Emits a comparison operation, that sets the flags for the following
    instruction. This might clobber TEMPREG1 or TEMPREG2. */
static int emit_compare(struct CgCtx *cg, unsigned reg1,
                        unsigned reg2, uint64 imm,
                        enum A64Type regtype)
{
    int is64 = (regtype==T_REG_X);
    /* TODO need to take mixed signedness into account */
    assert(reg1 != NO_REG);
    if (reg2 != NO_REG) {
        CMP_RR(reg1, reg2, is64);
    } else {
        if (imm <= 0xfff) {
            CMP_IMM12(reg1, imm, is64);
        } else if ((uint64)-imm <= 0xfff) {
            ADDS_IMM12(ZR, reg1, (unsigned)-imm, is64);
        } else {
            unsigned tempreg = (reg1 == TEMPREG1 ?
                    TEMPREG2 : TEMPREG1);
            ZCHK(emit_load_imm(cg, tempreg, regtype, imm));
            CMP_RR(reg1, tempreg, is64);
        }
    }
    return 1;
  oom:
    return 0;
}

static unsigned find_free_tempreg(unsigned used_reg1, unsigned used_reg2)
{
    if (used_reg1 != TEMPREG1 && used_reg2 != TEMPREG1) {
        return TEMPREG1;
    } else if (used_reg1 != TEMPREG2 && used_reg2 != TEMPREG2) {
        return TEMPREG2;
    } else {
        return TEMPREG3;
    }
}

static int emit_memcpy(struct CgCtx *cg, unsigned destptr, unsigned srcptr, uint64 size)
{
    unsigned tmpreg;
    uint64 offs;
    enum { MEMCPY_CHUNK_SIZE = 8 };

    if (!size) return 1;
    tmpreg = find_free_tempreg(destptr, srcptr);

    /* This emits an unrolled loop that copies all data */
    /* TODO call memcmp (or emit a loop) for large copies (it's stricly required over 4096 bytes).
            in either case, stack space is needed. */
    offs = 0;
    while (size >= MEMCPY_CHUNK_SIZE) {
        /* TODO use LDP/STP, and perhaps with a q-register (i.e. a vector register) */
        LDR_IMM_X(tmpreg, srcptr, offs/MEMCPY_CHUNK_SIZE);
        STR_IMM_X(tmpreg, destptr, offs/MEMCPY_CHUNK_SIZE);
        offs += MEMCPY_CHUNK_SIZE;
        size -= MEMCPY_CHUNK_SIZE;
    }
    /* Tail */
    if (size >= 8) {
        LDR_IMM_X(tmpreg, srcptr, offs/8);
        STR_IMM_X(tmpreg, destptr, offs/8);
        offs += 8;
        size -= 8;
    }
    if (size >= 4) {
        LDR_IMM_BY_TYPE(tmpreg, srcptr, offs/4, LOADTYPE_W);
        STR_IMM_BY_TYPE(tmpreg, destptr, offs/4, STORETYPE_W);
        offs += 4;
        size -= 4;
    }
    if (size >= 2) {
        LDR_IMM_BY_TYPE(tmpreg, srcptr, offs/2, LOADTYPE_H);
        STR_IMM_BY_TYPE(tmpreg, destptr, offs/2, STORETYPE_H);
        offs += 2;
        size -= 2;
    }
    if (size >= 1) {
        LDR_IMM_BY_TYPE(tmpreg, srcptr, offs, LOADTYPE_B);
        STR_IMM_BY_TYPE(tmpreg, destptr, offs, STORETYPE_B);
        size -= 1;
    }
    assert(size == 0);
    return 1;
  oom:
    return 0;
}

static int emit_binop(struct CgCtx *cg, const struct FuncIR *f,
                      const struct A64OpInfo *opinfo, const struct Op *op,
                      enum A64Type type, enum LoadMode mode, unsigned destreg,
                      unsigned opnd_offset)
{
    unsigned reg1, reg2;

    assert(opinfo->complexity != COMPLEX);
    ZCHK(load_operand(cg, f, op, opnd_offset+0, 0, type, &reg1,
                     REG_OR_IMM_TO_REG, TEMPREG2));
    ZCHK(load_operand(cg, f, op, opnd_offset+2, 0, type, &reg2,
                     mode, TEMPREG1));
    if (!IS_REG_TYPE(type)) {
        /* TODO floating point and large structures */
    } else if (opinfo->complexity != SIMPLE) {
        enum CsbeOp optype = op->op;
        unsigned cond;
        switch ((int)optype) {
        case CSBEO_MOD:
        case CSBEO_MOD_TRAP:
            /* TODO it should NOT handle negative values like in C99 */
            break;
        case CSBEO_LAND:
        case CSBEO_LOR:
        case CSBEO_LXOR: {
            uint32 opcode;
            /* Clamp inputs to [0,1] */
            CMP_IMM12(reg1, 0, IS_32);
            CSET_INV(reg1, EQ, IS_32);
            CMP_IMM12(reg2, 0, IS_32);
            CSET_INV(reg2, EQ, IS_32);
            opcode = opinfo->code[OPTY_REG|type<<1];
            opcode >>= 24;
            IEMIT_RR(destreg,reg1,0x00,reg2,0x00,opcode|(type==T_REG_X)<<7);
            break; }
        /* Comparison operations. Note that these use the
           "cset" instruction, so they use inverted logic */
        case CSBEO_LT:
            cond = GE; /* TODO or HS for unsigned */
            goto compare;
        case CSBEO_GT:
            cond = LE; /* TODO or LS for unsigned */
            goto compare;
        case CSBEO_LE:
            cond = GT; /* TODO or HI for unsigned */
            goto compare;
        case CSBEO_GE:
            cond = LT; /* TODO or LO for unsigned */
          compare:
            ZCHK(emit_compare(cg, reg1, reg2, op->opnd_large[0].u64, type));
            CSET_INV(destreg, cond, type==T_REG_X);
            break;
        default:
            assert(0);
        }
    } else if (reg2 == NO_REG) {
        uint64 imm = op->opnd_large[0].u64;
        uint32 opcode = opinfo->code[OPTY_IMM|type<<1];
        assert(reg1 <= R28);
        assert(imm <= 32767 || imm >= (uint64)-32768);
        /* TODO ADD/SUB supports negative values by flipping ADD<->SUB.
                currently an extra MOV is generated. */
        IEMIT_IMM12(destreg, reg1, imm, opcode>>16, opcode>>24);
    } else {
        uint32 opcode = opinfo->code[OPTY_REG|type<<1];
        assert(reg1 <= R28);
        assert(reg2 <= R28);
        IEMIT_RR(destreg, reg1, opcode>>8, reg2, opcode>>16, opcode>>24);
    }
    return 1;
  oom:
    return 0;
}

static int emit_unop(struct CgCtx *cg, const struct FuncIR *f,
                     const struct A64OpInfo *opinfo, const struct Op *op,
                     enum A64Type type, enum LoadMode mode, unsigned destreg,
                     unsigned opnd_offset)
{
    unsigned reg1;
    unsigned tempreg = TEMPREG1;

    assert(opinfo->complexity != COMPLEX);
    if (op->op==CSBEO_MOVE || op->op==CSBEO_MOVE_TRAP) {
        /* The tempreg is used for loading large immeds or from stack.
           But it is pointless to load that to a temp-reg in case of a MOVE */
        tempreg = destreg;
    }

    ZCHK(load_operand(cg, f, op, opnd_offset+0, 0, type, &reg1,
                      mode, tempreg));
    if (!IS_REG_TYPE(type)) {
        /* TODO floating point */
    } else if (opinfo->complexity != SIMPLE) {
        assert(op->op == CSBEO_LNOT); /* only one possibility so far */
        /* TODO if the value is clamped to [0,1],
                then this can be optimized to EOR <destreg>, <reg1>, 1 */
        /* TODO this can be optimized out if the following instruction is
                some form of conditional insn that can be inverted. */
        CMP_IMM12(reg1, 0, IS_32);
        CSET_INV(destreg, NE, IS_32);
    } else if (reg1 == NO_REG) {
        uint64 imm = op->opnd_large[0].u64;
        uint32 opcode = opinfo->code[OPTY_IMM|type<<1];
        assert(imm <= 32767 || imm >= (uint64)-32768);
        if ((op->op!=CSBEO_MOVE && op->op!=CSBEO_MOVE_TRAP) ||
                imm <= 32767) {
            IEMIT_IMM16(destreg, imm, opcode>>16, opcode>>24);
        } else {
            /* Signed "MOV immed" uses MOVN */
            MOVN_IMM16(destreg, ~imm, type==T_REG_X);
        }
    } else if (op->op==CSBEO_MOVE || op->op==CSBEO_MOVE_TRAP) {
        if (reg1 != tempreg) {
            /* Not already loaded by load_operand */
            unsigned var_id, sourcereg;
            assert(IS_VAR(op->opnd[opnd_offset]));
            var_id = op->opnd[opnd_offset+1];
            assert(var_id < f->num_vars);
            /* TODO handle floating point types */
            /* TODO handling of function parameters (correct reg, and save/load if needed) */
            sourcereg = assign_reg(cg, f, var_id);
            ZCHK(emit_rr_mov(cg, destreg, sourcereg, type));
        }
    } else {
        uint32 opcode = opinfo->code[OPTY_REG|type<<1];
        assert(reg1 <= R28);
        if (op->op==CSBEO_NEG || op->op==CSBEO_NEG_TRAP) {  /*SUB ZR*/
            IEMIT_RR(destreg, ZR, opcode>>8, reg1, opcode>>16, opcode>>24);
        } else {
            IEMIT_R(destreg, reg1, opcode>>8, opcode>>16, opcode>>24);
        }
    }
    return 1;
  oom:
    return 0;
}

static int aarch64_ir_op(struct CgCtx *cg, const struct FuncIR *f,
                         struct IrOpIter *ir_op_iter)
{
    const struct Op *op = ir_op_iter->op;
    struct A64OpInfo opinfo = a64_opinfos[op->op];
    if (opinfo.complexity != COMPLEX) {
        /*  complexity=SIMPLE: Simple rd=ra+rb|imm style instr.
            complexity=SIMPLE_INPUTS: Simple inputs, but a complex op. */
        unsigned destreg;
        enum A64Type type;
        enum LoadMode mode = opinfo.is_regonly ? REG_OR_IMM_TO_REG:REG_OR_IMM;
        unsigned index_dest = opinfo.is_binop ? 4 : 2;
        /* TODO detect and handle mixed-range types */
        /* TODO use dest flags (op->opnd[index_dest]) */
        /* XXX should the dest reg determine the type for CMP also?!? */
        type = determine_var_type(cg, f, op->opnd[index_dest+1]);
        if (type != T_LARGE) {
            ZCHK(prepare_destreg(cg, f, &opinfo, op, index_dest, type, &destreg));
            if (opinfo.is_binop) {
                ZCHK(emit_binop(cg, f, &opinfo, op, type, mode, destreg, 0));
            } else { /* unary op */
                ZCHK(emit_unop(cg, f, &opinfo, op, type, mode, destreg, 0));
            }
            ZCHK(clamp_result(cg, f, ir_op_iter, op->opnd[index_dest+1], destreg));
            ZCHK(store_result(cg, f, op->opnd[index_dest+1], type, destreg));
        } else {
            /* TODO large types */
            assert(op->op == CSBEO_MOVE ||
                   op->op == CSBEO_EQ || op->op == CSBEO_NEQ);
        }
        goto done;
    }
    /* Complex op */
    switch (op->op) {
    case CSBEO_NOP:
        goto done;
    case CSBEO_JUMP: {
        unsigned target_ebb = op->opnd[0];
        ZCHK(aarch64_jump_to_ebb(cg, target_ebb));
        goto done; }
    case CSBEO_CONDJUMP: {
        unsigned target_ebb = op->opnd[0];
        enum CsbeBranchType branchtype = op->opnd[3];
        /* TODO unlikely bblocks should be put last in the function! */
        /*enum CsbeBranchBalance balance = op->opnd[4]; TODO */
        enum A64Type type;
        uint32 reladdr;
        unsigned reg;

        assert(IS_VAR(op->opnd[1+0]));
        type = determine_var_type(cg, f, op->opnd[1+1]);
        ZCHK(load_operand(cg, f, op, 1, 0, type, &reg,
                              REG_OR_IMM, TEMPREG1));
        assert(reg <= R28);
        if ((branchtype == CSBEBT_Z || branchtype == CSBEBT_NZ) &&
                IS_REG_TYPE(type)) {
            /* Use CBZ / CBNZ */
            ZCHK(aarch64_add_condjump(cg, target_ebb, &reladdr));
            CB(reg, reladdr, type==T_REG_X, branchtype==CSBEBT_NZ);
        } else {
            CMP_IMM12(reg, 0, type==T_REG_X);
            ZCHK(aarch64_add_condjump(cg, target_ebb, &reladdr));
            B_COND(branchtype2cond[branchtype], reladdr);
        }
        goto done; }
    case CSBEO_COMPAREJUMP: {
        unsigned target_ebb = op->opnd[0];
        enum CsbeBranchType branchtype = op->opnd[5];
        /*enum CsbeBranchBalance balance = op->opnd[6]; TODO */
        enum A64Type type;
        uint32 reladdr;
        unsigned reg1, reg2;

        assert(IS_VAR(op->opnd[1+0]));
        type = determine_var_type(cg, f, op->opnd[1+1]);
        ZCHK(load_operand(cg, f, op, 1, 0, type, &reg1,
                         REG_OR_IMM_TO_REG, TEMPREG1));
        ZCHK(load_operand(cg, f, op, 3, 0, type, &reg2,
                         REG_OR_IMM_TO_REG, TEMPREG2));
        ZCHK(emit_compare(cg, reg1, reg2, op->opnd_large[0].u64, type));
        ZCHK(aarch64_add_condjump(cg, target_ebb, &reladdr));
        B_COND(branchtype2cond[branchtype], reladdr);
        goto done; }
    case CSBEO_TRAP:
        /* TODO support "catching" traps by jumping to a "trap bblock" */
        IEMIT4(0, 0, 0, 0);
        goto done;
    case CSBEO_CONDTRAP: {
        enum CsbeBranchType branchtype = op->opnd[2];
        enum A64Type type;
        unsigned reg;

        assert(IS_VAR(op->opnd[0+0]));
        type = determine_var_type(cg, f, op->opnd[0+1]);
        assert(type == T_REG_X || type == T_REG_W);
        ZCHK(load_operand(cg, f, op, 0, 0, type, &reg,
                         REG_OR_IMM_TO_REG, TEMPREG1));

        if ((branchtype == CSBEBT_Z || branchtype == CSBEBT_NZ) &&
                IS_REG_TYPE(type)) {
            CB(reg, +2*4, type==T_REG_X, branchtype!=CSBEBT_NZ);
        } else {
            CMP_IMM12(reg, 0, type==T_REG_X);
            B_COND(INVERSE_COND(branchtype2cond[branchtype]), +2*4);
        }
        IEMIT4(0, 0, 0, 0); /* this will be skipped (+2 above)
                               if the condition is false */
        goto done; }
    case CSBEO_RETURN_VOID:
        /* TODO can be optimized to just RET if there is no epilogue */
        if (!cg->is_last_op) {
            ZCHK(aarch64_jump_to_epilogue(cg));
        }
        goto done;
    case CSBEO_RETURN_ARG: {
        const struct CsbeType *rettype = f->funcdef->returntype;
        enum A64Type type = determine_type(cg, rettype);
        if (IS_REG_TYPE(type)) {
            unsigned outreg;
            ZCHK(load_operand(cg, f, op, 0, 0, type, &outreg,
                              REG_OR_IMM_TO_REG, R0));
            if (outreg != R0) {
                emit_rr_mov(cg, R0, outreg, type);
            }
        } else {
            /* TODO struct/array returns.
                    small types can be merged into one (or two?) regs */
            /* TODO check how floating point works in the ABI */
        }
        if (!cg->is_last_op) {
            ZCHK(aarch64_jump_to_epilogue(cg));
        }
        goto done; }
    case CSBEO_CALL_START: {
        const struct Op *call = op->opnd_large[0].ptr;
        unsigned num_args = op->opnd[0];
        /*cg->current_call = call;*/
        if (call->op == CSBEO_CALL_FUNCDEF) {
            unsigned func_id = call->opnd[1];
            const struct CsbeFuncdef *fd = &cg->csbe->funcdefs[func_id];
            assert(fd->num_params == num_args);
            cg->current_paramtype = fd->paramtypes;
        } else {
            assert(call->op == CSBEO_CALL_FUNCVAR);
            /*unsigned typedef_id = call->opnd[1];*/
            /* TODO */
        }

        /* Save our incoming paramters */
        /* TODO Copy the param regs to callee-saved regs in the epilogue instead
                (but don't do that for leaf functions!) */
        ZCHK(saverestore_param_regs(cg, f, SAVE));

        cg->current_param_reg = 0;

        goto done; }
    case CSBEO_CALL_ARG: {
        /* PTR(struct Op *prev_arg), VAR_IN_OR_IMMED(arg)) */
        unsigned paramreg = cg->current_param_reg;
        const struct CsbeType *paramtype = cg->current_paramtype++;
        enum A64Type regtype = determine_type(cg, paramtype);
        if (paramreg <= 7 && regtype != T_LARGE) {
            unsigned outreg;
            ZCHK(load_operand(cg, f, op, 0, 1, regtype, &outreg,
                     REG_OR_IMM_TO_REG, paramreg));
            if (outreg != paramreg) {
                emit_rr_mov(cg, paramreg, outreg, regtype);
            }
            cg->current_param_reg++;
        } else {
            /* TODO pass parameter on stack (...and can we avoid copying it?) */
        }
        goto done; }
    case CSBEO_CALL_FUNCDEF: {
        /* PTR(struct Op *last_arg), ENUM(enum CsbeCallMode callmode), FUNCDEF(funcdef)) */
        /*const struct Op *last_arg = OPARG_PTR(op, 0);*/
        /*enum CsbeCallMode callmode = op->opnd[0];*/
        unsigned func_id = op->opnd[1];
        const struct CsbeFuncdef *fd;

        assert(func_id < cg->csbe->num_funcs);
        fd = &cg->csbe->funcdefs[func_id];
        /* TODO take these into account: fd->abi, fd->flags (CSBEFD_NORETURN, CSBEFD_OBJGLOBAL) */
        if (fd->funcbody) { /* TODO ...and not exported+interposable */
            /* Local function */
            ZCHK(aarch64_call_internal(cg, func_id));
        } else {
            /* Imported function */
            /* TODO distinguish between: 1) import from PLT, 2) windows dllimport, 3) objfile import.
                         Use cg->arch->outformat. (all functions that go here are imports!) */
            PREALLOC_INSN;
            ZCHK(cg_add_reloc(cg, CG_RK_FUNCCALL, func_id, 0));
            BL(0);
        }
        goto done; }
    case CSBEO_CALL_FUNCVAR: {
        /* PTR(struct Op *last_arg), ENUM(enum CsbeCallMode callmode), TYPEDEF(typdef), VAR_IN(funcptr))*/
        /* TODO */
        goto done; }
    case CSBEO_CALL_GET_RETURN: {
        unsigned destvar_id = op->opnd[0+1];
        unsigned destreg;
        enum A64Type type;
        type = determine_var_type(cg, f, destvar_id);
        /* FIXME struct returns:
                - small struct return values can be reg-allocated
                - large return values are passed by reference. */
        ZCHK(prepare_destreg(cg, f, &opinfo, op, 0, type, &destreg));
        ZCHK(emit_rr_mov(cg, destreg, R0, type));
        ZCHK(store_result(cg, f, destvar_id, type, destreg));
        ZCHK(saverestore_param_regs(cg, f, RESTORE));
        goto done; }
    case CSBEO_CALL_DISCARD_RETURN: {
        ZCHK(saverestore_param_regs(cg, f, RESTORE));
        goto done; }
    case CSBEO_DISCARD: {
        /*unsigned var_id = op->opnd[0+1];*/
        /* TODO */
        goto done; }
    case CSBEO_CHOICE: {
        /* FIXME csbe_op_end could transform "static" choice into DISCARD(notselected_var)+MOVE(selected_var,out) */
        /*enum CsbeSelectBalance balance = op->opnd[2];
        unsigned var_out = op->opnd[7];
        ZCHK(load_operand(cg, f, op, 0, 0, type, &reg_sel,
                     REG_OR_IMM_TO_REG, destreg));
        ZCHK(load_operand(cg, f, op, 3, 0, type, &reg_true,
                     REG_OR_IMM, destreg));
        ZCHK(load_operand(cg, f, op, 5, 0, type, &reg_false,
                     REG_OR_IMM, destreg));*/
        /* TODO */
        goto done; }
    case CSBEO_SIZEOF:  /* TYPE(type), ENUM(enum CsbeSizeofKind), VAR_OUT(out) */
        /* TODO add common function to compute these */
        goto done;
    case CSBEO_COPY: {
        /* [0]: type, [1]: destptr var, [3]: srcptr var */
        /*unsigned destptr_var = op->opnd[0+1];
        unsigned srcptr_var = op->opnd[2+1];
        cosnt struct CsbeType *typdef = cg->typedefs[op->opnd[4]];*/
        /* TODO */
        goto done; }
    case CSBEO_CLEAR: {
        /* [0]: type, [1]: ptr var */
        /*unsigned ptr_var = op->opnd[0+1];
        const struct CsbeType *typdef = cg->typedefs[op->opnd[2]];*/
        /* TODO */
        goto done; }
    case CSBEO_MOVETOPTR: {
        /* [0]: src var/imm, [2]: var with ptr to destination */
        unsigned srcvar_or_type = op->opnd[0+1];
        enum A64Type type;
        if (IS_VAR(op->opnd[0+0])) {
            type = determine_var_type(cg, f, srcvar_or_type);
        } else {
            type = determine_type_simple(srcvar_or_type);
        }
        /*{ FIXME the frontend sometimes emits IR with a non-pointer type! (swapped/shifted args?)
            unsigned ptrvar_id = op->opnd[2+1];
            const struct Var *fvp;
            fvp = &f->vars[ptrvar_id];
            assert(!fvp->type.is_struct && !fvp->type.is_array);
            assert(fvp->type.simple_kind == CSBET_DPTR ||
                   fvp->type.simple_kind == CSBET_DPTR_ALIASED ||
                   fvp->type.simple_kind == CSBET_DPTR_THREADED);
        }*/
        if (type != T_LARGE) {
            unsigned srcreg, ptrreg;
            enum CsbeTypeKind valuetype;
            ZCHK(load_operand(cg, f, op, 0, 0, type, &srcreg,
                              REG_OR_IMM_TO_REG, TEMPREG1));
            ZCHK(load_operand(cg, f, op, 2, 0, T_REG_X, &ptrreg,
                              REG_OR_IMM_TO_REG, TEMPREG2));
            if (IS_VAR(op->opnd[0+0])) {
                /* Store variable to pointer */
                const struct Var *fv;
                fv = &f->vars[srcvar_or_type];
                assert(!fv->type.is_struct && !fv->type.is_array);
                valuetype = fv->type.simple_kind;
            } else {
                /* Store immediate to pointer */
                valuetype = srcvar_or_type;
            }
            assert(srcreg <= R28);
            assert(ptrreg <= R28);
            ZCHK(emit_store(cg, srcreg, type, ptrreg, 0, valuetype));
        } else {
            /* TODO this is equivalent to ADDRLOCAL+COPY */
        }
        goto done; }
    case CSBEO_DEREF: {
        /* [0]: pointer to dereference, [2]: destination var */
        /* FIXME dereferencing a function parameter does not work */
        unsigned destvar_id = op->opnd[2+1];
        enum A64Type type;
        type = determine_var_type(cg, f, destvar_id);
        if (type != T_LARGE) {
            unsigned destreg, ptrreg;
            const struct Var *fv;
            ZCHK(prepare_destreg(cg, f, &opinfo, op, 2, type, &destreg));
            ZCHK(load_operand(cg, f, op, 0, 0, type, &ptrreg,
                              REG_OR_IMM, destreg));
            fv = &f->vars[destvar_id];
            assert(!fv->type.is_struct && !fv->type.is_array);
            ZCHK(emit_load(cg, destreg, type, ptrreg, 0, fv->type.simple_kind));
            ZCHK(store_result(cg, f, destvar_id, type, destreg));
        } else {
            /* TODO this is equivalent to ADDRLOCAL+COPY */
        }
        goto done; }
    case CSBEO_ADDRELEM:
    case CSBEO_LOADELEM: {
        /* [ptr0]: type, [0-1]: base var, [2-3]: array index, [4]: field index, [5-6]: dest var */
        const struct CsbeType *container_type = op->opnd_large[0].ptr;
        unsigned baseptr_flags = op->opnd[0+0];
        unsigned index_flags = op->opnd[2+0];
        unsigned fieldnum = op->opnd[4];
        unsigned destvar_id = op->opnd[5+1];
        unsigned offset;
        enum A64Type regtype;
        unsigned destreg, ptrreg, ptrreg_out;

        assert(IS_VAR(baseptr_flags));
        ZCHK(cg_get_elem_offset(cg, container_type,
                (IS_IMMED(index_flags) ? op->opnd_large[1].u64 : 0), fieldnum,
                &offset));
        regtype = determine_var_type(cg, f, destvar_id);
        assert(op->op == CSBEO_LOADELEM || regtype == T_REG_X);
        ZCHK(prepare_destreg(cg, f, &opinfo, op, 5, regtype, &destreg));
        ZCHK(load_operand(cg, f, op, 0, 0, T_REG_X, &ptrreg,
                              REG_OR_IMM_TO_REG, destreg));
        ptrreg_out = (op->op == CSBEO_LOADELEM ? ptrreg : destreg);
        if (offset || ptrreg_out != ptrreg) {
            /* Static part of offset */
            if (offset < 4096) { /* 2^12 */
                ADD_IMM_X(ptrreg_out, ptrreg, offset);
            } else {
                unsigned tempreg = find_free_tempreg(ptrreg, ptrreg_out);
                ZCHK(emit_load_imm(cg, tempreg, T_REG_X, offset));
                ADD_RR(ptrreg_out, ptrreg, tempreg, IS_64);
            }
        }
        if (IS_VAR(index_flags)) {
            /* Dynamic part of offset */
            unsigned elemsize;
            unsigned align_dummy;
            unsigned indexreg;
            ZCHK(cg_get_elemsize(cg, container_type, &elemsize, &align_dummy));
            ZCHK(load_operand(cg, f, op, 2, 1, T_REG_X, &indexreg,
                      REG_OR_IMM_TO_REG, TEMPREG2));
            if (elemsize == 1) {
                ADD_RR(ptrreg_out, ptrreg_out, indexreg, IS_64);
            /* TODO optimize for power-of-two?
            } else if (is_pow2(elemsize)) {
                add to ptrreg_out with shift*/
            } else {
                unsigned tempreg = find_free_tempreg(indexreg, ptrreg_out);
                ZCHK(emit_load_imm(cg, tempreg, T_REG_X, elemsize));
                /* TODO use UMADDL if index and elemsize are < 2^32 */
                MADD(ptrreg_out, indexreg, tempreg, ptrreg_out, IS_64);
            }
        }
        if (op->op == CSBEO_LOADELEM) {
            if (regtype != T_LARGE) {
                const struct Var *fv;
                fv = &f->vars[destvar_id];
                assert(!fv->type.is_struct && !fv->type.is_array);
                ZCHK(emit_load(cg, destreg, regtype, ptrreg, 0,
                               fv->type.simple_kind));
            } else {
                /* TODO */
            }
        }
        ZCHK(store_result(cg, f, destvar_id, regtype, destreg));
        /* TODO this generates a mov, but it is possible that it is redundant:
                1. the baseptr var might come from a discarding move immediately before.
                2. the out var might be used by a discarding move immediately after.
                In those cases, the move can be eliminated (if the var is also changed),
                but this should ONLY be done if the operation is still done on a register. */
        goto done; }
    case CSBEO_ADDRLOCAL: {
        /* [0]: input var (to take address of), [2]: dest var */
        unsigned localvar_id = op->opnd[0+1];
        unsigned destvar_id = op->opnd[2+1];
        unsigned destreg;
        enum A64Type type;
        const struct CgVar *cv;

        cv = &cg->vars[localvar_id];
        assert(cv->has_stack_space);
        type = determine_var_type(cg, f, destvar_id);
        assert(type == T_REG_X);
        ZCHK(prepare_destreg(cg, f, &opinfo, op, 2, type, &destreg));
        ZCHK(emit_stackaddr_load(cg, destreg, cv->stackoffs));
        ZCHK(store_result(cg, f, destvar_id, type, destreg));
        goto done; }
    case CSBEO_ADDRGLOBAL:
    case CSBEO_LOADGLOBAL: {
        /* [0]: datadef id, [1]: dest var */
        unsigned datadef_id = op->opnd[0];
        const struct CsbeDatadef *dd = &cg->csbe->datadefs[datadef_id];
        unsigned destvar_id = op->opnd[1+1];
        unsigned destreg;
        enum A64Type regtype;
        int is_load = op->op == CSBEO_LOADGLOBAL;

        regtype = determine_var_type(cg, f, destvar_id);
        assert(regtype == T_REG_X || is_load);
        ZCHK(prepare_destreg(cg, f, &opinfo, op, 1, regtype, &destreg));
        if ((dd->flags & CSBEDD_EXTERNAL) != 0) {
            /* TODO load address from GOT. can be used in CSBEO_ADDRFUNC also */
        } else {
            int is_small_load = (is_load &&
                    !dd->type.is_struct && !dd->type.is_array);
            PREALLOC_INSN;
            ZCHK(cg_add_reloc(cg, is_small_load ? CG_RK_DATALOAD : CG_RK_DATAADDR,
                              datadef_id, 0));
            ADRP(destreg, 0);          /* high bits of addr */
            if (is_small_load) {
                /* Load into register/variable */
                ZCHK(emit_load(cg, destreg, regtype, destreg, 0,
                               dd->type.simple_kind));
            } else if (is_load) {
                /* Load large object using memory copying */
                const struct CgVar *cv;
                unsigned addrreg = find_free_tempreg(destreg, NO_REG);
                ADD_IMM_X(destreg, destreg, 0); /* low bits */
                cv = &cg->vars[destvar_id];
                assert(cv->has_stack_space);
                ZCHK(emit_stackaddr_load(cg, addrreg, cv->stackoffs));
                ZCHK(emit_memcpy(cg, /*destination*/addrreg,
                                 /*source(!)*/destreg, cv->size));
                goto done;
            } else {
                /* Get address */
                ADD_IMM_X(destreg, destreg, 0); /* low bits */
            }
        }
        ZCHK(store_result(cg, f, destvar_id, regtype, destreg));
        goto done; }
    case CSBEO_ADDRFUNC: {
        /* [0]: funcdef id, [1]: dest var */
        /*const struct CsbeFuncdef *fdef = OPARG_FUNCDEF(op, 0);
        unsigned var_out = op->opnd[1+1];*/
        /* TODO */
        goto done; }
    CODEGEN_CASE_SIMPLE_OPS
    CODEGEN_CASE_SIMPLE_INPUTS_OPS
        /* Unreachable */ ;
    }
    CG_INTERNAL_ERROR();
  oom:
    return 0;
  done:
    return 1;
}

static int aarch64_codegen(struct CgCtx *cg, const struct CsbeFuncdef *f)
{
    struct EbbIter ebb_iter;
    unsigned ebb_id;
    struct FuncIR *fb = f->funcbody;

    aarch64_process_addrslots(cg, cg->internal_calls[f->func_id]);
    /* TODO add support for passing structs by value */
    if (!aarch64_prologue(cg, fb)) goto oom;

    ebb_id = 0;
    EBB_ITER(ebb_iter, fb) {
        struct IrOpIter ir_op_iter;
        cg->ebb_offsets[ebb_id] = cg_get_position(cg);
        aarch64_process_addrslots(cg, cg->relative_jumps[ebb_id++]);
        IR_OP_ITER(ir_op_iter, ebb_iter) {
            cg->is_last_op = IS_LAST_OP(ebb_iter, ir_op_iter);
            if (!aarch64_ir_op(cg, fb, &ir_op_iter)) return 0;
        }
    }
    if (!aarch64_epilogue(cg, fb)) goto oom;
    return 1;
  oom:
    return 0;
}

static int aarch64_gen_elf_plt(struct CgCtx *cg, uint32 got, uint32 gotoffs,
                               uint32 pc)
{
    uint32 base = ((got+gotoffs)>>12) - (pc>>12);
    uint32 offset;

    /* TODO this could probably use a PC-relative load if the code + delta to .got is small */
    ADRP(R17, base);
    offset = got+gotoffs;
    LDR_IMM_X(R17, R17, (offset & 4095)/8);
    BR(R17);
    return 1;
  oom:
    return 0;
}

static int aarch64_get_elf_plt_size(const struct CgCtx *cg)
{
    (void)cg;
    return 4*3;
}

static int aarch64_process_reloc(struct CgCtx *cg, struct BinWriter *bw,
                                 struct CgRelocEntry *re, uint32 fileoffs,
                                 uint32 addr, uint32 newaddr)
{
    (void)cg;
    switch (re->kind) {
    case CG_RK_FUNCCALL: {
        uint32 delta = (newaddr>>2) - (addr>>2);

        assert(delta <= 0x3ffffffU*4 ||
               delta >= (uint32)(-(int32)0x4000000*4));
        binwriter_seekto(bw, fileoffs);
        out32(bw, (delta&0x3ffffffU) | 0x94000000);
        break; }
    case CG_RK_FUNCADDR:
    case CG_RK_DATAADDR: {
        uint32 pagedelta = (newaddr>>12) - (addr>>12);

        binwriter_seekto(bw, fileoffs);
        /* Update adrp */
        out32_or(bw, (pagedelta&~3U)<<3 | (pagedelta&3U)<<29);
        /* Update add */
        out32_or(bw, (newaddr & 4095) << 10);
        break; }
    case CG_RK_DATALOAD: {
        /* TODO */
        break; }
    case CG_RK_GOTPC:
        /* Not used with aarch64 */
        assert(0);
        break;
    }
    return 1;
}

static int aarch64_gen_start_code(struct CgCtx *cg)
{
    unsigned main_id = cg->csbe->main_func->func_id;
    if (cg->arch->outformat == CSBEOF_ELF_EXE) {
        PREALLOC_INSN;
        ZCHK(cg_add_reloc(cg, CG_RK_FUNCADDR, main_id, 0));
        ADRP(R0, 0);          /* pointer to main(), high bits */
        ADD_IMM_X(R0, R0, 0); /* pointer to main(),  low bits */
        LDR_X(R1, SP);      /* argc */
        ADD_IMM_X(R2, SP, 8); /* argv */
        MOV_IMM16(R3, 0, IS_64);   /* init (not used) */
        MOV_IMM16(R4, 0, IS_64);   /* fini (not used) */
        MOV_IMM16(R5, 0, IS_64);   /* rtld_fini (not used) */
        MOV_R_XSP(R6);
        PREALLOC_INSN;
        ZCHK(cg_add_reloc(cg, CG_RK_FUNCCALL, FUNCID_LIBC_INIT, 0));
        BL(0);
    } else {
        /* TODO for PE:
            - align stack
            - call main addr (which will have argv == NULL)
        */
    }
    return 1;
  oom:
    return 0;
}

static int aarch64_gen_helper_code(struct CgCtx *cg)
{
    /* Does nothing for now */
    (void)cg;
    return 1;
}

void aarch64_get_codegen(struct Codegen *cgen)
{
    cgen->has_endianness = aarch64_has_endianness;
    cgen->has_feature_state = aarch64_has_feature_state;
    cgen->initialize = aarch64_initialize;
    cgen->codegen = aarch64_codegen;
    cgen->gen_elf_plt = aarch64_gen_elf_plt;
    cgen->get_elf_plt_size = aarch64_get_elf_plt_size;
    cgen->process_reloc = aarch64_process_reloc;
    cgen->gen_start_code = aarch64_gen_start_code;
    cgen->gen_helper_code = aarch64_gen_helper_code;
}