/* * +----------------------------------------------------------------------+ * | Zend JIT | * +----------------------------------------------------------------------+ * | Copyright (c) The PHP Group | * +----------------------------------------------------------------------+ * | This source file is subject to version 3.01 of the PHP license, | * | that is bundled with this package in the file LICENSE, and is | * | available through the world-wide-web at the following url: | * | https://www.php.net/license/3_01.txt | * | If you did not receive a copy of the PHP license and are unable to | * | obtain it through the world-wide-web, please send a note to | * | license@php.net so we can mail you a copy immediately. | * +----------------------------------------------------------------------+ * | Authors: Dmitry Stogov | * +----------------------------------------------------------------------+ */ #include "Zend/zend_types.h" #include "Zend/zend_type_info.h" #include "jit/ir/ir.h" #include "jit/ir/ir_builder.h" #include "jit/tls/zend_jit_tls.h" #if defined(IR_TARGET_X86) # define IR_REG_SP 4 /* IR_REG_RSP */ # define IR_REG_FP 5 /* IR_REG_RBP */ # define ZREG_FP 6 /* IR_REG_RSI */ # define ZREG_IP 7 /* IR_REG_RDI */ # define ZREG_FIRST_FPR 8 # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7)) /* all preserved registers */ # if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL # error # endif #elif defined(IR_TARGET_X64) # define IR_REG_SP 4 /* IR_REG_RSP */ # define IR_REG_FP 5 /* IR_REG_RBP */ # if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL /* Use the first two arg registers of the preserve_none calling convention for FP/IP * https://github.com/llvm/llvm-project/blob/68bfe91b5a34f80dbcc4f0a7fa5d7aa1cdf959c2/llvm/lib/Target/X86/X86CallingConv.td#L1029 */ # define ZREG_FP 12 /* IR_REG_R12 */ # define ZREG_IP 13 /* IR_REG_R13 */ # else # define ZREG_FP 14 /* IR_REG_R14 */ # define ZREG_IP 15 /* IR_REG_R15 */ # endif # define ZREG_FIRST_FPR 16 # if defined(_WIN64) # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15)) /* # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<6) | (1<<7) | (1<<12) | (1<<13) | (1<<14) | (1<<15) | \ (1<<(16+6)) | (1<<(16+7)) | (1<<(16+8)) | (1<<(16+9)) | (1<<(16+10)) | \ (1<<(16+11)) | (1<<(16+12)) | (1<<(16+13)) | (1<<(16+14)) | (1<<(16+15))) */ # define IR_SHADOW_ARGS 32 # else # define IR_REGSET_PRESERVED ((1<<3) | (1<<5) | (1<<12) | (1<<13) | (1<<14) | (1<<15)) /* all preserved registers */ # endif #elif defined(IR_TARGET_AARCH64) # define IR_REG_SP 31 /* IR_REG_RSP */ # define IR_REG_LR 30 /* IR_REG_X30 */ # define IR_REG_FP 29 /* IR_REG_X29 */ # if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL /* Use the first two arg registers of the preserve_none calling convention for FP/IP * https://github.com/llvm/llvm-project/blob/68bfe91b5a34f80dbcc4f0a7fa5d7aa1cdf959c2/llvm/lib/Target/AArch64/AArch64CallingConvention.td#L541 */ # define ZREG_FP 20 /* IR_REG_X20 */ # define ZREG_IP 21 /* IR_REG_X21 */ # else # define ZREG_FP 27 /* IR_REG_X27 */ # define ZREG_IP 28 /* IR_REG_X28 */ # endif # define ZREG_FIRST_FPR 32 # define IR_REGSET_PRESERVED ((1<<19) | (1<<20) | (1<<21) | (1<<22) | (1<<23) | \ (1<<24) | (1<<25) | (1<<26) | (1<<27) | (1<<28)) /* all preserved registers */ #else # error "Unknown IR target" #endif #define ZREG_RX ZREG_IP #define OPTIMIZE_FOR_SIZE 0 /* IR builder defines */ #undef _ir_CTX #define _ir_CTX (&jit->ctx) #if GCC_GLOBAL_REGS # define IR_OPCODE_HANDLER_RET IR_VOID #else # define IR_OPCODE_HANDLER_RET IR_ADDR #endif #undef ir_CONST_ADDR #define ir_CONST_ADDR(_addr) jit_CONST_ADDR(jit, (uintptr_t)(_addr)) #define ir_CONST_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), 0) #define ir_CONST_FC_FUNC(_addr) jit_CONST_FUNC(jit, (uintptr_t)(_addr), IR_FASTCALL_FUNC) #define ir_CAST_FC_FUNC(_addr) ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \ ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_I32)) # define ir_CONST_OPCODE_HANDLER_FUNC(_addr) \ jit_CONST_OPCODE_HANDLER_FUNC(jit, _addr) # define ir_CAST_OPCODE_HANDLER_FUNC(_addr) ir_fold2(_ir_CTX, IR_OPT(IR_PROTO, IR_ADDR), (_addr), \ ir_proto_0(_ir_CTX, IR_FASTCALL_FUNC, IR_OPCODE_HANDLER_RET)) #define ir_CONST_FUNC_PROTO(_addr, _proto) \ jit_CONST_FUNC_PROTO(jit, (uintptr_t)(_addr), (_proto)) #undef ir_ADD_OFFSET #define ir_ADD_OFFSET(_addr, _offset) \ jit_ADD_OFFSET(jit, _addr, _offset) #ifdef ZEND_ENABLE_ZVAL_LONG64 # define IR_LONG IR_I64 # define ir_CONST_LONG ir_CONST_I64 # define ir_UNARY_OP_L ir_UNARY_OP_I64 # define ir_BINARY_OP_L ir_BINARY_OP_I64 # define ir_ADD_L ir_ADD_I64 # define ir_SUB_L ir_SUB_I64 # define ir_MUL_L ir_MUL_I64 # define ir_DIV_L ir_DIV_I64 # define ir_MOD_L ir_MOD_I64 # define ir_NEG_L ir_NEG_I64 # define ir_ABS_L ir_ABS_I64 # define ir_SEXT_L ir_SEXT_I64 # define ir_ZEXT_L ir_ZEXT_I64 # define ir_TRUNC_L ir_TRUNC_I64 # define ir_BITCAST_L ir_BITCAST_I64 # define ir_FP2L ir_FP2I64 # define ir_ADD_OV_L ir_ADD_OV_I64 # define ir_SUB_OV_L ir_SUB_OV_I64 # define ir_MUL_OV_L ir_MUL_OV_I64 # define ir_NOT_L ir_NOT_I64 # define ir_OR_L ir_OR_I64 # define ir_AND_L ir_AND_I64 # define ir_XOR_L ir_XOR_I64 # define ir_SHL_L ir_SHL_I64 # define ir_SHR_L ir_SHR_I64 # define ir_SAR_L ir_SAR_I64 # define ir_ROL_L ir_ROL_I64 # define ir_ROR_L ir_ROR_I64 # define ir_MIN_L ir_MIN_I64 # define ir_MAX_L ir_MAX_I64 # define ir_LOAD_L ir_LOAD_I64 #else # define IR_LONG IR_I32 # define ir_CONST_LONG ir_CONST_I32 # define ir_UNARY_OP_L ir_UNARY_OP_I32 # define ir_BINARY_OP_L ir_BINARY_OP_I32 # define ir_ADD_L ir_ADD_I32 # define ir_SUB_L ir_SUB_I32 # define ir_MUL_L ir_MUL_I32 # define ir_DIV_L ir_DIV_I32 # define ir_MOD_L ir_MOD_I32 # define ir_NEG_L ir_NEG_I32 # define ir_ABS_L ir_ABS_I32 # define ir_SEXT_L ir_SEXT_I32 # define ir_ZEXT_L ir_ZEXT_I32 # define ir_TRUNC_L ir_TRUNC_I32 # define ir_BITCAST_L ir_BITCAST_I32 # define ir_FP2L ir_FP2I32 # define ir_ADD_OV_L ir_ADD_OV_I32 # define ir_SUB_OV_L ir_SUB_OV_I32 # define ir_MUL_OV_L ir_MUL_OV_I32 # define ir_NOT_L ir_NOT_I32 # define ir_OR_L ir_OR_I32 # define ir_AND_L ir_AND_I32 # define ir_XOR_L ir_XOR_I32 # define ir_SHL_L ir_SHL_I32 # define ir_SHR_L ir_SHR_I32 # define ir_SAR_L ir_SAR_I32 # define ir_ROL_L ir_ROL_I32 # define ir_ROR_L ir_ROR_I32 # define ir_MIN_L ir_MIN_I32 # define ir_MAX_L ir_MAX_I32 # define ir_LOAD_L ir_LOAD_I32 #endif /* A helper structure to collect IT rers for the following use in (MERGE/PHI)_N */ typedef struct _ir_refs { uint32_t count; uint32_t limit; ir_ref refs[] ZEND_ELEMENT_COUNT(count); } ir_refs; #define ir_refs_size(_n) (offsetof(ir_refs, refs) + sizeof(ir_ref) * (_n)) #define ir_refs_init(_name, _n) _name = alloca(ir_refs_size(_n)); \ do {_name->count = 0; _name->limit = (_n);} while (0) static void ir_refs_add(ir_refs *refs, ir_ref ref) { ir_ref *ptr; ZEND_ASSERT(refs->count < refs->limit); ptr = refs->refs; ptr[refs->count++] = ref; } static size_t zend_jit_trace_prologue_size = (size_t)-1; #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) static uint32_t allowed_opt_flags = 0; static uint32_t default_mflags = 0; #endif static bool delayed_call_chain = 0; // TODO: remove this var (use jit->delayed_call_level) ??? #ifdef ZTS static size_t tsrm_ls_cache_tcb_offset = 0; static size_t tsrm_tls_index = -1; static size_t tsrm_tls_offset = -1; # define EG_TLS_OFFSET(field) \ (executor_globals_offset + offsetof(zend_executor_globals, field)) # define CG_TLS_OFFSET(field) \ (compiler_globals_offset + offsetof(zend_compiler_globals, field)) # define jit_EG(_field) \ ir_ADD_OFFSET(jit_TLS(jit), EG_TLS_OFFSET(_field)) # define jit_CG(_field) \ ir_ADD_OFFSET(jit_TLS(jit), CG_TLS_OFFSET(_field)) #else # define jit_EG(_field) \ ir_CONST_ADDR(&EG(_field)) # define jit_CG(_field) \ ir_CONST_ADDR(&CG(_field)) #endif #define jit_CALL(_call, _field) \ ir_ADD_OFFSET(_call, offsetof(zend_execute_data, _field)) #define jit_EX(_field) \ jit_CALL(jit_FP(jit), _field) #define jit_RX(_field) \ jit_CALL(jit_IP(jit), _field) #define JIT_STUBS(_) \ _(exception_handler, IR_SKIP_PROLOGUE) \ _(exception_handler_undef, IR_SKIP_PROLOGUE) \ _(exception_handler_free_op2, IR_SKIP_PROLOGUE) \ _(exception_handler_free_op1_op2, IR_SKIP_PROLOGUE) \ _(interrupt_handler, IR_SKIP_PROLOGUE) \ _(leave_function_handler, IR_SKIP_PROLOGUE) \ _(negative_shift, IR_SKIP_PROLOGUE) \ _(mod_by_zero, IR_SKIP_PROLOGUE) \ _(invalid_this, IR_SKIP_PROLOGUE) \ _(undefined_function, IR_SKIP_PROLOGUE) \ _(throw_cannot_pass_by_ref, IR_SKIP_PROLOGUE) \ _(icall_throw, IR_SKIP_PROLOGUE) \ _(leave_throw, IR_SKIP_PROLOGUE) \ _(hybrid_runtime_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \ _(hybrid_profile_jit, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \ _(hybrid_func_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \ _(hybrid_loop_hot_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \ _(hybrid_func_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \ _(hybrid_ret_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \ _(hybrid_loop_trace_counter, IR_SKIP_PROLOGUE | IR_START_BR_TARGET) \ _(trace_halt, IR_SKIP_PROLOGUE) \ _(trace_escape, IR_SKIP_PROLOGUE) \ _(trace_exit, IR_SKIP_PROLOGUE) \ _(undefined_offset, IR_FUNCTION | IR_FASTCALL_FUNC) \ _(undefined_key, IR_FUNCTION | IR_FASTCALL_FUNC) \ _(cannot_add_element, IR_FUNCTION | IR_FASTCALL_FUNC) \ _(assign_const, IR_FUNCTION | IR_FASTCALL_FUNC) \ _(assign_tmp, IR_FUNCTION | IR_FASTCALL_FUNC) \ _(assign_var, IR_FUNCTION | IR_FASTCALL_FUNC) \ _(assign_cv_noref, IR_FUNCTION | IR_FASTCALL_FUNC) \ _(assign_cv, IR_FUNCTION | IR_FASTCALL_FUNC) \ _(new_array, IR_FUNCTION | IR_FASTCALL_FUNC) \ #define JIT_STUB_ID(name, flags) \ jit_stub_ ## name, #define JIT_STUB_FORWARD(name, flags) \ static int zend_jit_ ## name ## _stub(zend_jit_ctx *jit); #define JIT_STUB(name, flags) \ {JIT_STUB_PREFIX #name, zend_jit_ ## name ## _stub, flags}, typedef enum _jit_stub_id { JIT_STUBS(JIT_STUB_ID) jit_last_stub } jit_stub_id; typedef struct _zend_jit_reg_var { ir_ref ref; uint32_t flags; } zend_jit_reg_var; typedef struct _zend_jit_ctx { ir_ctx ctx; const zend_op *last_valid_opline; bool use_last_valid_opline; bool track_last_valid_opline; bool reuse_ip; uint32_t delayed_call_level; int b; /* current basic block number or -1 */ #ifdef ZTS ir_ref tls; #endif ir_ref fp; ir_ref poly_func_ref; /* restored from parent trace snapshot */ ir_ref poly_this_ref; /* restored from parent trace snapshot */ ir_ref trace_loop_ref; ir_ref return_inputs; const zend_op_array *op_array; const zend_op_array *current_op_array; zend_ssa *ssa; zend_string *name; ir_ref *bb_start_ref; /* PHP BB -> IR ref mapping */ ir_ref *bb_predecessors; /* PHP BB -> index in bb_edges -> IR refs of predessors */ ir_ref *bb_edges; zend_jit_trace_info *trace; zend_jit_reg_var *ra; int delay_var; ir_refs *delay_refs; ir_ref eg_exception_addr; HashTable addr_hash; ir_ref stub_addr[jit_last_stub]; } zend_jit_ctx; typedef int8_t zend_reg; typedef struct _zend_jit_registers_buf { #if defined(IR_TARGET_X64) uint64_t gpr[16]; /* general purpose integer register */ double fpr[16]; /* floating point registers */ #elif defined(IR_TARGET_X86) uint32_t gpr[8]; /* general purpose integer register */ double fpr[8]; /* floating point registers */ #elif defined (IR_TARGET_AARCH64) uint64_t gpr[32]; /* general purpose integer register */ double fpr[32]; /* floating point registers */ #else # error "Unknown IR target" #endif } zend_jit_registers_buf; /* Keep 32 exit points in a single code block */ #define ZEND_JIT_EXIT_POINTS_SPACING 4 // push byte + short jmp = bytes #define ZEND_JIT_EXIT_POINTS_PER_GROUP 32 // number of continuous exit points static uint32_t zend_jit_exit_point_by_addr(const void *addr); int ZEND_FASTCALL zend_jit_trace_exit(uint32_t exit_num, zend_jit_registers_buf *regs); static int zend_jit_assign_to_variable(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_use_addr, zend_jit_addr var_addr, uint32_t var_info, uint32_t var_def_info, uint8_t val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr res_addr, zend_jit_addr ref_addr, bool check_exception); static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags); typedef struct _zend_jit_stub { const char *name; int (*stub)(zend_jit_ctx *jit); uint32_t flags; } zend_jit_stub; JIT_STUBS(JIT_STUB_FORWARD) static const zend_jit_stub zend_jit_stubs[] = { JIT_STUBS(JIT_STUB) }; #if defined(_WIN32) || defined(IR_TARGET_AARCH64) /* We keep addresses in SHM to share them between sepaeate processes (on Windows) or to support veneers (on AArch64) */ static void** zend_jit_stub_handlers = NULL; #else static void* zend_jit_stub_handlers[sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0])]; #endif #if defined(IR_TARGET_AARCH64) # ifdef __FreeBSD__ /* https://github.com/freebsd/freebsd-src/blob/c52ca7dd09066648b1cc40f758289404d68ab886/libexec/rtld-elf/aarch64/reloc.c#L180-L184 */ typedef struct TLSDescriptor { void* thunk; int index; size_t offset; } TLSDescriptor; # endif #define IR_HAS_VENEERS (1U<<31) /* IR_RESERVED_FLAG_1 */ static const void *zend_jit_get_veneer(ir_ctx *ctx, const void *addr) { int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]); for (i = 0; i < count; i++) { if (zend_jit_stub_handlers[i] == addr) { return zend_jit_stub_handlers[count + i]; } } if (((zend_jit_ctx*)ctx)->trace && (void*)addr >= dasm_buf && (void*)addr < dasm_end) { uint32_t exit_point = zend_jit_exit_point_by_addr(addr); if (exit_point != (uint32_t)-1) { zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace; ZEND_ASSERT(exit_point < t->exit_count); return (const void*)((char*)ctx->deoptimization_exits_base + (exit_point * 4)); } } return NULL; } static bool zend_jit_set_veneer(ir_ctx *ctx, const void *addr, const void *veneer) { int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]); uint32_t exit_point = zend_jit_exit_point_by_addr(addr); if (exit_point != (uint32_t)-1) { return 1; } for (i = 0; i < count; i++) { if (zend_jit_stub_handlers[i] == addr) { const void **ptr = (const void**)&zend_jit_stub_handlers[count + i]; *ptr = veneer; ctx->flags2 |= IR_HAS_VENEERS; #ifdef HAVE_CAPSTONE int64_t offset; if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { const char *name = ir_disasm_find_symbol((uint64_t)(uintptr_t)addr, &offset); if (name && !offset) { if (strstr(name, "@veneer") == NULL) { char *new_name; zend_spprintf(&new_name, 0, "%s@veneer", name); ir_disasm_add_symbol(new_name, (uint64_t)(uintptr_t)veneer, 4); efree(new_name); } else { ir_disasm_add_symbol(name, (uint64_t)(uintptr_t)veneer, 4); } } } #endif return 1; } } return 0; } static void zend_jit_commit_veneers(void) { int i, count = sizeof(zend_jit_stubs) / sizeof(zend_jit_stubs[0]); for (i = 0; i < count; i++) { if (zend_jit_stub_handlers[count + i]) { zend_jit_stub_handlers[i] = zend_jit_stub_handlers[count + i]; zend_jit_stub_handlers[count + i] = NULL; } } } #endif static bool zend_jit_prefer_const_addr_load(zend_jit_ctx *jit, uintptr_t addr) { #if defined(IR_TARGET_X86) return 0; /* always use immediate value */ #elif defined(IR_TARGET_X64) return addr > 0xffffffff; /* prefer loading long constant from memery */ #elif defined(IR_TARGET_AARCH64) return addr > 0xffff; #else # error "Unknown IR target" #endif } static const char* zend_reg_name(int8_t reg) { return ir_reg_name(reg, ir_reg_is_int(reg) ? IR_LONG : IR_DOUBLE); } /* IR helpers */ #ifdef ZTS static void * ZEND_FASTCALL zend_jit_get_tsrm_ls_cache(void) { return _tsrm_ls_cache; } static ir_ref jit_TLS(zend_jit_ctx *jit) { ZEND_ASSERT(jit->ctx.control); if (jit->tls) { /* Emit "TLS" once for basic block */ ir_insn *insn; ir_ref ref = jit->ctx.control; while (1) { if (ref == jit->tls) { return jit->tls; } insn = &jit->ctx.ir_base[ref]; if (insn->op >= IR_START || insn->op == IR_CALL) { break; } ref = insn->op1; } } if (tsrm_ls_cache_tcb_offset == 0 && tsrm_tls_index == -1) { jit->tls = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_tsrm_ls_cache)); } else { jit->tls = ir_TLS( tsrm_ls_cache_tcb_offset ? tsrm_ls_cache_tcb_offset : tsrm_tls_index, tsrm_ls_cache_tcb_offset ? IR_NULL : tsrm_tls_offset); } return jit->tls; } #endif static ir_ref jit_CONST_ADDR(zend_jit_ctx *jit, uintptr_t addr) { ir_ref ref; zval *zv; if (addr == 0) { return IR_NULL; } zv = zend_hash_index_lookup(&jit->addr_hash, addr); if (Z_TYPE_P(zv) == IS_LONG) { ref = Z_LVAL_P(zv); ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_ADDR, IR_ADDR)); } else { ref = ir_unique_const_addr(&jit->ctx, addr); ZVAL_LONG(zv, ref); } return ref; } static ir_ref jit_CONST_FUNC_PROTO(zend_jit_ctx *jit, uintptr_t addr, ir_ref proto) { ir_ref ref; ir_insn *insn; zval *zv; ZEND_ASSERT(addr != 0); zv = zend_hash_index_lookup(&jit->addr_hash, addr); if (Z_TYPE_P(zv) == IS_LONG) { ref = Z_LVAL_P(zv); ZEND_ASSERT(jit->ctx.ir_base[ref].opt == IR_OPT(IR_FUNC_ADDR, IR_ADDR) && jit->ctx.ir_base[ref].proto == proto); } else { ref = ir_unique_const_addr(&jit->ctx, addr); insn = &jit->ctx.ir_base[ref]; insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR); insn->proto = proto; ZVAL_LONG(zv, ref); } return ref; } static ir_ref jit_CONST_FUNC(zend_jit_ctx *jit, uintptr_t addr, uint16_t flags) { #if defined(IR_TARGET_X86) /* TODO: dummy prototype (only flags matter) ??? */ ir_ref proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0; #else ir_ref proto = 0; #endif return jit_CONST_FUNC_PROTO(jit, addr, proto); } static ir_ref jit_CONST_OPCODE_HANDLER_FUNC(zend_jit_ctx *jit, zend_vm_opcode_handler_t handler) { return jit_CONST_FUNC(jit, (uintptr_t)handler, IR_FASTCALL_FUNC); } static ir_ref jit_ADD_OFFSET(zend_jit_ctx *jit, ir_ref addr, uintptr_t offset) { if (offset) { addr = ir_ADD_A(addr, ir_CONST_ADDR(offset)); } return addr; } static ir_ref jit_EG_exception(zend_jit_ctx *jit) { #ifdef ZTS return jit_EG(exception); #else ir_ref ref = jit->eg_exception_addr; if (UNEXPECTED(!ref)) { ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)&EG(exception)); jit->eg_exception_addr = ref; } return ref; #endif } static ir_ref jit_STUB_ADDR(zend_jit_ctx *jit, jit_stub_id id) { ir_ref ref = jit->stub_addr[id]; if (UNEXPECTED(!ref)) { ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]); jit->stub_addr[id] = ref; } return ref; } static ir_ref jit_STUB_FUNC_ADDR(zend_jit_ctx *jit, jit_stub_id id, uint16_t flags) { ir_ref ref = jit->stub_addr[id]; ir_insn *insn; if (UNEXPECTED(!ref)) { ref = ir_unique_const_addr(&jit->ctx, (uintptr_t)zend_jit_stub_handlers[id]); insn = &jit->ctx.ir_base[ref]; insn->optx = IR_OPT(IR_FUNC_ADDR, IR_ADDR); #if defined(IR_TARGET_X86) /* TODO: dummy prototype (only flags matter) ??? */ insn->proto = flags ? ir_proto_0(&jit->ctx, flags, IR_I32) : 0; #else insn->proto = 0; #endif jit->stub_addr[id] = ref; } return ref; } static void jit_SNAPSHOT(zend_jit_ctx *jit, ir_ref addr) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame)) { const void *ptr = (const void*)jit->ctx.ir_base[addr].val.addr; const zend_op_array *op_array = &JIT_G(current_frame)->func->op_array; uint32_t stack_size = op_array->last_var + op_array->T; if (ptr == zend_jit_stub_handlers[jit_stub_exception_handler] || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_undef] || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op1_op2] || ptr == zend_jit_stub_handlers[jit_stub_exception_handler_free_op2] || ptr == zend_jit_stub_handlers[jit_stub_interrupt_handler] || ptr == zend_jit_stub_handlers[jit_stub_leave_function_handler] || ptr == zend_jit_stub_handlers[jit_stub_negative_shift] || ptr == zend_jit_stub_handlers[jit_stub_mod_by_zero] || ptr == zend_jit_stub_handlers[jit_stub_invalid_this] || ptr == zend_jit_stub_handlers[jit_stub_undefined_function] || ptr == zend_jit_stub_handlers[jit_stub_throw_cannot_pass_by_ref] || ptr == zend_jit_stub_handlers[jit_stub_icall_throw] || ptr == zend_jit_stub_handlers[jit_stub_leave_throw] || ptr == zend_jit_stub_handlers[jit_stub_trace_halt] || ptr == zend_jit_stub_handlers[jit_stub_trace_escape]) { /* This is a GUARD that trigger exit through a stub code (without deoptimization) */ return; } /* Check if we need snapshot entries for polymorphic method call */ zend_jit_trace_info *t = jit->trace; uint32_t exit_point = 0, n = 0; if (addr < 0) { /* addr is not always the address of the *last* exit point, * so we can not optimize this to 'exit_point = t->exit_count-1' */ exit_point = zend_jit_exit_point_by_addr(ptr); ZEND_ASSERT(exit_point != -1); if (t->exit_info[exit_point].flags & ZEND_JIT_EXIT_METHOD_CALL) { n = 2; } } if (stack_size || n) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t snapshot_size, i; snapshot_size = stack_size; while (snapshot_size > 0) { ir_ref ref = STACK_REF(stack, snapshot_size - 1); if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, snapshot_size - 1) & (/*ZREG_LOAD|*/ZREG_STORE))) { snapshot_size--; } else { break; } } if (snapshot_size || n) { ir_ref snapshot; snapshot = ir_SNAPSHOT(snapshot_size + n); for (i = 0; i < snapshot_size; i++) { ir_ref ref = STACK_REF(stack, i); if (!ref || ref == IR_NULL || (STACK_FLAGS(stack, i) & (/*ZREG_LOAD|*/ZREG_STORE))) { ref = IR_UNUSED; } ir_SNAPSHOT_SET_OP(snapshot, i + 1, ref); } if (n) { ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 1, t->exit_info[exit_point].poly_func.ref); ir_SNAPSHOT_SET_OP(snapshot, snapshot_size + 2, t->exit_info[exit_point].poly_this.ref); } } } } } static int32_t _add_trace_const(zend_jit_trace_info *t, int64_t val) { int32_t i; for (i = 0; i < t->consts_count; i++) { if (t->constants[i].i == val) { return i; } } ZEND_ASSERT(i < 0x7fffffff); t->consts_count = i + 1; t->constants = erealloc(t->constants, (i + 1) * sizeof(zend_jit_exit_const)); t->constants[i].i = val; return i; } uint32_t zend_jit_duplicate_exit_point(ir_ctx *ctx, zend_jit_trace_info *t, uint32_t exit_point, ir_ref snapshot_ref) { uint32_t stack_size, stack_offset; uint32_t new_exit_point = t->exit_count; if (new_exit_point >= ZEND_JIT_TRACE_MAX_EXITS) { ctx->status = -ZEND_JIT_TRACE_STOP_TOO_MANY_EXITS; return exit_point; } t->exit_count++; memcpy(&t->exit_info[new_exit_point], &t->exit_info[exit_point], sizeof(zend_jit_trace_exit_info)); stack_size = t->exit_info[new_exit_point].stack_size; if (stack_size != 0) { stack_offset = t->stack_map_size; t->stack_map_size += stack_size; // TODO: reduce number of reallocations ??? t->stack_map = erealloc(t->stack_map, t->stack_map_size * sizeof(zend_jit_trace_stack)); memcpy(t->stack_map + stack_offset, t->stack_map + t->exit_info[new_exit_point].stack_offset, stack_size * sizeof(zend_jit_trace_stack)); t->exit_info[new_exit_point].stack_offset = stack_offset; } t->exit_info[new_exit_point].flags &= ~ZEND_JIT_EXIT_FIXED; return new_exit_point; } static void zend_jit_resolve_ref_snapshot(zend_jit_ref_snapshot *dest, ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, int op) { int8_t *reg_ops = ctx->regs[snapshot_ref]; ZEND_ASSERT(reg_ops[op] != ZREG_NONE); int8_t reg = reg_ops[op]; int32_t offset; if (IR_REG_SPILLED(reg)) { reg = ((ctx->flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP) | IR_REG_SPILL_LOAD; offset = ir_get_spill_slot_offset(ctx, ir_insn_op(snapshot, op)); } else { offset = 0; } dest->reg = reg; dest->offset = offset; } static bool zend_jit_ref_snapshot_equals(const zend_jit_ref_snapshot *a, const zend_jit_ref_snapshot *b) { return a->reg == b->reg && (!IR_REG_SPILLED(a->reg) || (a->offset == b->offset)); } void *zend_jit_snapshot_handler(ir_ctx *ctx, ir_ref snapshot_ref, ir_insn *snapshot, void *addr) { zend_jit_trace_info *t = ((zend_jit_ctx*)ctx)->trace; uint32_t exit_point, exit_flags; ir_ref n = snapshot->inputs_count; ir_ref i; exit_point = zend_jit_exit_point_by_addr(addr); ZEND_ASSERT(exit_point < t->exit_count); exit_flags = t->exit_info[exit_point].flags; if (exit_flags & ZEND_JIT_EXIT_METHOD_CALL) { zend_jit_ref_snapshot func, this; zend_jit_resolve_ref_snapshot(&func, ctx, snapshot_ref, snapshot, n - 1); zend_jit_resolve_ref_snapshot(&this, ctx, snapshot_ref, snapshot, n); if ((exit_flags & ZEND_JIT_EXIT_FIXED) && (!zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_func, &func) || !zend_jit_ref_snapshot_equals(&t->exit_info[exit_point].poly_this, &this))) { exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); addr = (void*)zend_jit_trace_get_exit_addr(exit_point); exit_flags &= ~ZEND_JIT_EXIT_FIXED; } t->exit_info[exit_point].poly_func = func; t->exit_info[exit_point].poly_this = this; n -= 2; } for (i = 2; i <= n; i++) { ir_ref ref = ir_insn_op(snapshot, i); if (ref) { int8_t *reg_ops = ctx->regs[snapshot_ref]; int8_t reg = reg_ops[i]; ir_ref var = i - 2; ZEND_ASSERT(var < t->exit_info[exit_point].stack_size); if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags == ZREG_ZVAL_COPY) { ZEND_ASSERT(reg != ZREG_NONE); if ((exit_flags & ZEND_JIT_EXIT_FIXED) && t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) { exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); addr = (void*)zend_jit_trace_get_exit_addr(exit_point); exit_flags &= ~ZEND_JIT_EXIT_FIXED; } t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg); } else if (t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_CONST) { ZEND_ASSERT(t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_LONG || t->stack_map[t->exit_info[exit_point].stack_offset + var].type == IS_DOUBLE); if (ref > 0) { if (reg != ZREG_NONE) { if (reg & IR_REG_SPILL_LOAD) { ZEND_ASSERT(!(reg & IR_REG_SPILL_SPECIAL)); /* spill slot on a CPU stack */ if ((exit_flags & ZEND_JIT_EXIT_FIXED) && (t->stack_map[t->exit_info[exit_point].stack_offset + var].ref != ref || t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE || !(t->stack_map[t->exit_info[exit_point].stack_offset + var].flags & ZREG_SPILL_SLOT))) { exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); addr = (void*)zend_jit_trace_get_exit_addr(exit_point); exit_flags &= ~ZEND_JIT_EXIT_FIXED; } t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = ref; t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE; t->stack_map[t->exit_info[exit_point].stack_offset + var].flags |= ZREG_SPILL_SLOT; } else if (reg & IR_REG_SPILL_SPECIAL) { /* spill slot on a VM stack */ if ((exit_flags & ZEND_JIT_EXIT_FIXED) && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) { exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); addr = (void*)zend_jit_trace_get_exit_addr(exit_point); exit_flags &= ~ZEND_JIT_EXIT_FIXED; } t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = ZREG_NONE; t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY; } else { if ((exit_flags & ZEND_JIT_EXIT_FIXED) && t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != IR_REG_NUM(reg)) { exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); addr = (void*)zend_jit_trace_get_exit_addr(exit_point); exit_flags &= ~ZEND_JIT_EXIT_FIXED; } t->stack_map[t->exit_info[exit_point].stack_offset + var].reg = IR_REG_NUM(reg); } } else { if ((exit_flags & ZEND_JIT_EXIT_FIXED) && (t->stack_map[t->exit_info[exit_point].stack_offset + var].reg != ZREG_NONE || t->stack_map[t->exit_info[exit_point].stack_offset + var].flags != ZREG_TYPE_ONLY)) { exit_point = zend_jit_duplicate_exit_point(ctx, t, exit_point, snapshot_ref); addr = (void*)zend_jit_trace_get_exit_addr(exit_point); exit_flags &= ~ZEND_JIT_EXIT_FIXED; } t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_TYPE_ONLY; } } else if (!(exit_flags & ZEND_JIT_EXIT_FIXED)) { int32_t idx = _add_trace_const(t, ctx->ir_base[ref].val.i64); t->stack_map[t->exit_info[exit_point].stack_offset + var].flags = ZREG_CONST; t->stack_map[t->exit_info[exit_point].stack_offset + var].ref = idx; } } } } t->exit_info[exit_point].flags |= ZEND_JIT_EXIT_FIXED; return addr; } static void jit_SIDE_EXIT(zend_jit_ctx *jit, ir_ref addr) { jit_SNAPSHOT(jit, addr); ir_IJMP(addr); } /* PHP JIT helpers */ static ir_ref jit_EMALLOC(zend_jit_ctx *jit, size_t size, const zend_op_array *op_array, const zend_op *opline) { #if ZEND_DEBUG return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size), op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL, ir_CONST_U32(opline ? opline->lineno : 0), IR_NULL, ir_CONST_U32(0)); #elif defined(HAVE_BUILTIN_CONSTANT_P) if (size > 24 && size <= 32) { return ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_emalloc_32)); } else { return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size)); } #else return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_emalloc), ir_CONST_ADDR(size)); #endif } static ir_ref jit_EFREE(zend_jit_ctx *jit, ir_ref ptr, size_t size, const zend_op_array *op_array, const zend_op *opline) { #if ZEND_DEBUG return ir_CALL_5(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr, op_array && op_array->filename ? ir_CONST_ADDR(op_array->filename->val) : IR_NULL, ir_CONST_U32(opline ? opline->lineno : 0), IR_NULL, ir_CONST_U32(0)); #elif defined(HAVE_BUILTIN_CONSTANT_P) if (size > 24 && size <= 32) { return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree_32), ptr); } else { return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr); } #else return ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(_efree), ptr); #endif } static ir_ref jit_FP(zend_jit_ctx *jit) { ZEND_ASSERT(jit->ctx.control); if (jit->fp == IR_UNUSED) { /* Emit "RLOAD FP" once for basic block */ jit->fp = ir_RLOAD_A(ZREG_FP); } else { ir_insn *insn; ir_ref ref = jit->ctx.control; while (1) { if (ref == jit->fp) { break; } insn = &jit->ctx.ir_base[ref]; if (insn->op >= IR_START || insn->op == IR_CALL) { jit->fp = ir_RLOAD_A(ZREG_FP); break; } ref = insn->op1; } } return jit->fp; } static void jit_STORE_FP(zend_jit_ctx *jit, ir_ref ref) { ir_RSTORE(ZREG_FP, ref); jit->fp = IR_UNUSED; } static ir_ref jit_IP(zend_jit_ctx *jit) { return ir_RLOAD_A(ZREG_IP); } static void jit_STORE_IP(zend_jit_ctx *jit, ir_ref ref) { ir_RSTORE(ZREG_IP, ref); } static ir_ref jit_IP32(zend_jit_ctx *jit) { return ir_RLOAD_U32(ZREG_IP); } static void jit_LOAD_IP_ADDR(zend_jit_ctx *jit, const zend_op *target) { jit_STORE_IP(jit, ir_CONST_ADDR(target)); } static void zend_jit_track_last_valid_opline(zend_jit_ctx *jit) { jit->use_last_valid_opline = 0; jit->track_last_valid_opline = 1; } static void zend_jit_use_last_valid_opline(zend_jit_ctx *jit) { if (jit->track_last_valid_opline) { jit->use_last_valid_opline = 1; jit->track_last_valid_opline = 0; } } static bool zend_jit_trace_uses_initial_ip(zend_jit_ctx *jit) { return jit->use_last_valid_opline; } static void zend_jit_set_last_valid_opline(zend_jit_ctx *jit, const zend_op *opline) { if (!jit->reuse_ip) { jit->track_last_valid_opline = 1; jit->last_valid_opline = opline; } } static void zend_jit_reset_last_valid_opline(zend_jit_ctx *jit) { jit->track_last_valid_opline = 0; jit->last_valid_opline = NULL; } static void zend_jit_start_reuse_ip(zend_jit_ctx *jit) { zend_jit_reset_last_valid_opline(jit); jit->reuse_ip = 1; } static int zend_jit_reuse_ip(zend_jit_ctx *jit) { if (!jit->reuse_ip) { zend_jit_start_reuse_ip(jit); // RX = EX(call); jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call))); } return 1; } static void zend_jit_stop_reuse_ip(zend_jit_ctx *jit) { jit->reuse_ip = 0; } static int zend_jit_save_call_chain(zend_jit_ctx *jit, uint32_t call_level) { ir_ref rx, call; if (call_level == 1) { // JIT: call = NULL; call = IR_NULL; } else { // JIT: call = EX(call); call = ir_LOAD_A(jit_EX(call)); } rx = jit_IP(jit); // JIT: call->prev_execute_data = call; ir_STORE(jit_CALL(rx, prev_execute_data), call); // JIT: EX(call) = call; ir_STORE(jit_EX(call), rx); jit->delayed_call_level = 0; delayed_call_chain = 0; return 1; } static int zend_jit_set_ip(zend_jit_ctx *jit, const zend_op *target) { ir_ref ref; if (jit->delayed_call_level) { if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } if (jit->last_valid_opline) { zend_jit_use_last_valid_opline(jit); if (jit->last_valid_opline != target) { ref = jit_IP(jit); if (target > jit->last_valid_opline) { ref = ir_ADD_OFFSET(ref, (uintptr_t)target - (uintptr_t)jit->last_valid_opline); } else { ref = ir_SUB_A(ref, ir_CONST_ADDR((uintptr_t)jit->last_valid_opline - (uintptr_t)target)); } jit_STORE_IP(jit, ref); } } else { jit_STORE_IP(jit, ir_CONST_ADDR(target)); } jit->reuse_ip = 0; zend_jit_set_last_valid_opline(jit, target); return 1; } static void jit_SET_EX_OPLINE(zend_jit_ctx *jit, const zend_op *target) { if (jit->last_valid_opline == target) { zend_jit_use_last_valid_opline(jit); // EX(opline) = opline ir_STORE(jit_EX(opline), jit_IP(jit)); } else { ir_STORE(jit_EX(opline), ir_CONST_ADDR(target)); } } static ir_ref jit_ZVAL_ADDR(zend_jit_ctx *jit, zend_jit_addr addr) { if (Z_MODE(addr) == IS_MEM_ZVAL) { ir_ref reg; if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); } else if (Z_REG(addr) == ZREG_RX) { reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } return ir_ADD_OFFSET(reg, Z_OFFSET(addr)); } else if (Z_MODE(addr) == IS_REF_ZVAL) { return Z_IR_REF(addr); } else { ZEND_ASSERT(Z_MODE(addr) == IS_CONST_ZVAL); return ir_CONST_ADDR(Z_ZV(addr)); } } static ir_ref jit_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref) { return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type))); } static ir_ref jit_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr) { if (Z_MODE(addr) == IS_CONST_ZVAL) { return ir_CONST_U8(Z_TYPE_P(Z_ZV(addr))); } else if (Z_MODE(addr) == IS_MEM_ZVAL) { ir_ref reg; ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); } else if (Z_REG(addr) == ZREG_RX) { reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type))); } else { return jit_Z_TYPE_ref(jit, jit_ZVAL_ADDR(jit, addr)); } } static ir_ref jit_Z_TYPE_FLAGS_ref(zend_jit_ctx *jit, ir_ref ref) { return ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zval, u1.v.type_flags))); } static ir_ref jit_Z_TYPE_FLAGS(zend_jit_ctx *jit, zend_jit_addr addr) { if (Z_MODE(addr) == IS_CONST_ZVAL) { return ir_CONST_U8(Z_TYPE_FLAGS_P(Z_ZV(addr))); } else if (Z_MODE(addr) == IS_MEM_ZVAL) { ir_ref reg; ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); } else if (Z_REG(addr) == ZREG_RX) { reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } return ir_LOAD_U8(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.v.type_flags))); } else { return jit_Z_TYPE_FLAGS_ref(jit, jit_ZVAL_ADDR(jit, addr)); } } static ir_ref jit_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref) { return ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info))); } static ir_ref jit_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr) { if (Z_MODE(addr) == IS_CONST_ZVAL) { return ir_CONST_U32(Z_TYPE_INFO_P(Z_ZV(addr))); } else if (Z_MODE(addr) == IS_MEM_ZVAL) { ir_ref reg; ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); } else if (Z_REG(addr) == ZREG_RX) { reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } return ir_LOAD_U32(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info))); } else { return jit_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr)); } } static void jit_set_Z_TYPE_INFO_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type_info) { ir_STORE(ir_ADD_OFFSET(ref, offsetof(zval, u1.type_info)), type_info); } static void jit_set_Z_TYPE_INFO_ex(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref type_info) { if (Z_MODE(addr) == IS_MEM_ZVAL) { ir_ref reg; ZEND_ASSERT(Z_MODE(addr) == IS_MEM_ZVAL); if (Z_REG(addr) == ZREG_FP) { reg = jit_FP(jit); } else if (Z_REG(addr) == ZREG_RX) { reg = jit_IP(jit); } else { ZEND_UNREACHABLE(); } ir_STORE(ir_ADD_OFFSET(reg, Z_OFFSET(addr) + offsetof(zval, u1.type_info)), type_info); } else { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, addr), type_info); } } static void jit_set_Z_TYPE_INFO(zend_jit_ctx *jit, zend_jit_addr addr, uint32_t type_info) { if (type_info < IS_STRING && Z_MODE(addr) == IS_MEM_ZVAL && Z_REG(addr) == ZREG_FP && JIT_G(current_frame) && STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(addr))) == type_info) { /* type is already set */ return; } jit_set_Z_TYPE_INFO_ex(jit, addr, ir_CONST_U32(type_info)); } static ir_ref jit_if_Z_TYPE_ref(zend_jit_ctx *jit, ir_ref ref, ir_ref type) { return ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), type)); } static ir_ref jit_if_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type) { ZEND_ASSERT(type != IS_UNDEF); return ir_IF(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type))); } static ir_ref jit_if_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type) { ir_ref ref = jit_Z_TYPE(jit, addr); if (type != IS_UNDEF) { ref = ir_NE(ref, ir_CONST_U8(type)); } return ir_IF(ref); } static void jit_guard_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr) { ir_ref ref = jit_Z_TYPE(jit, addr); if (type != IS_UNDEF) { ir_GUARD(ir_EQ(ref, ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } } static void jit_guard_not_Z_TYPE(zend_jit_ctx *jit, zend_jit_addr addr, uint8_t type, const void *exit_addr) { ir_ref ref = jit_Z_TYPE(jit, addr); if (type != IS_UNDEF) { ref = ir_NE(ref, ir_CONST_U8(type)); } ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } static ir_ref jit_if_REFCOUNTED(zend_jit_ctx *jit, zend_jit_addr addr) { return ir_IF(jit_Z_TYPE_FLAGS(jit, addr)); } static ir_ref jit_if_COLLECTABLE_ref(zend_jit_ctx *jit, ir_ref addr_ref) { return ir_IF(ir_AND_U8(jit_Z_TYPE_FLAGS_ref(jit, addr_ref), ir_CONST_U8(IS_TYPE_COLLECTABLE))); } static ir_ref jit_Z_LVAL_ref(zend_jit_ctx *jit, ir_ref ref) { return ir_LOAD_L(ref); } static ir_ref jit_Z_DVAL_ref(zend_jit_ctx *jit, ir_ref ref) { return ir_LOAD_D(ref); } static bool zend_jit_spilling_may_cause_conflict(zend_jit_ctx *jit, int var, ir_ref val) { if (jit->ctx.ir_base[val].op == IR_RLOAD) { /* Deoptimization */ return 0; } // if (jit->ctx.ir_base[val].op == IR_LOAD // && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD // && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD // && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP // && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2) // && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr == (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var)) { // /* LOAD from the same location (the LOAD is pinned) */ // // TODO: should be anti-dependent with the following stores ??? // return 0; // } if (jit->ssa->vars[var].var < jit->current_op_array->last_var) { /* IS_CV */ if (jit->ctx.ir_base[val].op == IR_LOAD && jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op == IR_ADD && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op == IR_RLOAD && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op1].op2 == ZREG_FP && IR_IS_CONST_REF(jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2) && jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr != (uintptr_t)EX_NUM_TO_VAR(jit->ssa->vars[var].var) && EX_VAR_TO_NUM(jit->ctx.ir_base[jit->ctx.ir_base[jit->ctx.ir_base[val].op2].op2].val.addr) < jit->current_op_array->last_var) { /* binding between different CVs may cause spill conflict */ return 1; } else if (jit->ssa->vars[var].definition >= 0 && jit->ssa->ops[jit->ssa->vars[var].definition].op1_def == var && jit->ssa->ops[jit->ssa->vars[var].definition].op1_use >= 0 && jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].no_val && jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].definition_phi && (jit->ssa->cfg.blocks[jit->ssa->vars[jit->ssa->ops[jit->ssa->vars[var].definition].op1_use].definition_phi->block].flags & ZEND_BB_LOOP_HEADER)) { /* Avoid moving spill store out of loop */ return 1; } else if (jit->ssa->vars[var].definition >= 0 && jit->ssa->ops[jit->ssa->vars[var].definition].op1_def == var && jit->ssa->ops[jit->ssa->vars[var].definition].op1_use >= 0 && jit->ssa->ops[jit->ssa->vars[var].definition].op2_use >= 0 && jit->ra[jit->ssa->ops[jit->ssa->vars[var].definition].op2_use].ref == val) { /* Avoid spill conflict between of ASSIGN.op1_def and ASSIGN.op1_use */ return 1; } return 0; } return 1; } static void zend_jit_def_reg(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref val) { int var; ZEND_ASSERT(Z_MODE(addr) == IS_REG); var = Z_SSA_VAR(addr); if (var == jit->delay_var) { ir_refs_add(jit->delay_refs, val); return; } ZEND_ASSERT(jit->ra && jit->ra[var].ref == IR_NULL); /* Negative "var" has special meaning for IR */ if (val > 0) { if (jit->ctx.binding) { ir_ref old = ir_binding_find(&jit->ctx, val); if (old && old != -EX_NUM_TO_VAR(jit->ssa->vars[var].var)) { val = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[val].type), val, 1); } } if (!zend_jit_spilling_may_cause_conflict(jit, var, val)) { val = ir_bind(&jit->ctx, -EX_NUM_TO_VAR(jit->ssa->vars[var].var), val); } } jit->ra[var].ref = val; if (jit->ra[var].flags & ZREG_FORWARD) { zend_ssa_phi *phi = jit->ssa->vars[var].phi_use_chain; zend_basic_block *bb; int n, j, *p; ir_ref *q; jit->ra[var].flags &= ~ZREG_FORWARD; while (phi != NULL) { zend_ssa_phi *dst_phi = phi; int src_var = var; if (dst_phi->pi >= 0) { jit->ra[src_var].ref = val; src_var = dst_phi->ssa_var; if (!(jit->ra[src_var].flags & ZREG_FORWARD)) { phi = zend_ssa_next_use_phi(jit->ssa, var, phi); continue; } dst_phi = jit->ssa->vars[src_var].phi_use_chain; ZEND_ASSERT(dst_phi != NULL && "reg forwarding"); ZEND_ASSERT(!zend_ssa_next_use_phi(jit->ssa, src_var, dst_phi) && "reg forwarding"); jit->ra[src_var].flags &= ~ZREG_FORWARD; } if (jit->ra[dst_phi->ssa_var].ref > 0) { ir_insn *phi_insn = &jit->ctx.ir_base[jit->ra[dst_phi->ssa_var].ref]; if (phi_insn->op == IR_PHI) { // ZEND_ASSERT(ir_operands_count(ctx, phi_insn) == n + 1); bb = &jit->ssa->cfg.blocks[dst_phi->block]; n = bb->predecessors_count; for (j = 0, p = &dst_phi->sources[0], q = phi_insn->ops + 2; j < n; j++, p++, q++) { if (*p == src_var) { *q = val; } } } } phi = zend_ssa_next_use_phi(jit->ssa, var, phi); } } } static ir_ref zend_jit_use_reg(zend_jit_ctx *jit, zend_jit_addr addr) { int var = Z_SSA_VAR(addr); ZEND_ASSERT(Z_MODE(addr) == IS_REG); ZEND_ASSERT(jit->ra && jit->ra[var].ref); if (jit->ra[var].ref == IR_NULL) { zend_jit_addr mem_addr; ir_ref ref; ZEND_ASSERT(jit->ra[var].flags & ZREG_LOAD); mem_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(jit->ssa->vars[var].var)); if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_LONG) { ref = jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr)); } else if ((jit->ssa->var_info[var].type & MAY_BE_ANY) == MAY_BE_DOUBLE) { ref = jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, mem_addr)); } else { ZEND_UNREACHABLE(); } zend_jit_def_reg(jit, addr, ref); return ref; } return jit->ra[Z_SSA_VAR(addr)].ref; } static void zend_jit_gen_pi(zend_jit_ctx *jit, zend_ssa_phi *phi) { int src_var = phi->sources[0]; int dst_var = phi->ssa_var; ZEND_ASSERT(phi->pi >= 0); ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD)); ZEND_ASSERT(jit->ra[src_var].ref); if (jit->ra[src_var].ref == IR_NULL) { /* Not defined yet */ if (jit->ssa->vars[dst_var].use_chain < 0 && jit->ssa->vars[dst_var].phi_use_chain) { zend_ssa_phi *phi = jit->ssa->vars[dst_var].phi_use_chain; if (!zend_ssa_next_use_phi(jit->ssa, dst_var, phi)) { /* This is a Pi forwarded to Phi */ jit->ra[src_var].flags |= ZREG_FORWARD; return; } } ZEND_ASSERT(0 && "Not defined Pi source"); } /* Reuse register */ zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var))); } static void zend_jit_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi) { int dst_var = phi->ssa_var; zend_basic_block *bb = &jit->ssa->cfg.blocks[phi->block]; int n = bb->predecessors_count; int i; ir_type type = (jit->ssa->var_info[phi->ssa_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE; ir_ref merge = jit->bb_start_ref[phi->block]; ir_ref ref; ir_ref old_insns_count = jit->ctx.insns_count; ir_ref same_src_ref = IR_UNUSED; bool phi_inputs_are_the_same = 1; ZEND_ASSERT(phi->pi < 0); ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD)); ZEND_ASSERT(merge); ZEND_ASSERT(jit->ctx.ir_base[merge].op == IR_MERGE || jit->ctx.ir_base[merge].op == IR_LOOP_BEGIN); ZEND_ASSERT(n == jit->ctx.ir_base[merge].inputs_count); ref = ir_emit_N(&jit->ctx, IR_OPT(IR_PHI, type), n + 1); ir_set_op(&jit->ctx, ref, 1, merge); for (i = 0; i < n; i++) { int src_var = phi->sources[i]; ZEND_ASSERT(jit->ra[src_var].ref); if (jit->ra[src_var].ref == IR_NULL) { jit->ra[src_var].flags |= ZREG_FORWARD; phi_inputs_are_the_same = 0; } else { ir_ref src_ref = zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)); if (i == 0) { same_src_ref = src_ref; } else if (same_src_ref != src_ref) { phi_inputs_are_the_same = 0; } ir_set_op(&jit->ctx, ref, i + 2, src_ref); } } if (phi_inputs_are_the_same) { ref = same_src_ref; jit->ctx.insns_count = old_insns_count; } zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref); } static ir_ref jit_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr) { if (Z_MODE(addr) == IS_CONST_ZVAL) { return ir_CONST_LONG(Z_LVAL_P(Z_ZV(addr))); } else if (Z_MODE(addr) == IS_REG) { return zend_jit_use_reg(jit, addr); } else { return jit_Z_LVAL_ref(jit, jit_ZVAL_ADDR(jit, addr)); } } static void jit_set_Z_LVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval) { if (Z_MODE(addr) == IS_REG) { zend_jit_def_reg(jit, addr, lval); } else { ir_STORE(jit_ZVAL_ADDR(jit, addr), lval); } } #if SIZEOF_ZEND_LONG == 4 static ir_ref jit_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr) { if (Z_MODE(addr) == IS_CONST_ZVAL) { return ir_CONST_U32((Z_ZV(addr))->value.ww.w2); } else { return ir_LOAD_L(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2))); } } static void jit_set_Z_W2(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref lval) { ir_STORE(ir_ADD_OFFSET(jit_ZVAL_ADDR(jit, addr), offsetof(zval, value.ww.w2)), lval); } #endif static ir_ref jit_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr) { if (Z_MODE(addr) == IS_CONST_ZVAL) { return ir_CONST_DOUBLE(Z_DVAL_P(Z_ZV(addr))); } else if (Z_MODE(addr) == IS_REG) { return zend_jit_use_reg(jit, addr); } else { return jit_Z_DVAL_ref(jit, jit_ZVAL_ADDR(jit, addr)); } } static void jit_set_Z_DVAL(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref dval) { if (Z_MODE(addr) == IS_REG) { zend_jit_def_reg(jit, addr, dval); } else { ir_STORE(jit_ZVAL_ADDR(jit, addr), dval); } } static ir_ref jit_Z_PTR_ref(zend_jit_ctx *jit, ir_ref ref) { return ir_LOAD_A(ref); } static ir_ref jit_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr) { if (Z_MODE(addr) == IS_CONST_ZVAL) { return ir_CONST_ADDR(Z_PTR_P(Z_ZV(addr))); } else { return jit_Z_PTR_ref(jit, jit_ZVAL_ADDR(jit, addr)); } } static void jit_set_Z_PTR(zend_jit_ctx *jit, zend_jit_addr addr, ir_ref ptr) { ir_STORE(jit_ZVAL_ADDR(jit, addr), ptr); } static ir_ref jit_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref) { return ir_LOAD_U32(ref); } static void jit_set_GC_REFCOUNT(zend_jit_ctx *jit, ir_ref ref, uint32_t refcount) { ir_STORE(ref, ir_CONST_U32(refcount)); } static void jit_GC_ADDREF(zend_jit_ctx *jit, ir_ref ref) { ir_STORE(ref, ir_ADD_U32(ir_LOAD_U32(ref), ir_CONST_U32(1))); } static void jit_GC_ADDREF2(zend_jit_ctx *jit, ir_ref ref) { ir_ref counter = ir_LOAD_U32(ref); ir_STORE(ref, ir_ADD_U32(counter, ir_CONST_U32(2))); } static ir_ref jit_GC_DELREF(zend_jit_ctx *jit, ir_ref ref) { ir_ref counter = ir_LOAD_U32(ref); counter = ir_SUB_U32(counter, ir_CONST_U32(1)); ir_STORE(ref, counter); return counter; } static ir_ref jit_if_GC_MAY_NOT_LEAK(zend_jit_ctx *jit, ir_ref ref) { return ir_IF( ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_refcounted, gc.u.type_info))), ir_CONST_U32(GC_INFO_MASK | (GC_NOT_COLLECTABLE << GC_FLAGS_SHIFT)))); } static void jit_ZVAL_COPY_CONST(zend_jit_ctx *jit, zend_jit_addr dst, uint32_t dst_info, uint32_t dst_def_info, zval *zv, bool addref) { ir_ref ref = IR_UNUSED; if (Z_TYPE_P(zv) > IS_TRUE) { if (Z_TYPE_P(zv) == IS_DOUBLE) { jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE(Z_DVAL_P(zv))); } else if (Z_TYPE_P(zv) == IS_LONG && dst_def_info == MAY_BE_DOUBLE) { jit_set_Z_DVAL(jit, dst, ir_CONST_DOUBLE((double)Z_LVAL_P(zv))); } else if (Z_TYPE_P(zv) == IS_LONG) { jit_set_Z_LVAL(jit, dst, ir_CONST_LONG(Z_LVAL_P(zv))); } else { ref = ir_CONST_ADDR(Z_PTR_P(zv)); jit_set_Z_PTR(jit, dst, ref); if (addref && Z_REFCOUNTED_P(zv)) { jit_GC_ADDREF(jit, ref); } } } if (Z_MODE(dst) != IS_REG) { if (dst_def_info == MAY_BE_DOUBLE) { if ((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); } } else if (((dst_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) != (1<last_valid_opline == opline) { ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_interrupt_handler)); } else { ir_ref if_timeout = ir_IF(ref); ir_IF_TRUE_cold(if_timeout); jit_LOAD_IP_ADDR(jit, opline); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_interrupt_handler)); ir_IF_FALSE(if_timeout); } } static void zend_jit_vm_enter(zend_jit_ctx *jit, ir_ref to_opline) { // ZEND_VM_ENTER() ir_RETURN(ir_OR_A(to_opline, ir_CONST_ADDR(ZEND_VM_ENTER_BIT))); } static void zend_jit_vm_leave(zend_jit_ctx *jit, ir_ref to_opline) { // ZEND_VM_LEAVE() ir_RETURN(ir_OR_A(to_opline, ir_CONST_ADDR(ZEND_VM_ENTER_BIT))); } static void zend_jit_tailcall_handler(zend_jit_ctx *jit, ir_ref handler) { #if defined(IR_TARGET_X86) if (!IR_IS_CONST_REF(handler)) { handler = ir_CAST_OPCODE_HANDLER_FUNC(handler); } #endif if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { ir_TAILCALL(IR_OPCODE_HANDLER_RET, handler); } else { ir_TAILCALL_2(IR_ADDR, handler, jit_FP(jit), jit_IP(jit)); } } /* stubs */ static int zend_jit_exception_handler_stub(zend_jit_ctx *jit) { if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) { zend_vm_opcode_handler_func_t handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(EG(exception_op)); ir_CALL(IR_VOID, ir_CONST_FUNC(handler)); ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); } else { zend_vm_opcode_handler_t handler = EG(exception_op)->handler; if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { zend_jit_tailcall_handler(jit, ir_CONST_OPCODE_HANDLER_FUNC(handler)); } else { ir_ref ref = ir_CALL_2(IR_ADDR, ir_CONST_OPCODE_HANDLER_FUNC(handler), jit_FP(jit), jit_IP(jit)); zend_jit_vm_enter(jit, ref); } } return 1; } static int zend_jit_exception_handler_undef_stub(zend_jit_ctx *jit) { ir_ref ref, result_type, if_result_used; ref = jit_EG(opline_before_exception); result_type = ir_LOAD_U8(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result_type))); if_result_used = ir_IF(ir_AND_U8(result_type, ir_CONST_U8(IS_TMP_VAR|IS_VAR))); ir_IF_TRUE(if_result_used); ref = ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref), offsetof(zend_op, result.var))); if (sizeof(void*) == 8) { ref = ir_ZEXT_A(ref); } ir_STORE(ir_ADD_OFFSET(ir_ADD_A(jit_FP(jit), ref), offsetof(zval, u1.type_info)), ir_CONST_U32(IS_UNDEF)); ir_MERGE_WITH_EMPTY_FALSE(if_result_used); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); return 1; } static int zend_jit_exception_handler_free_op1_op2_stub(zend_jit_ctx *jit) { ir_ref ref, if_dtor; zend_jit_addr var_addr; ref = ir_LOAD_A(jit_EG(opline_before_exception)); if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op1_type))), ir_CONST_U8(IS_TMP_VAR|IS_VAR))); ir_IF_TRUE(if_dtor); ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op1.var))); if (sizeof(void*) == 8) { ref = ir_ZEXT_A(ref); } ref = ir_ADD_A(jit_FP(jit), ref); var_addr = ZEND_ADDR_REF_ZVAL(ref); jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL); ir_MERGE_WITH_EMPTY_FALSE(if_dtor); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); return 1; } static int zend_jit_exception_handler_free_op2_stub(zend_jit_ctx *jit) { ir_ref ref, if_dtor; zend_jit_addr var_addr; ref = ir_LOAD_A(jit_EG(opline_before_exception)); if_dtor = ir_IF(ir_AND_U8(ir_LOAD_U8(ir_ADD_OFFSET(ref, offsetof(zend_op, op2_type))), ir_CONST_U8(IS_TMP_VAR|IS_VAR))); ir_IF_TRUE(if_dtor); ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.var))); if (sizeof(void*) == 8) { ref = ir_ZEXT_A(ref); } ref = ir_ADD_A(jit_FP(jit), ref); var_addr = ZEND_ADDR_REF_ZVAL(ref); jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL); ir_MERGE_WITH_EMPTY_FALSE(if_dtor); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); return 1; } static int zend_jit_interrupt_handler_stub(zend_jit_ctx *jit) { ir_ref if_timeout, if_exception; // EX(opline) = opline ir_STORE(jit_EX(opline), jit_IP(jit)); ir_STORE(jit_EG(vm_interrupt), ir_CONST_U8(0)); if_timeout = ir_IF(ir_EQ(ir_LOAD_U8(jit_EG(timed_out)), ir_CONST_U8(0))); ir_IF_FALSE(if_timeout); ir_CALL(IR_VOID, ir_CONST_FUNC(zend_timeout)); ir_MERGE_WITH_EMPTY_TRUE(if_timeout); if (zend_interrupt_function) { ir_CALL_1(IR_VOID, ir_CONST_FUNC(zend_interrupt_function), jit_FP(jit)); if_exception = ir_IF(ir_LOAD_A(jit_EG(exception))); ir_IF_TRUE(if_exception); ir_CALL(IR_VOID, ir_CONST_FUNC(zend_jit_exception_in_interrupt_handler_helper)); ir_MERGE_WITH_EMPTY_FALSE(if_exception); jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data))); jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); } if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit))); } else { zend_jit_vm_enter(jit, jit_IP(jit)); } return 1; } static int zend_jit_leave_function_handler_stub(zend_jit_ctx *jit) { #if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL ir_TAILCALL(IR_OPCODE_HANDLER_RET, ir_CONST_OPCODE_HANDLER_FUNC(zend_jit_leave_func_helper_tailcall)); return 1; #else ir_ref call_info = ir_LOAD_U32(jit_EX(This.u1.type_info)); ir_ref if_top = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_TOP))); ir_IF_FALSE(if_top); if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info); jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); } else if (GCC_GLOBAL_REGS) { ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), call_info); } else { ir_TAILCALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_leave_nested_func_helper), jit_FP(jit), jit_IP(jit), call_info); } ir_IF_TRUE(if_top); if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info); ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); } else if (GCC_GLOBAL_REGS) { ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), call_info); } else { ir_TAILCALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_leave_top_func_helper), jit_FP(jit), jit_IP(jit), call_info); } return 1; #endif /* ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL */ } static int zend_jit_negative_shift_stub(zend_jit_ctx *jit) { ir_CALL_2(IR_VOID, ir_CONST_FUNC_PROTO(zend_throw_error, ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), ir_CONST_ADDR(zend_ce_arithmetic_error), ir_CONST_ADDR("Bit shift by negative number")); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2)); return 1; } static int zend_jit_mod_by_zero_stub(zend_jit_ctx *jit) { ir_CALL_2(IR_VOID, ir_CONST_FUNC_PROTO(zend_throw_error, ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), ir_CONST_ADDR(zend_ce_division_by_zero_error), ir_CONST_ADDR("Modulo by zero")); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op1_op2)); return 1; } static int zend_jit_invalid_this_stub(zend_jit_ctx *jit) { ir_CALL_2(IR_VOID, ir_CONST_FUNC_PROTO(zend_throw_error, ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), IR_NULL, ir_CONST_ADDR("Using $this when not in object context")); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); return 1; } static int zend_jit_undefined_function_stub(zend_jit_ctx *jit) { // JIT: load EX(opline) ir_ref ref = ir_LOAD_A(jit_FP(jit)); ir_ref arg3 = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(zend_op, op2.constant))); if (sizeof(void*) == 8) { arg3 = ir_LOAD_A(ir_ADD_A(ref, ir_SEXT_A(arg3))); } else { arg3 = ir_LOAD_A(arg3); } arg3 = ir_ADD_OFFSET(arg3, offsetof(zend_string, val)); ir_CALL_3(IR_VOID, ir_CONST_FUNC_PROTO(zend_throw_error, ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), IR_NULL, ir_CONST_ADDR("Call to undefined function %s()"), arg3); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); return 1; } static int zend_jit_throw_cannot_pass_by_ref_stub(zend_jit_ctx *jit) { ir_ref opline, ref, rx, if_eq, if_tmp; // JIT: opline = EX(opline) opline = ir_LOAD_A(jit_FP(jit)); // JIT: ZVAL_UNDEF(ZEND_CALL_VAR(RX, opline->result.var)) ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var))); if (sizeof(void*) == 8) { ref = ir_ZEXT_A(ref); } rx = jit_IP(jit); jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(rx, ref), ir_CONST_U32(IS_UNDEF)); // last EX(call) frame may be delayed // JIT: if (EX(call) == RX) ref = ir_LOAD_A(jit_EX(call)); if_eq = ir_IF(ir_EQ(rx, ref)); ir_IF_FALSE(if_eq); // JIT: RX->prev_execute_data == EX(call) ir_STORE(jit_CALL(rx, prev_execute_data), ref); // JIT: EX(call) = RX ir_STORE(jit_EX(call), rx); ir_MERGE_WITH_EMPTY_TRUE(if_eq); // JIT: IP = opline jit_STORE_IP(jit, opline); // JIT: zend_cannot_pass_by_reference(opline->op2.num) ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_cannot_pass_by_reference), ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, op2.num)))); // JIT: if (IP->op1_type == IS_TMP_VAR) ref = ir_LOAD_U8(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1_type))); if_tmp = ir_IF(ir_EQ(ref, ir_CONST_U8(IS_TMP_VAR))); ir_IF_TRUE(if_tmp); // JIT: zval_ptr_dtor(EX_VAR(IP->op1.var)) ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_IP(jit), offsetof(zend_op, op1.var))); if (sizeof(void*) == 8) { ref = ir_ZEXT_A(ref); } ref = ir_ADD_A(jit_FP(jit), ref); jit_ZVAL_PTR_DTOR(jit, ZEND_ADDR_REF_ZVAL(ref), MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF, 0, NULL); ir_MERGE_WITH_EMPTY_FALSE(if_tmp); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); return 1; } static int zend_jit_icall_throw_stub(zend_jit_ctx *jit) { ir_ref ip, if_set; // JIT: zend_rethrow_exception(zend_execute_data *execute_data) // JIT: if (EX(opline)->opcode != ZEND_HANDLE_EXCEPTION) { jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); ip = jit_IP(jit); if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))), ir_CONST_U8(ZEND_HANDLE_EXCEPTION))); ir_IF_FALSE(if_set); // JIT: EG(opline_before_exception) = opline; ir_STORE(jit_EG(opline_before_exception), ip); ir_MERGE_WITH_EMPTY_TRUE(if_set); // JIT: opline = EG(exception_op); jit_STORE_IP(jit, jit_EG(exception_op)); ir_STORE(jit_EX(opline), jit_IP(jit)); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); return 1; } static int zend_jit_leave_throw_stub(zend_jit_ctx *jit) { ir_ref ip, if_set; // JIT: if (opline->opcode != ZEND_HANDLE_EXCEPTION) { jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); ip = jit_IP(jit); if_set = ir_IF(ir_EQ(ir_LOAD_U8(ir_ADD_OFFSET(ip, offsetof(zend_op, opcode))), ir_CONST_U8(ZEND_HANDLE_EXCEPTION))); ir_IF_FALSE(if_set); // JIT: EG(opline_before_exception) = opline; ir_STORE(jit_EG(opline_before_exception), ip); ir_MERGE_WITH_EMPTY_TRUE(if_set); // JIT: opline = EG(exception_op); jit_STORE_IP(jit, jit_EG(exception_op)); if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { ir_STORE(jit_EX(opline), jit_IP(jit)); // JIT: HANDLE_EXCEPTION() ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); } else { zend_jit_vm_leave(jit, jit_IP(jit)); } return 1; } static int zend_jit_hybrid_runtime_jit_stub(zend_jit_ctx *jit) { if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID) { return 0; } ir_CALL(IR_VOID, ir_CONST_OPCODE_HANDLER_FUNC(zend_runtime_jit)); ir_IJMP(ir_LOAD_A(jit_IP(jit))); return 1; } static int zend_jit_hybrid_profile_jit_stub(zend_jit_ctx *jit) { ir_ref addr, func, run_time_cache, jit_extension; if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID) { return 0; } addr = ir_CONST_ADDR(&zend_jit_profile_counter), ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1))); func = ir_LOAD_A(jit_EX(func)); run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid]))); if (zend_jit_profile_counter_rid) { addr = ir_ADD_OFFSET(run_time_cache, zend_jit_profile_counter_rid * sizeof(void*)); } else { addr = run_time_cache; } ir_STORE(addr, ir_ADD_L(ir_LOAD_L(addr), ir_CONST_LONG(1))); addr = ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_extension, orig_handler)); ir_IJMP(ir_LOAD_A(addr)); return 1; } static int _zend_jit_hybrid_hot_counter_stub(zend_jit_ctx *jit, uint32_t cost) { ir_ref func, jit_extension, addr, ref, if_overflow; func = ir_LOAD_A(jit_EX(func)); jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid]))); addr = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, counter))); ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost)); ir_STORE(addr, ref); if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0))); ir_IF_TRUE_cold(if_overflow); ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT)); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_hot_func), jit_FP(jit), jit_IP(jit)); ir_IJMP(ir_LOAD_A(jit_IP(jit))); ir_IF_FALSE(if_overflow); ref = ir_SUB_A(jit_IP(jit), ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, opcodes)))); ref = ir_DIV_A(ref, ir_CONST_ADDR(sizeof(zend_op) / sizeof(void*))); addr = ir_ADD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_hot_extension, orig_handlers)), ref); ir_IJMP(ir_LOAD_A(addr)); return 1; } static int zend_jit_hybrid_func_hot_counter_stub(zend_jit_ctx *jit) { if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { return 0; } return _zend_jit_hybrid_hot_counter_stub(jit, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); } static int zend_jit_hybrid_loop_hot_counter_stub(zend_jit_ctx *jit) { if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { return 0; } return _zend_jit_hybrid_hot_counter_stub(jit, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); } static ir_ref _zend_jit_orig_opline_handler(zend_jit_ctx *jit, ir_ref offset) { ir_ref addr = ir_ADD_A(offset, jit_IP(jit)); return ir_LOAD_A(addr); } static ir_ref zend_jit_orig_opline_handler(zend_jit_ctx *jit) { ir_ref func, jit_extension, offset; func = ir_LOAD_A(jit_EX(func)); jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid]))); offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset))); return _zend_jit_orig_opline_handler(jit, offset); } static int _zend_jit_hybrid_trace_counter_stub(zend_jit_ctx *jit, uint32_t cost) { ir_ref func, jit_extension, offset, addr, ref, if_overflow, ret, if_halt; func = ir_LOAD_A(jit_EX(func)); jit_extension = ir_LOAD_A(ir_ADD_OFFSET(func, offsetof(zend_op_array, reserved[zend_func_info_rid]))); offset = ir_LOAD_A(ir_ADD_OFFSET(jit_extension, offsetof(zend_jit_op_array_trace_extension, offset))); addr = ir_LOAD_A(ir_ADD_OFFSET(ir_ADD_A(offset, jit_IP(jit)), offsetof(zend_op_trace_info, counter))); ref = ir_SUB_I16(ir_LOAD_I16(addr), ir_CONST_I16(cost)); ir_STORE(addr, ref); if_overflow = ir_IF(ir_LE(ref, ir_CONST_I16(0))); ir_IF_TRUE_cold(if_overflow); ir_STORE(addr, ir_CONST_I16(ZEND_JIT_COUNTER_INIT)); ret = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_trace_hot_root), jit_FP(jit), jit_IP(jit)); if_halt = ir_IF(ir_LT(ret, ir_CONST_I32(0))); ir_IF_FALSE(if_halt); ref = jit_EG(current_execute_data); jit_STORE_FP(jit, ir_LOAD_A(ref)); ref = ir_LOAD_A(jit_EX(opline)); jit_STORE_IP(jit, ref); ir_IJMP(ir_LOAD_A(jit_IP(jit))); ir_IF_FALSE(if_overflow); ir_IJMP(_zend_jit_orig_opline_handler(jit, offset)); ir_IF_TRUE(if_halt); ir_IJMP(ir_CONST_OPCODE_HANDLER_FUNC(zend_jit_halt_op->handler)); return 1; } static int zend_jit_hybrid_func_trace_counter_stub(zend_jit_ctx *jit) { if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID || !JIT_G(hot_func)) { return 0; } return _zend_jit_hybrid_trace_counter_stub(jit, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_func) - 1) / JIT_G(hot_func))); } static int zend_jit_hybrid_ret_trace_counter_stub(zend_jit_ctx *jit) { if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID || !JIT_G(hot_return)) { return 0; } return _zend_jit_hybrid_trace_counter_stub(jit, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_return) - 1) / JIT_G(hot_return))); } static int zend_jit_hybrid_loop_trace_counter_stub(zend_jit_ctx *jit) { if (ZEND_VM_KIND != ZEND_VM_KIND_HYBRID || !JIT_G(hot_loop)) { return 0; } return _zend_jit_hybrid_trace_counter_stub(jit, ((ZEND_JIT_COUNTER_INIT + JIT_G(hot_loop) - 1) / JIT_G(hot_loop))); } static int zend_jit_trace_halt_stub(zend_jit_ctx *jit) { if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) { ir_TAILCALL(IR_VOID, ir_CONST_OPCODE_HANDLER_FUNC(zend_jit_halt_op->handler)); } else if (GCC_GLOBAL_REGS) { jit_STORE_IP(jit, IR_NULL); ir_RETURN(IR_VOID); } else { ir_RETURN(ir_CONST_ADDR(ZEND_VM_ENTER_BIT)); // ZEND_VM_RETURN } return 1; } static int zend_jit_trace_escape_stub(zend_jit_ctx *jit) { if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit))); } else { zend_jit_vm_enter(jit, jit_IP(jit)); } return 1; } static int zend_jit_trace_exit_stub(zend_jit_ctx *jit) { ir_ref ref, ret, if_zero, addr; // EX(opline) = opline ir_STORE(jit_EX(opline), jit_IP(jit)); ret = ir_EXITCALL(ir_CONST_FC_FUNC(zend_jit_trace_exit)); if_zero = ir_IF(ir_EQ(ret, ir_CONST_I32(0))); ir_IF_TRUE(if_zero); ref = jit_EG(current_execute_data); jit_STORE_FP(jit, ir_LOAD_A(ref)); ref = ir_LOAD_A(jit_EX(opline)); jit_STORE_IP(jit, ref); if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit))); } else { zend_jit_vm_enter(jit, ref); } ir_IF_FALSE(if_zero); ir_GUARD(ir_GE(ret, ir_CONST_I32(0)), jit_STUB_ADDR(jit, jit_stub_trace_halt)); ref = jit_EG(current_execute_data); jit_STORE_FP(jit, ir_LOAD_A(ref)); ref = ir_LOAD_A(jit_EX(opline)); jit_STORE_IP(jit, ref); // check for interrupt (try to avoid this ???) zend_jit_check_timeout(jit, NULL, NULL); addr = zend_jit_orig_opline_handler(jit); if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { zend_jit_tailcall_handler(jit, addr); } else { #if defined(IR_TARGET_X86) addr = ir_CAST_OPCODE_HANDLER_FUNC(addr); #endif ref = ir_CALL_2(IR_ADDR, addr, jit_FP(jit), jit_IP(jit)); zend_jit_vm_enter(jit, ref); } return 1; } static int zend_jit_undefined_offset_stub(zend_jit_ctx *jit) { if (GCC_GLOBAL_REGS) { ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key)); } else { ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key), jit_FP(jit)); } return 1; } static int zend_jit_undefined_key_stub(zend_jit_ctx *jit) { if (GCC_GLOBAL_REGS) { ir_TAILCALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key)); } else { ir_TAILCALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_string_key), jit_FP(jit)); } return 1; } static int zend_jit_cannot_add_element_stub(zend_jit_ctx *jit) { ir_ref opline = ir_LOAD_A(jit_EX(opline)); ir_ref ref, if_result_used; if_result_used = ir_IF(ir_AND_U8( ir_LOAD_U8(ir_ADD_OFFSET(opline, offsetof(zend_op, result_type))), ir_CONST_U8(IS_TMP_VAR|IS_VAR))); ir_IF_TRUE(if_result_used); ref = ir_LOAD_U32(ir_ADD_OFFSET(opline, offsetof(zend_op, result.var))); if (sizeof(void*) == 8) { ref = ir_ZEXT_A(ref); } jit_set_Z_TYPE_INFO_ref(jit, ir_ADD_A(jit_FP(jit), ref), ir_CONST_U32(IS_UNDEF)); ir_MERGE_WITH_EMPTY_FALSE(if_result_used); ir_CALL_2(IR_VOID, ir_CONST_FUNC_PROTO(zend_throw_error, ir_proto_2(&jit->ctx, IR_VARARG_FUNC, IR_VOID, IR_ADDR, IR_ADDR)), IR_NULL, ir_CONST_ADDR("Cannot add element to the array as the next element is already occupied")); ir_RETURN(IR_VOID); return 1; } static int zend_jit_assign_const_stub(zend_jit_ctx *jit) { ir_ref var = ir_PARAM(IR_ADDR, "var", 1); ir_ref val = ir_PARAM(IR_ADDR, "val", 2); zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var); zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; if (!zend_jit_assign_to_variable( jit, NULL, var_addr, var_addr, -1, -1, IS_CONST, val_addr, val_info, 0, 0, 0)) { return 0; } ir_RETURN(IR_VOID); return 1; } static int zend_jit_assign_tmp_stub(zend_jit_ctx *jit) { ir_ref var = ir_PARAM(IR_ADDR, "var", 1); ir_ref val = ir_PARAM(IR_ADDR, "val", 2); zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var); zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN; if (!zend_jit_assign_to_variable( jit, NULL, var_addr, var_addr, -1, -1, IS_TMP_VAR, val_addr, val_info, 0, 0, 0)) { return 0; } ir_RETURN(IR_VOID); return 1; } static int zend_jit_assign_var_stub(zend_jit_ctx *jit) { ir_ref var = ir_PARAM(IR_ADDR, "var", 1); ir_ref val = ir_PARAM(IR_ADDR, "val", 2); zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var); zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF; if (!zend_jit_assign_to_variable( jit, NULL, var_addr, var_addr, -1, -1, IS_VAR, val_addr, val_info, 0, 0, 0)) { return 0; } ir_RETURN(IR_VOID); return 1; } static int zend_jit_assign_cv_noref_stub(zend_jit_ctx *jit) { ir_ref var = ir_PARAM(IR_ADDR, "var", 1); ir_ref val = ir_PARAM(IR_ADDR, "val", 2); zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var); zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN/*|MAY_BE_UNDEF*/; if (!zend_jit_assign_to_variable( jit, NULL, var_addr, var_addr, -1, -1, IS_CV, val_addr, val_info, 0, 0, 0)) { return 0; } ir_RETURN(IR_VOID); return 1; } static int zend_jit_new_array_stub(zend_jit_ctx *jit) { ir_ref var = ir_PARAM(IR_ADDR, "var", 1); zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var); ir_ref ref = ir_CALL(IR_ADDR, ir_CONST_FC_FUNC(_zend_new_array_0)); jit_set_Z_PTR(jit, var_addr, ref); jit_set_Z_TYPE_INFO(jit, var_addr, IS_ARRAY_EX); ir_RETURN(ref); return 1; } static int zend_jit_assign_cv_stub(zend_jit_ctx *jit) { ir_ref var = ir_PARAM(IR_ADDR, "var", 1); ir_ref val = ir_PARAM(IR_ADDR, "val", 2); zend_jit_addr var_addr = ZEND_ADDR_REF_ZVAL(var); zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val); uint32_t val_info = MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN|MAY_BE_REF/*|MAY_BE_UNDEF*/; if (!zend_jit_assign_to_variable( jit, NULL, var_addr, var_addr, -1, -1, IS_CV, val_addr, val_info, 0, 0, 0)) { return 0; } ir_RETURN(IR_VOID); return 1; } static void zend_jit_init_ctx(zend_jit_ctx *jit, uint32_t flags) { #if defined (__CET__) && (__CET__ & 1) != 0 flags |= IR_GEN_ENDBR; #endif flags |= IR_OPT_FOLDING | IR_OPT_CFG | IR_OPT_CODEGEN; ir_init(&jit->ctx, flags, 256, 1024); jit->ctx.ret_type = -1; #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) jit->ctx.mflags |= default_mflags; if (JIT_G(opt_flags) & allowed_opt_flags & ZEND_JIT_CPU_AVX) { jit->ctx.mflags |= IR_X86_AVX; } #elif defined(IR_TARGET_AARCH64) jit->ctx.get_veneer = zend_jit_get_veneer; jit->ctx.set_veneer = zend_jit_set_veneer; #endif jit->ctx.fixed_regset = (1<ctx.flags |= IR_NO_STACK_COMBINE; if (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { jit->ctx.flags |= IR_FUNCTION; /* Stack must be 16 byte aligned */ /* TODO: select stack size ??? */ #if ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL # if defined(IR_TARGET_AARCH64) /* Must save LR */ jit->ctx.flags |= IR_USE_FRAME_POINTER; /* Same as HYBRID VM */ jit->ctx.fixed_stack_frame_size = sizeof(void*) * 4; /* 4 spill slots */ # else /* Same as HYBRID VM, plus 1 slot for re-alignment (caller pushes return address, frame is not aligned on entry) */ jit->ctx.fixed_stack_frame_size = sizeof(void*) * 5; /* 5 spill slots (8 bytes) or 10 spill slots (4 bytes) */ # endif #elif defined(IR_TARGET_AARCH64) jit->ctx.flags |= IR_USE_FRAME_POINTER; jit->ctx.fixed_stack_frame_size = sizeof(void*) * 16; /* 10 saved registers and 6 spill slots (8 bytes) */ #elif defined(_WIN64) jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 8 saved registers and 3 spill slots (8 bytes) */ #elif defined(IR_TARGET_X86_64) jit->ctx.fixed_stack_frame_size = sizeof(void*) * 9; /* 6 saved registers and 3 spill slots (8 bytes) */ #else /* IR_TARGET_x86 */ jit->ctx.fixed_stack_frame_size = sizeof(void*) * 11; /* 4 saved registers and 7 spill slots (4 bytes) */ #endif /* JIT-ed code is called only from execute_ex, which takes care * of saving ZREG_FP, ZREG_IP when GCC_GLOBAL_REGS is 1, so we don't * have to save them. */ if (GCC_GLOBAL_REGS) { jit->ctx.fixed_save_regset = IR_REGSET_PRESERVED & ~((1<ctx.fixed_regset |= (1<ctx.fixed_regset |= (1<ctx.fixed_save_regset = IR_REGSET_PRESERVED; //#ifdef _WIN64 // jit->ctx.fixed_save_regset &= 0xffff; // TODO: don't save FP registers ??? //#endif } #ifdef _WIN64 jit->ctx.fixed_call_stack_size = 16 + IR_SHADOW_ARGS; #else jit->ctx.fixed_call_stack_size = 16; #endif } else { #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE jit->ctx.fixed_stack_red_zone = ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE; if (jit->ctx.fixed_stack_red_zone > 16) { jit->ctx.fixed_stack_frame_size = jit->ctx.fixed_stack_red_zone - 16; jit->ctx.fixed_call_stack_size = 16; } jit->ctx.flags |= IR_MERGE_EMPTY_ENTRIES; #else jit->ctx.fixed_stack_red_zone = 0; jit->ctx.fixed_stack_frame_size = 32; /* 4 spill slots (8 bytes) or 8 spill slots (4 bytes) */ jit->ctx.fixed_call_stack_size = 16; #endif #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) jit->ctx.fixed_regset |= (1<ctx.snapshot_create = (ir_snapshot_create_t)jit_SNAPSHOT; jit->op_array = NULL; jit->current_op_array = NULL; jit->ssa = NULL; jit->name = NULL; jit->last_valid_opline = NULL; jit->use_last_valid_opline = 0; jit->track_last_valid_opline = 0; jit->reuse_ip = 0; jit->delayed_call_level = 0; delayed_call_chain = 0; jit->b = -1; #ifdef ZTS jit->tls = IR_UNUSED; #endif jit->fp = IR_UNUSED; jit->poly_func_ref = IR_UNUSED; jit->poly_this_ref = IR_UNUSED; jit->trace_loop_ref = IR_UNUSED; jit->return_inputs = IR_UNUSED; jit->bb_start_ref = NULL; jit->bb_predecessors = NULL; jit->bb_edges = NULL; jit->trace = NULL; jit->ra = NULL; jit->delay_var = -1; jit->delay_refs = NULL; jit->eg_exception_addr = 0; zend_hash_init(&jit->addr_hash, 64, NULL, NULL, 0); memset(jit->stub_addr, 0, sizeof(jit->stub_addr)); ir_START(); } static int zend_jit_free_ctx(zend_jit_ctx *jit) { if (jit->name) { zend_string_release(jit->name); } zend_hash_destroy(&jit->addr_hash); ir_free(&jit->ctx); return 1; } static void *zend_jit_ir_compile(ir_ctx *ctx, size_t *size, const char *name) { void *entry; ir_code_buffer code_buffer; if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_SRC) { if (name) fprintf(stderr, "%s: ; after folding\n", name); ir_save(ctx, 0, stderr); } #if ZEND_DEBUG ir_check(ctx); #endif ir_build_def_use_lists(ctx); #if ZEND_DEBUG ir_check(ctx); #endif #if 1 ir_sccp(ctx); #endif if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCCP) { if (name) fprintf(stderr, "%s: ; after SCCP\n", name); ir_save(ctx, 0, stderr); } ir_build_cfg(ctx); ir_build_dominators_tree(ctx); ir_find_loops(ctx); if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_CFG) { if (name) fprintf(stderr, "%s: ; after CFG\n", name); ir_save(ctx, IR_SAVE_CFG, stderr); } ir_gcm(ctx); if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_GCM) { if (name) fprintf(stderr, "%s: ; after GCM\n", name); ir_save(ctx, IR_SAVE_CFG|IR_SAVE_CFG_MAP, stderr); } ir_schedule(ctx); if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE) { if (name) fprintf(stderr, "%s: ; after schedule\n", name); ir_save(ctx, IR_SAVE_CFG, stderr); } ir_match(ctx); #if !defined(IR_TARGET_AARCH64) ctx->flags &= ~IR_USE_FRAME_POINTER; /* don't use FRAME_POINTER even with ALLOCA, TODO: cleanup this ??? */ #endif ir_assign_virtual_registers(ctx); ir_compute_live_ranges(ctx); ir_coalesce(ctx); ir_reg_alloc(ctx); if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_AFTER_REGS) { if (name) fprintf(stderr, "%s: ; after register allocation\n", name); ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr); ir_dump_live_ranges(ctx, stderr); } ir_schedule_blocks(ctx); if (JIT_G(debug) & (ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) { if (JIT_G(debug) & ZEND_JIT_DEBUG_IR_CODEGEN) { if (name) fprintf(stderr, "%s: ; codegen\n", name); ir_dump_codegen(ctx, stderr); } else { if (name) fprintf(stderr, "%s: ; final\n", name); ir_save(ctx, IR_SAVE_CFG|IR_SAVE_RULES|IR_SAVE_REGS, stderr); } } #if ZEND_DEBUG ir_check(ctx); #endif code_buffer.start = dasm_buf; code_buffer.end = dasm_end; code_buffer.pos = *dasm_ptr; ctx->code_buffer = &code_buffer; entry = ir_emit_code(ctx, size); *dasm_ptr = code_buffer.pos; #if defined(IR_TARGET_AARCH64) if (ctx->flags2 & IR_HAS_VENEERS) { zend_jit_commit_veneers(); } #endif return entry; } static void zend_jit_setup_stubs(void) { zend_jit_ctx jit; void *entry; size_t size; uint32_t i; for (i = 0; i < sizeof(zend_jit_stubs)/sizeof(zend_jit_stubs[0]); i++) { zend_jit_init_ctx(&jit, zend_jit_stubs[i].flags); if (!zend_jit_stubs[i].stub(&jit)) { zend_jit_free_ctx(&jit); zend_jit_stub_handlers[i] = NULL; continue; } entry = zend_jit_ir_compile(&jit.ctx, &size, zend_jit_stubs[i].name); if (!entry) { zend_jit_free_ctx(&jit); zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile stub"); } zend_jit_stub_handlers[i] = entry; if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) { #ifdef HAVE_CAPSTONE if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { ir_disasm_add_symbol(zend_jit_stubs[i].name, (uintptr_t)entry, size); } if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_STUBS) { ir_disasm(zend_jit_stubs[i].name, entry, size, (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0, &jit.ctx, stderr); } #endif #ifndef _WIN32 if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) { // ir_mem_unprotect(entry, size); ir_gdb_register(zend_jit_stubs[i].name, entry, size, 0, 0); // ir_mem_protect(entry, size); } if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) { ir_perf_map_register(zend_jit_stubs[i].name, entry, size); if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) { ir_perf_jitdump_register(zend_jit_stubs[i].name, entry, size); } } #endif } zend_jit_free_ctx(&jit); } } #define REGISTER_HELPER(n) \ ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)n, sizeof(void*)); #define REGISTER_DATA(n) \ ir_disasm_add_symbol(#n, (uint64_t)(uintptr_t)&n, sizeof(n)); static void zend_jit_setup_disasm(void) { #ifdef HAVE_CAPSTONE ir_disasm_init(); if (zend_vm_kind() == ZEND_VM_KIND_HYBRID) { zend_op opline; memset(&opline, 0, sizeof(opline)); opline.opcode = ZEND_DO_UCALL; opline.result_type = IS_UNUSED; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); opline.opcode = ZEND_DO_UCALL; opline.result_type = IS_VAR; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_DO_UCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); opline.opcode = ZEND_DO_FCALL_BY_NAME; opline.result_type = IS_UNUSED; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); opline.opcode = ZEND_DO_FCALL_BY_NAME; opline.result_type = IS_VAR; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_DO_FCALL_BY_NAME_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); opline.opcode = ZEND_DO_FCALL; opline.result_type = IS_UNUSED; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_UNUSED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); opline.opcode = ZEND_DO_FCALL; opline.result_type = IS_VAR; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_DO_FCALL_SPEC_RETVAL_USED_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); opline.opcode = ZEND_RETURN; opline.op1_type = IS_CONST; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_RETURN_SPEC_CONST_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); opline.opcode = ZEND_RETURN; opline.op1_type = IS_TMP_VAR; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_RETURN_SPEC_TMP_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); opline.opcode = ZEND_RETURN; opline.op1_type = IS_VAR; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_RETURN_SPEC_VAR_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); opline.opcode = ZEND_RETURN; opline.op1_type = IS_CV; zend_vm_set_opcode_handler(&opline); ir_disasm_add_symbol("ZEND_RETURN_SPEC_CV_LABEL", (uint64_t)(uintptr_t)opline.handler, sizeof(void*)); ir_disasm_add_symbol("ZEND_HYBRID_HALT_LABEL", (uint64_t)(uintptr_t)zend_jit_halt_op->handler, sizeof(void*)); } REGISTER_DATA(zend_jit_profile_counter); REGISTER_HELPER(zend_runtime_jit); REGISTER_HELPER(zend_jit_hot_func); REGISTER_HELPER(zend_jit_trace_hot_root); REGISTER_HELPER(zend_jit_trace_exit); REGISTER_HELPER(zend_jit_array_free); REGISTER_HELPER(zend_jit_undefined_op_helper); REGISTER_HELPER(zend_jit_pre_inc_typed_ref); REGISTER_HELPER(zend_jit_pre_dec_typed_ref); REGISTER_HELPER(zend_jit_post_inc_typed_ref); REGISTER_HELPER(zend_jit_post_dec_typed_ref); REGISTER_HELPER(zend_jit_pre_inc); REGISTER_HELPER(zend_jit_pre_dec); REGISTER_HELPER(zend_jit_add_arrays_helper); REGISTER_HELPER(zend_jit_fast_assign_concat_helper); REGISTER_HELPER(zend_jit_fast_concat_helper); REGISTER_HELPER(zend_jit_fast_concat_tmp_helper); REGISTER_HELPER(zend_jit_assign_op_to_typed_ref_tmp); REGISTER_HELPER(zend_jit_assign_op_to_typed_ref); REGISTER_HELPER(zend_jit_assign_const_to_typed_ref); REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref); REGISTER_HELPER(zend_jit_assign_var_to_typed_ref); REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref); REGISTER_HELPER(zend_jit_assign_const_to_typed_ref2); REGISTER_HELPER(zend_jit_assign_tmp_to_typed_ref2); REGISTER_HELPER(zend_jit_assign_var_to_typed_ref2); REGISTER_HELPER(zend_jit_assign_cv_to_typed_ref2); REGISTER_HELPER(zend_jit_check_constant); REGISTER_HELPER(zend_jit_get_constant); REGISTER_HELPER(zend_jit_int_extend_stack_helper); REGISTER_HELPER(zend_jit_extend_stack_helper); REGISTER_HELPER(zend_jit_init_func_run_time_cache_helper); REGISTER_HELPER(zend_jit_find_func_helper); REGISTER_HELPER(zend_jit_find_ns_func_helper); REGISTER_HELPER(zend_jit_jmp_frameless_helper); REGISTER_HELPER(zend_jit_unref_helper); REGISTER_HELPER(zend_jit_invalid_method_call); REGISTER_HELPER(zend_jit_invalid_method_call_tmp); REGISTER_HELPER(zend_jit_find_method_helper); REGISTER_HELPER(zend_jit_find_method_tmp_helper); REGISTER_HELPER(zend_jit_push_static_method_call_frame); REGISTER_HELPER(zend_jit_push_static_method_call_frame_tmp); REGISTER_HELPER(zend_jit_find_class_helper); REGISTER_HELPER(zend_jit_find_static_method_helper); REGISTER_HELPER(zend_jit_push_this_method_call_frame); REGISTER_HELPER(zend_jit_free_trampoline_helper); REGISTER_HELPER(zend_jit_verify_return_slow); REGISTER_HELPER(zend_jit_deprecated_helper); REGISTER_HELPER(zend_jit_undefined_long_key); REGISTER_HELPER(zend_jit_undefined_long_key_ex); REGISTER_HELPER(zend_jit_undefined_string_key); REGISTER_HELPER(zend_jit_copy_extra_args_helper); REGISTER_HELPER(zend_jit_copy_extra_args_helper_no_skip_recv); REGISTER_HELPER(zend_jit_vm_stack_free_args_helper); REGISTER_HELPER(zend_free_extra_named_params); REGISTER_HELPER(zend_jit_free_call_frame); REGISTER_HELPER(zend_jit_exception_in_interrupt_handler_helper); REGISTER_HELPER(zend_jit_verify_arg_slow); REGISTER_HELPER(zend_missing_arg_error); REGISTER_HELPER(zend_jit_only_vars_by_reference); REGISTER_HELPER(zend_jit_leave_func_helper); REGISTER_HELPER(zend_jit_leave_nested_func_helper); REGISTER_HELPER(zend_jit_leave_top_func_helper); REGISTER_HELPER(zend_jit_fetch_global_helper); REGISTER_HELPER(zend_jit_hash_index_lookup_rw_no_packed); REGISTER_HELPER(zend_jit_hash_index_lookup_rw); REGISTER_HELPER(zend_jit_hash_lookup_rw); REGISTER_HELPER(zend_jit_symtable_find); REGISTER_HELPER(zend_jit_symtable_lookup_w); REGISTER_HELPER(zend_jit_symtable_lookup_rw); REGISTER_HELPER(zend_jit_fetch_dim_r_helper); REGISTER_HELPER(zend_jit_fetch_dim_is_helper); REGISTER_HELPER(zend_jit_fetch_dim_isset_helper); REGISTER_HELPER(zend_jit_fetch_dim_rw_helper); REGISTER_HELPER(zend_jit_fetch_dim_w_helper); REGISTER_HELPER(zend_jit_fetch_dim_str_offset_r_helper); REGISTER_HELPER(zend_jit_fetch_dim_str_r_helper); REGISTER_HELPER(zend_jit_fetch_dim_str_is_helper); REGISTER_HELPER(zend_jit_fetch_dim_obj_r_helper); REGISTER_HELPER(zend_jit_fetch_dim_obj_is_helper); REGISTER_HELPER(zend_jit_invalid_array_access); REGISTER_HELPER(zend_jit_zval_array_dup); REGISTER_HELPER(zend_jit_prepare_assign_dim_ref); REGISTER_HELPER(zend_jit_fetch_dim_obj_w_helper); REGISTER_HELPER(zend_jit_fetch_dim_obj_rw_helper); REGISTER_HELPER(zend_jit_isset_dim_helper); REGISTER_HELPER(zend_jit_assign_dim_helper); REGISTER_HELPER(zend_jit_assign_dim_op_helper); REGISTER_HELPER(zend_jit_fetch_obj_w_slow); REGISTER_HELPER(zend_jit_fetch_obj_r_slow); REGISTER_HELPER(zend_jit_fetch_obj_r_slow_ex); REGISTER_HELPER(zend_jit_fetch_obj_is_slow); REGISTER_HELPER(zend_jit_fetch_obj_is_slow_ex); REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic); REGISTER_HELPER(zend_jit_fetch_obj_r_dynamic_ex); REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic); REGISTER_HELPER(zend_jit_fetch_obj_is_dynamic_ex); REGISTER_HELPER(zend_jit_check_array_promotion); REGISTER_HELPER(zend_jit_create_typed_ref); REGISTER_HELPER(zend_jit_invalid_property_write); REGISTER_HELPER(zend_jit_invalid_property_read); REGISTER_HELPER(zend_jit_extract_helper); REGISTER_HELPER(zend_jit_invalid_property_assign); REGISTER_HELPER(zend_jit_assign_to_typed_prop); REGISTER_HELPER(zend_jit_assign_obj_helper); REGISTER_HELPER(zend_jit_invalid_property_assign_op); REGISTER_HELPER(zend_jit_assign_op_to_typed_prop); REGISTER_HELPER(zend_jit_assign_obj_op_helper); REGISTER_HELPER(zend_jit_invalid_property_incdec); REGISTER_HELPER(zend_jit_inc_typed_prop); REGISTER_HELPER(zend_jit_dec_typed_prop); REGISTER_HELPER(zend_jit_pre_inc_typed_prop); REGISTER_HELPER(zend_jit_post_inc_typed_prop); REGISTER_HELPER(zend_jit_pre_dec_typed_prop); REGISTER_HELPER(zend_jit_post_dec_typed_prop); REGISTER_HELPER(zend_jit_pre_inc_obj_helper); REGISTER_HELPER(zend_jit_post_inc_obj_helper); REGISTER_HELPER(zend_jit_pre_dec_obj_helper); REGISTER_HELPER(zend_jit_post_dec_obj_helper); REGISTER_HELPER(zend_jit_uninit_static_prop); REGISTER_HELPER(zend_jit_rope_end); REGISTER_HELPER(zend_fcall_interrupt); #ifndef ZTS REGISTER_DATA(EG(current_execute_data)); REGISTER_DATA(EG(exception)); REGISTER_DATA(EG(opline_before_exception)); REGISTER_DATA(EG(vm_interrupt)); REGISTER_DATA(EG(timed_out)); REGISTER_DATA(EG(uninitialized_zval)); REGISTER_DATA(EG(zend_constants)); REGISTER_DATA(EG(jit_trace_num)); REGISTER_DATA(EG(vm_stack_top)); REGISTER_DATA(EG(vm_stack_end)); REGISTER_DATA(EG(exception_op)); REGISTER_DATA(EG(symbol_table)); REGISTER_DATA(CG(map_ptr_base)); #else /* ZTS */ REGISTER_HELPER(zend_jit_get_tsrm_ls_cache); #endif #endif } static void zend_jit_calc_trace_prologue_size(void) { zend_jit_ctx jit_ctx; zend_jit_ctx *jit = &jit_ctx; void *entry; size_t size; zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) ? 0 : IR_START_BR_TARGET); if (!GCC_GLOBAL_REGS) { if (ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) { ir_ref execute_data_ref = ir_PARAM(IR_ADDR, "execute_data", 1); ir_ref opline_ref = ir_PARAM(IR_ADDR, "opline", 2); jit_STORE_FP(jit, execute_data_ref); jit_STORE_IP(jit, opline_ref); } jit->ctx.flags |= IR_FASTCALL_FUNC; } ir_UNREACHABLE(); entry = zend_jit_ir_compile(&jit->ctx, &size, "JIT$trace_prologue"); zend_jit_free_ctx(jit); if (!entry) { zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: could not compile prologue"); } zend_jit_trace_prologue_size = size; } #if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64) static uintptr_t zend_jit_hybrid_vm_sp_adj = 0; typedef struct _Unwind_Context _Unwind_Context; typedef int (*_Unwind_Trace_Fn)(_Unwind_Context *, void *); extern int _Unwind_Backtrace(_Unwind_Trace_Fn, void *); extern uintptr_t _Unwind_GetCFA(_Unwind_Context *); typedef struct _zend_jit_unwind_arg { int cnt; uintptr_t cfa[3]; } zend_jit_unwind_arg; static int zend_jit_unwind_cb(_Unwind_Context *ctx, void *a) { zend_jit_unwind_arg *arg = (zend_jit_unwind_arg*)a; arg->cfa[arg->cnt] = _Unwind_GetCFA(ctx); arg->cnt++; if (arg->cnt == 3) { return 5; // _URC_END_OF_STACK } return 0; // _URC_NO_REASON; } static void ZEND_FASTCALL zend_jit_touch_vm_stack_data(void *vm_stack_data) { zend_jit_unwind_arg arg; memset(&arg, 0, sizeof(arg)); _Unwind_Backtrace(zend_jit_unwind_cb, &arg); if (arg.cnt == 3) { zend_jit_hybrid_vm_sp_adj = arg.cfa[2] - arg.cfa[1]; } } extern void (ZEND_FASTCALL *zend_touch_vm_stack_data)(void *vm_stack_data); static zend_never_inline void zend_jit_set_sp_adj_vm(void) { void (ZEND_FASTCALL *orig_zend_touch_vm_stack_data)(void *); orig_zend_touch_vm_stack_data = zend_touch_vm_stack_data; zend_touch_vm_stack_data = zend_jit_touch_vm_stack_data; execute_ex(NULL); // set sp_adj[SP_ADJ_VM] zend_touch_vm_stack_data = orig_zend_touch_vm_stack_data; } #endif #ifdef _WIN64 /* * We use a single unwind entry for the whole JIT buffer. * This works, because all the JIT-ed PHP functions have the same "fixed stack frame". */ static PRUNTIME_FUNCTION zend_jit_uw_func = NULL; #ifdef ZEND_JIT_RT_UNWINDER static PRUNTIME_FUNCTION zend_jit_unwind_callback(DWORD64 pc, PVOID context) { return zend_jit_uw_func; } #endif static void zend_jit_setup_unwinder(void) { /* Hardcoded SEH unwind data for JIT-ed PHP functions with "fixed stack frame" */ static const unsigned char uw_data[] = { 0x01, // UBYTE: 3 Version , UBYTE: 5 Flags 0x10, // UBYTE Size of prolog 0x09, // UBYTE Count of unwind codes 0x00, // UBYTE: 4 Frame Register, UBYTE: 4 Frame Register offset (scaled) // USHORT * n Unwind codes array 0x10, 0x82, // c: subq $0x48, %rsp 0x0c, 0xf0, // a: pushq %r15 0x0a, 0xe0, // 8: pushq %r14 0x08, 0xd0, // 6: pushq %r13 0x06, 0xc0, // 4: pushq %r12 0x04, 0x70, // 3: pushq %rdi 0x03, 0x60, // 2: pushq %rsi 0x02, 0x50, // 1: pushq %rbp 0x01, 0x30, // 0: pushq %rbx 0x00, 0x00, }; static const unsigned char uw_data_exitcall[] = { 0x01, // UBYTE: 3 Version , UBYTE: 5 Flags 0x10, // UBYTE Size of prolog 0x0a, // UBYTE Count of unwind codes 0x00, // UBYTE: 4 Frame Register, UBYTE: 4 Frame Register offset (scaled) // USHORT * n Unwind codes array 0x10, 0x01, 0x2f, 0x00, // c: subq 376, %rsp ; 0x48 + 32+16*8+16*8+8+8 0x0c, 0xf0, // a: pushq %r15 0x0a, 0xe0, // 8: pushq %r14 0x08, 0xd0, // 6: pushq %r13 0x06, 0xc0, // 4: pushq %r12 0x04, 0x70, // 3: pushq %rdi 0x03, 0x60, // 2: pushq %rsi 0x02, 0x50, // 1: pushq %rbp 0x01, 0x30, // 0: pushq %rbx }; zend_jit_uw_func = (PRUNTIME_FUNCTION)*dasm_ptr; *dasm_ptr = (char*)*dasm_ptr + ZEND_MM_ALIGNED_SIZE_EX(sizeof(RUNTIME_FUNCTION) * 4 + sizeof(uw_data) + sizeof(uw_data_exitcall) + sizeof(uw_data), 16); zend_jit_uw_func[0].BeginAddress = 0; zend_jit_uw_func[1].BeginAddress = (uintptr_t)zend_jit_stub_handlers[jit_stub_trace_exit] - (uintptr_t)dasm_buf; zend_jit_uw_func[2].BeginAddress = (uintptr_t)zend_jit_stub_handlers[jit_stub_undefined_offset] - (uintptr_t)dasm_buf; zend_jit_uw_func[0].EndAddress = zend_jit_uw_func[1].BeginAddress; zend_jit_uw_func[1].EndAddress = zend_jit_uw_func[2].BeginAddress; zend_jit_uw_func[2].EndAddress = (uintptr_t)dasm_end - (uintptr_t)dasm_buf; zend_jit_uw_func[0].UnwindData = (uintptr_t)zend_jit_uw_func - (uintptr_t)dasm_buf + sizeof(RUNTIME_FUNCTION) * 4; zend_jit_uw_func[1].UnwindData = zend_jit_uw_func[0].UnwindData + sizeof(uw_data); zend_jit_uw_func[2].UnwindData = zend_jit_uw_func[1].UnwindData + sizeof(uw_data_exitcall); memcpy((char*)dasm_buf + zend_jit_uw_func[0].UnwindData, uw_data, sizeof(uw_data)); memcpy((char*)dasm_buf + zend_jit_uw_func[1].UnwindData, uw_data_exitcall, sizeof(uw_data_exitcall)); memcpy((char*)dasm_buf + zend_jit_uw_func[2].UnwindData, uw_data, sizeof(uw_data)); #ifdef ZEND_JIT_RT_UNWINDER RtlInstallFunctionTableCallback( (uintptr_t)dasm_buf | 3, (uintptr_t) dasm_buf, (uintptr_t)dasm_end - (uintptr_t)dasm_buf, zend_jit_unwind_callback, NULL, NULL); #else RtlAddFunctionTable(zend_jit_uw_func, 3, (uintptr_t)dasm_buf); #endif } #endif static void zend_jit_setup(bool reattached) { #if defined(IR_TARGET_X86) if (!zend_cpu_supports_sse2()) { zend_accel_error_noreturn(ACCEL_LOG_FATAL, "Could not enable JIT: CPU doesn't support SSE2"); } #endif #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) allowed_opt_flags = 0; if (zend_cpu_supports_avx()) { allowed_opt_flags |= ZEND_JIT_CPU_AVX; } # if defined(PHP_HAVE_BUILTIN_CPU_SUPPORTS) && defined(__GNUC__) && (ZEND_GCC_VERSION >= 11000) if (zend_cpu_supports_cldemote()) { default_mflags |= IR_X86_CLDEMOTE; } # endif #endif #ifdef ZTS zend_result result = zend_jit_resolve_tsrm_ls_cache_offsets( &tsrm_ls_cache_tcb_offset, &tsrm_tls_index, &tsrm_tls_offset ); if (result == FAILURE) { zend_accel_error(ACCEL_LOG_INFO, "Could not get _tsrm_ls_cache offsets, will fallback to runtime resolution"); } #endif #if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64) if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) { zend_jit_set_sp_adj_vm(); // set zend_jit_hybrid_vm_sp_adj } #endif if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { zend_jit_setup_disasm(); } #ifndef _WIN32 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) { ir_perf_jitdump_open(); } #endif zend_long debug = JIT_G(debug); if (!(debug & ZEND_JIT_DEBUG_ASM_STUBS)) { JIT_G(debug) &= ~(ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_FINAL| ZEND_JIT_DEBUG_IR_CODEGEN| ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_CFG|ZEND_JIT_DEBUG_IR_AFTER_GCM| ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS); } zend_jit_calc_trace_prologue_size(); if (!reattached) { zend_jit_setup_stubs(); } JIT_G(debug) = debug; #ifdef _WIN64 zend_jit_setup_unwinder(); #endif } static void zend_jit_shutdown_ir(void) { #ifndef _WIN32 if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) { ir_perf_jitdump_close(); } if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) { ir_gdb_unregister_all(); } #endif #ifdef HAVE_CAPSTONE if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_ASM_STUBS)) { ir_disasm_free(); } #endif } /* PHP control flow reconstruction helpers */ static ir_ref jit_IF_ex(zend_jit_ctx *jit, ir_ref condition, ir_ref true_block) { ir_ref ref = ir_IF(condition); /* op3 is used as a temporary storage for PHP BB number to reconstruct PHP control flow. * * It's used in jit_IF_TRUE_FALSE_ex() to select IF_TRUE or IF_FALSE instructions * to start target block */ ir_set_op(&jit->ctx, ref, 3, true_block); return ref; } static void jit_IF_TRUE_FALSE_ex(zend_jit_ctx *jit, ir_ref if_ref, ir_ref true_block) { ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE); ZEND_ASSERT(if_ref); ZEND_ASSERT(jit->ctx.ir_base[if_ref].op == IR_IF); ZEND_ASSERT(jit->ctx.ir_base[if_ref].op3); if (jit->ctx.ir_base[if_ref].op3 == true_block) { ir_IF_TRUE(if_ref); } else { ir_IF_FALSE(if_ref); } } static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref); static void _zend_jit_add_predecessor_ref(zend_jit_ctx *jit, int b, int pred, ir_ref ref) { int i, *p; zend_basic_block *bb; ir_ref *r, header; ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE); bb = &jit->ssa->cfg.blocks[b]; p = &jit->ssa->cfg.predecessors[bb->predecessor_offset]; r = &jit->bb_edges[jit->bb_predecessors[b]]; for (i = 0; i < bb->predecessors_count; i++, p++, r++) { if (*p == pred) { ZEND_ASSERT(*r == IR_UNUSED || *r == ref); header = jit->bb_start_ref[b]; if (header) { /* this is back edge */ ZEND_ASSERT(jit->ctx.ir_base[header].op == IR_LOOP_BEGIN); if (jit->ctx.ir_base[ref].op == IR_END) { jit->ctx.ir_base[ref].op = IR_LOOP_END; } else if (jit->ctx.ir_base[ref].op == IR_IF) { jit_IF_TRUE_FALSE_ex(jit, ref, b); ref = ir_LOOP_END(); } else if (jit->ctx.ir_base[ref].op == IR_SWITCH) { zend_jit_case_start(jit, pred, b, ref); ref = ir_LOOP_END(); } else if (jit->ctx.ir_base[ref].op == IR_UNREACHABLE) { ir_BEGIN(ref); ref = ir_LOOP_END(); } else { ZEND_UNREACHABLE(); } ir_MERGE_SET_OP(header, i + 1, ref); } *r = ref; return; } } ZEND_UNREACHABLE(); } static void _zend_jit_merge_smart_branch_inputs(zend_jit_ctx *jit, uint32_t true_label, uint32_t false_label, ir_ref true_inputs, ir_ref false_inputs) { ir_ref true_path = IR_UNUSED, false_path = IR_UNUSED; ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE); if (true_inputs) { ZEND_ASSERT(jit->ctx.ir_base[true_inputs].op == IR_END); if (!jit->ctx.ir_base[true_inputs].op2) { true_path = true_inputs; } else { ir_MERGE_list(true_inputs); true_path = ir_END(); } } if (false_inputs) { ZEND_ASSERT(jit->ctx.ir_base[false_inputs].op == IR_END); if (!jit->ctx.ir_base[false_inputs].op2) { false_path = false_inputs; } else { ir_MERGE_list(false_inputs); false_path = ir_END(); } } if (true_label == false_label && true_path && false_path) { ir_MERGE_2(true_path, false_path); _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END()); } else if (!true_path && !false_path) { /* dead code */ true_path = ir_END(); _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path); } else { if (true_path) { _zend_jit_add_predecessor_ref(jit, true_label, jit->b, true_path); } if (false_path) { _zend_jit_add_predecessor_ref(jit, false_label, jit->b, false_path); } } jit->b = -1; } static void _zend_jit_fix_merges(zend_jit_ctx *jit) { int i, count; ir_ref j, k, n, *p, *q, *r; ir_ref ref; ir_insn *insn, *phi; ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE); count = jit->ssa->cfg.blocks_count; for (i = 0, p = jit->bb_start_ref; i < count; i++, p++) { ref = *p; if (ref) { insn = &jit->ctx.ir_base[ref]; if (insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) { n = insn->inputs_count; /* Remove IS_UNUSED inputs */ for (j = k = 0, q = r = insn->ops + 1; j < n; j++, q++) { if (*q) { if (q != r) { *r = *q; phi = insn + 1 + (n >> 2); while (phi->op == IR_PI) { phi++; } while (phi->op == IR_PHI) { ir_insn_set_op(phi, k + 2, ir_insn_op(phi, j + 2)); phi += 1 + ((n + 1) >> 2); } } k++; r++; } } if (k != n) { ir_ref n2, k2; if (k <= 1) { insn->op = IR_BEGIN; insn->inputs_count = 0; } else { insn->inputs_count = k; } n2 = 1 + (n >> 2); k2 = 1 + (k >> 2); while (k2 != n2) { (insn+k2)->optx = IR_NOP; k2++; } phi = insn + 1 + (n >> 2); while (phi->op == IR_PI) { phi++; } while (phi->op == IR_PHI) { if (k <= 1) { phi->op = IR_COPY; phi->op1 = phi->op2; phi->op2 = 1; phi->inputs_count = 0; } else { phi->inputs_count = k + 1; } n2 = 1 + ((n + 1) >> 2); k2 = 1 + ((k + 1) >> 2); while (k2 != n2) { (phi+k2)->optx = IR_NOP; k2++; } phi += 1 + ((n + 1) >> 2); } } } } } } static void zend_jit_case_start(zend_jit_ctx *jit, int switch_b, int case_b, ir_ref switch_ref) { zend_basic_block *bb = &jit->ssa->cfg.blocks[switch_b]; const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1]; if (opline->opcode == ZEND_SWITCH_LONG || opline->opcode == ZEND_SWITCH_STRING || opline->opcode == ZEND_MATCH) { HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); int default_b = jit->ssa->cfg.map[default_opline - jit->op_array->opcodes]; zval *zv; ir_ref list = IR_UNUSED, idx; bool first = 1; ZEND_HASH_FOREACH_VAL(jumptable, zv) { const zend_op *target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); int b = jit->ssa->cfg.map[target - jit->op_array->opcodes]; if (b == case_b) { if (!first) { ir_END_list(list); } if (HT_IS_PACKED(jumptable)) { idx = ir_CONST_LONG(zv - jumptable->arPacked); } else { idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData); } ir_CASE_VAL(switch_ref, idx); first = 0; } } ZEND_HASH_FOREACH_END(); if (default_b == case_b) { if (!first) { ir_END_list(list); } if (jit->ctx.ir_base[switch_ref].op3) { /* op3 may contain a list of additional "default" path inputs for MATCH */ ir_ref ref = jit->ctx.ir_base[switch_ref].op3; jit->ctx.ir_base[switch_ref].op3 = IS_UNDEF; ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END); ir_ref end = ref; while (jit->ctx.ir_base[end].op2) { ZEND_ASSERT(jit->ctx.ir_base[end].op == IR_END); end = jit->ctx.ir_base[end].op2; } jit->ctx.ir_base[end].op2 = list; list = ref; } ir_CASE_DEFAULT(switch_ref); } if (list) { ir_END_list(list); ir_MERGE_list(list); } } else { ZEND_UNREACHABLE(); } } static int zend_jit_bb_start(zend_jit_ctx *jit, int b) { zend_basic_block *bb; int i, n, *p, pred; ir_ref ref, bb_start; ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE); ZEND_ASSERT(b < jit->ssa->cfg.blocks_count); bb = &jit->ssa->cfg.blocks[b]; ZEND_ASSERT((bb->flags & ZEND_BB_REACHABLE) != 0); n = bb->predecessors_count; if (n == 0) { /* pass */ ZEND_ASSERT(jit->ctx.control); #if ZEND_DEBUG ref = jit->ctx.control; ir_insn *insn = &jit->ctx.ir_base[ref]; while (insn->op >= IR_CALL && insn->op <= IR_TRAP) { ref = insn->op1; insn = &jit->ctx.ir_base[ref]; } ZEND_ASSERT(insn->op == IR_START); ZEND_ASSERT(ref == 1); #endif bb_start = 1; if (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) { /* prvent END/BEGIN merging */ jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ir_END()); bb_start = jit->ctx.control; } } else if (n == 1) { ZEND_ASSERT(!jit->ctx.control); pred = jit->ssa->cfg.predecessors[bb->predecessor_offset]; ref = jit->bb_edges[jit->bb_predecessors[b]]; if (ref == IR_UNUSED) { if (!jit->ctx.control) { ir_BEGIN(IR_UNUSED); /* unreachable block */ } } else { ir_op op = jit->ctx.ir_base[ref].op; if (op == IR_IF) { if (!jit->ctx.control) { jit_IF_TRUE_FALSE_ex(jit, ref, b); } else { ir_ref entry_path = ir_END(); jit_IF_TRUE_FALSE_ex(jit, ref, b); ir_MERGE_WITH(entry_path); } } else if (op == IR_SWITCH) { zend_jit_case_start(jit, pred, b, ref); } else { if (!jit->ctx.control) { ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN); if ((jit->ssa->cfg.blocks[b].flags & ZEND_BB_RECV_ENTRY) && (jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY)) { /* prvent END/BEGIN merging */ jit->ctx.control = ir_emit1(&jit->ctx, IR_BEGIN, ref); } else { ir_BEGIN(ref); } } else { ir_MERGE_WITH(ref); } } } bb_start = jit->ctx.control; } else { int forward_edges_count = 0; int back_edges_count = 0; ir_ref *pred_refs; ir_ref entry_path = IR_UNUSED; ALLOCA_FLAG(use_heap); ZEND_ASSERT(!jit->ctx.control); if (jit->ctx.control) { entry_path = ir_END(); } pred_refs = (ir_ref *)do_alloca(sizeof(ir_ref) * n, use_heap); for (i = 0, p = jit->ssa->cfg.predecessors + bb->predecessor_offset; i < n; p++, i++) { pred = *p; if (jit->bb_start_ref[pred]) { /* forward edge */ forward_edges_count++; ref = jit->bb_edges[jit->bb_predecessors[b] + i]; if (ref == IR_UNUSED) { /* dead edge */ pred_refs[i] = IR_UNUSED; } else { ir_op op = jit->ctx.ir_base[ref].op; if (op == IR_IF) { jit_IF_TRUE_FALSE_ex(jit, ref, b); pred_refs[i] = ir_END(); } else if (op == IR_SWITCH) { zend_jit_case_start(jit, pred, b, ref); pred_refs[i] = ir_END(); } else { ZEND_ASSERT(op == IR_END || op == IR_UNREACHABLE || op == IR_RETURN); pred_refs[i] = ref; } } } else { /* backward edge */ back_edges_count++; pred_refs[i] = IR_UNUSED; } } if (bb->flags & ZEND_BB_LOOP_HEADER) { ZEND_ASSERT(back_edges_count != 0); ZEND_ASSERT(forward_edges_count != 0); ir_MERGE_N(n, pred_refs); jit->ctx.ir_base[jit->ctx.control].op = IR_LOOP_BEGIN; bb_start = jit->ctx.control; if (entry_path) { ir_MERGE_WITH(entry_path); } } else { // ZEND_ASSERT(back_edges_count != 0); /* edges from exceptional blocks may be counted as back edges */ ir_MERGE_N(n, pred_refs); bb_start = jit->ctx.control; if (entry_path) { ir_MERGE_WITH(entry_path); } } free_alloca(pred_refs, use_heap); } jit->b = b; jit->bb_start_ref[b] = bb_start; if ((bb->flags & ZEND_BB_ENTRY) || (bb->idom >= 0 && jit->bb_start_ref[bb->idom] < jit->ctx.fold_cse_limit)) { jit->ctx.fold_cse_limit = bb_start; } return 1; } static int zend_jit_bb_end(zend_jit_ctx *jit, int b) { int succ; zend_basic_block *bb; ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE); if (jit->b != b) { return 1; } bb = &jit->ssa->cfg.blocks[b]; ZEND_ASSERT(bb->successors_count != 0); if (bb->successors_count == 1) { succ = bb->successors[0]; } else { const zend_op *opline = &jit->op_array->opcodes[bb->start + bb->len - 1]; /* Use only the following successor of SWITCH and FE_RESET_R */ ZEND_ASSERT(opline->opcode == ZEND_SWITCH_LONG || opline->opcode == ZEND_SWITCH_STRING || opline->opcode == ZEND_MATCH || opline->opcode == ZEND_FE_RESET_R); succ = b + 1; } _zend_jit_add_predecessor_ref(jit, succ, b, ir_END()); jit->b = -1; return 1; } static int jit_CMP_IP(zend_jit_ctx *jit, ir_op op, const zend_op *next_opline) { ir_ref ref; #if 1 ref = jit_IP32(jit); ref = ir_CMP_OP(op, ref, ir_CONST_U32((uint32_t)(uintptr_t)next_opline)); #else ref = jit_IP(jit); ref = ir_CMP_OP(op, ref, ir_CONST_ADDR(next_opline)); #endif return ref; } static int zend_jit_jmp_frameless( zend_jit_ctx *jit, const zend_op *opline, const void *exit_addr, zend_jmp_fl_result guard ) { ir_ref ref, if_ref, cache_result, function_result, phi_result, cache_slot_ref; zend_basic_block *bb; // JIT: CACHED_PTR(opline->extended_value) cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value); cache_result = ir_LOAD_L(cache_slot_ref); // JIT: if (UNEXPECTED(!result)) if_ref = ir_IF(cache_result); ir_IF_FALSE_cold(if_ref); zval *func_name_zv = RT_CONSTANT(opline, opline->op1); function_result = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_jit_jmp_frameless_helper), ir_CONST_ADDR(func_name_zv), cache_slot_ref); ir_MERGE_WITH_EMPTY_TRUE(if_ref); phi_result = ir_PHI_2(IR_LONG, function_result, cache_result); if (exit_addr) { ir_GUARD(ir_EQ(phi_result, ir_CONST_LONG(guard)), ir_CONST_ADDR(exit_addr)); } else { ZEND_ASSERT(jit->b >= 0); bb = &jit->ssa->cfg.blocks[jit->b]; // JIT: if (result == ZEND_JMP_FL_HIT) ref = jit_IF_ex(jit, ir_EQ(phi_result, ir_CONST_LONG(ZEND_JMP_FL_HIT)), bb->successors[0]); _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); jit->b = -1; } return 1; } static int zend_jit_cond_jmp(zend_jit_ctx *jit, const zend_op *next_opline, int target_block) { ir_ref ref; zend_basic_block *bb; ZEND_ASSERT(jit->b >= 0); bb = &jit->ssa->cfg.blocks[jit->b]; ZEND_ASSERT(bb->successors_count == 2); if (bb->successors[0] == bb->successors[1]) { _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END()); jit->b = -1; zend_jit_set_last_valid_opline(jit, next_opline); return 1; } ref = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, next_opline), target_block); _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); jit->b = -1; zend_jit_set_last_valid_opline(jit, next_opline); return 1; } static int zend_jit_set_cond(zend_jit_ctx *jit, const zend_op *next_opline, uint32_t var) { ir_ref ref; ref = ir_ADD_U32(ir_ZEXT_U32(jit_CMP_IP(jit, IR_EQ, next_opline)), ir_CONST_U32(IS_FALSE)); // EX_VAR(var) = ... ir_STORE(ir_ADD_OFFSET(jit_FP(jit), var + offsetof(zval, u1.type_info)), ref); zend_jit_reset_last_valid_opline(jit); return zend_jit_set_ip(jit, next_opline - 1); } /* PHP JIT handlers */ static void zend_jit_check_exception(zend_jit_ctx *jit) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler)); } static void zend_jit_check_exception_undef_result(zend_jit_ctx *jit, const zend_op *opline) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, (opline->result_type & (IS_TMP_VAR|IS_VAR)) ? jit_stub_exception_handler_undef : jit_stub_exception_handler)); } static void zend_jit_type_check_undef(zend_jit_ctx *jit, ir_ref type, uint32_t var, const zend_op *opline, bool check_exception, bool in_cold_path, bool undef_result) { ir_ref if_def = ir_IF(type); if (!in_cold_path) { ir_IF_FALSE_cold(if_def); } else { ir_IF_FALSE(if_def); } if (opline) { jit_SET_EX_OPLINE(jit, opline); } ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var)); if (check_exception) { if (undef_result) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); } } ir_MERGE_WITH_EMPTY_TRUE(if_def); } static ir_ref zend_jit_zval_check_undef(zend_jit_ctx *jit, ir_ref ref, uint32_t var, const zend_op *opline, bool check_exception) { ir_ref if_def, ref2; if_def = ir_IF(jit_Z_TYPE_ref(jit, ref)); ir_IF_FALSE_cold(if_def); if (opline) { jit_SET_EX_OPLINE(jit, opline); } ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(var)); if (check_exception) { zend_jit_check_exception(jit); } ref2 = jit_EG(uninitialized_zval); ir_MERGE_WITH_EMPTY_TRUE(if_def); return ir_PHI_2(IR_ADDR, ref2, ref); } static void zend_jit_recv_entry(zend_jit_ctx *jit, int b) { zend_basic_block *bb = &jit->ssa->cfg.blocks[b]; int pred; ir_ref ref; ZEND_ASSERT(bb->predecessors_count > 0); pred = jit->bb_predecessors[b]; ref = jit->bb_edges[pred]; ZEND_ASSERT(ref); ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_END); /* Insert a MERGE block with additional ENTRY input between predecessor and this one */ ir_ENTRY(ref, bb->start); if (!GCC_GLOBAL_REGS && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) { /* 2 and 3 are hardcoded reference to IR_PARAMs */ ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM); ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1); jit_STORE_FP(jit, 2); ZEND_ASSERT(jit->ctx.ir_base[3].op == IR_PARAM); ZEND_ASSERT(jit->ctx.ir_base[3].op3 == 2); jit_STORE_IP(jit, 3); } ir_MERGE_WITH(ref); jit->bb_edges[pred] = ir_END(); } static void zend_jit_osr_entry(zend_jit_ctx *jit, int b) { zend_basic_block *bb = &jit->ssa->cfg.blocks[b]; ir_ref ref = ir_END(); /* Insert a MERGE block with additional ENTRY input between predecessor and this one */ ir_ENTRY(ref, bb->start); if (!GCC_GLOBAL_REGS && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) { /* 2 and 3 are hardcoded reference to IR_PARAMs */ ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM); ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1); jit_STORE_FP(jit, 2); ZEND_ASSERT(jit->ctx.ir_base[3].op == IR_PARAM); ZEND_ASSERT(jit->ctx.ir_base[3].op3 == 2); jit_STORE_IP(jit, 3); } ir_MERGE_WITH(ref); } static ir_ref zend_jit_continue_entry(zend_jit_ctx *jit, ir_ref src, unsigned int label) { ir_ENTRY(src, label); if (!GCC_GLOBAL_REGS && ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) { /* 2 and 3 are hardcoded reference to IR_PARAMs */ ZEND_ASSERT(jit->ctx.ir_base[2].op == IR_PARAM); ZEND_ASSERT(jit->ctx.ir_base[2].op3 == 1); jit_STORE_FP(jit, 2); ZEND_ASSERT(jit->ctx.ir_base[3].op == IR_PARAM); ZEND_ASSERT(jit->ctx.ir_base[3].op3 == 2); jit_STORE_IP(jit, 3); } return ir_END(); } static int zend_jit_handler(zend_jit_ctx *jit, const zend_op *opline, int may_throw) { zend_jit_set_ip(jit, opline); if (GCC_GLOBAL_REGS) { zend_vm_opcode_handler_func_t handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(opline); ir_CALL(IR_VOID, ir_CONST_FUNC(handler)); } else if (ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { zend_vm_opcode_handler_func_t handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(opline); ir_ref ip = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(handler), jit_FP(jit), jit_IP(jit)); jit_STORE_IP(jit, ip); } else { zend_vm_opcode_handler_t handler = opline->handler; ir_ref ip = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(handler), jit_FP(jit), jit_IP(jit)); jit_STORE_IP(jit, ip); } if (may_throw) { zend_jit_check_exception(jit); } /* Skip the following OP_DATA */ switch (opline->opcode) { case ZEND_ASSIGN_DIM: case ZEND_ASSIGN_OBJ: case ZEND_ASSIGN_STATIC_PROP: case ZEND_ASSIGN_DIM_OP: case ZEND_ASSIGN_OBJ_OP: case ZEND_ASSIGN_STATIC_PROP_OP: case ZEND_ASSIGN_STATIC_PROP_REF: case ZEND_ASSIGN_OBJ_REF: case ZEND_FRAMELESS_ICALL_3: case ZEND_DECLARE_ATTRIBUTED_CONST: zend_jit_set_last_valid_opline(jit, opline + 2); break; default: zend_jit_set_last_valid_opline(jit, opline + 1); break; } return 1; } static int zend_jit_tail_handler(zend_jit_ctx *jit, const zend_op *opline) { ir_ref ref; zend_basic_block *bb; zend_jit_set_ip(jit, opline); if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) { if (opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME || opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_RETURN) { /* Use inlined HYBRID VM handler */ zend_vm_opcode_handler_t handler = opline->handler; ir_TAILCALL(IR_VOID, ir_CONST_FUNC(handler)); } else { zend_vm_opcode_handler_func_t handler = (zend_vm_opcode_handler_func_t)zend_get_opcode_handler_func(opline); ir_CALL(IR_VOID, ir_CONST_FUNC(handler)); ref = ir_LOAD_A(jit_IP(jit)); ir_TAILCALL(IR_VOID, ref); } } else { zend_vm_opcode_handler_t handler = opline->handler; if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { zend_jit_tailcall_handler(jit, ir_CONST_OPCODE_HANDLER_FUNC(handler)); } else if ((jit->ssa->cfg.flags & ZEND_FUNC_RECURSIVE_DIRECTLY) && (opline->opcode == ZEND_CATCH || opline->opcode == ZEND_FAST_CALL || opline->opcode == ZEND_FAST_RET || opline->opcode == ZEND_MATCH_ERROR || opline->opcode == ZEND_THROW || opline->opcode == ZEND_VERIFY_NEVER_TYPE)) { ir_ref ip = ir_CALL_2(IR_ADDR, ir_CONST_OPCODE_HANDLER_FUNC(handler), jit_FP(jit), jit_IP(jit)); zend_jit_vm_enter(jit, ip); } else { ir_TAILCALL_2(IR_ADDR, ir_CONST_OPCODE_HANDLER_FUNC(handler), jit_FP(jit), jit_IP(jit)); } } if (jit->b >= 0) { bb = &jit->ssa->cfg.blocks[jit->b]; if (bb->successors_count > 0 && (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME || opline->opcode == ZEND_INCLUDE_OR_EVAL || opline->opcode == ZEND_GENERATOR_CREATE || opline->opcode == ZEND_YIELD || opline->opcode == ZEND_YIELD_FROM || opline->opcode == ZEND_FAST_CALL)) { /* Add a fake control edge from UNREACHABLE to the following ENTRY */ int succ; if (bb->successors_count == 1) { succ = bb->successors[0]; ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY); } else { /* Use only the following successor of FAST_CALL */ ZEND_ASSERT(opline->opcode == ZEND_FAST_CALL); succ = jit->b + 1; /* we need an entry */ jit->ssa->cfg.blocks[succ].flags |= ZEND_BB_ENTRY; } ref = jit->ctx.insns_count - 1; ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE || jit->ctx.ir_base[ref].op == IR_RETURN); ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start); _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref); } jit->b = -1; zend_jit_reset_last_valid_opline(jit); } return 1; } static int zend_jit_call(zend_jit_ctx *jit, const zend_op *opline, unsigned int next_block) { return zend_jit_tail_handler(jit, opline); } static int zend_jit_spill_store(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info, bool set_type) { ZEND_ASSERT(Z_MODE(src) == IS_REG); ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src)); if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src)); if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); } } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_spill_store_inv(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { ZEND_ASSERT(Z_MODE(src) == IS_REG); ZEND_ASSERT(Z_MODE(dst) == IS_MEM_ZVAL); if (Z_LOAD(src) || Z_STORE(src)) { /* it's not necessary to store register if it was previously loaded or already stored */ return 1; } if ((info & MAY_BE_ANY) == MAY_BE_LONG) { jit_set_Z_LVAL(jit, dst, zend_jit_use_reg(jit, src)); if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) { jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); } else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG) { /* invalidate memory type */ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN; jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { jit_set_Z_DVAL(jit, dst, zend_jit_use_reg(jit, src)); if (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame)) { jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); } else if (STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE) { /* invalidate memory type */ STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) = IS_UNKNOWN; jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); } } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_load_reg(zend_jit_ctx *jit, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { ZEND_ASSERT(Z_MODE(src) == IS_MEM_ZVAL); ZEND_ASSERT(Z_MODE(dst) == IS_REG); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { zend_jit_def_reg(jit, dst, jit_Z_LVAL(jit, src)); } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { zend_jit_def_reg(jit, dst, jit_Z_DVAL(jit, src)); } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_store_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var, bool set_type) { zend_jit_addr src = ZEND_ADDR_REG(ssa_var); zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); return zend_jit_spill_store(jit, src, dst, info, set_type); } static int zend_jit_store_ref(zend_jit_ctx *jit, uint32_t info, int var, int32_t src, bool set_type) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { jit_set_Z_LVAL(jit, dst, src); if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { jit_set_Z_DVAL(jit, dst, src); if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); } } else { ZEND_UNREACHABLE(); } return 1; } static ir_ref zend_jit_deopt_rload(zend_jit_ctx *jit, ir_type type, int32_t reg) { ir_ref ref = jit->ctx.control; ir_insn *insn; while (1) { insn = &jit->ctx.ir_base[ref]; if (insn->op == IR_RLOAD && insn->op2 == reg) { ZEND_ASSERT(insn->type == type); return ref; } else if (insn->op == IR_START) { break; } ref = insn->op1; } return ir_RLOAD(type, reg); } /* Same as zend_jit_deopt_rload(), but 'reg' may be spilled on C stack */ static ir_ref zend_jit_deopt_rload_spilled(zend_jit_ctx *jit, ir_type type, int8_t reg, int32_t offset) { ZEND_ASSERT(reg >= 0); if (IR_REG_SPILLED(reg)) { return ir_LOAD(type, ir_ADD_OFFSET(zend_jit_deopt_rload(jit, type, IR_REG_NUM(reg)), offset)); } else { return zend_jit_deopt_rload(jit, type, reg); } } static int zend_jit_store_const_long(zend_jit_ctx *jit, int var, zend_long val) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); ir_ref src = ir_CONST_LONG(val); if (jit->ra && jit->ra[var].ref == IR_NULL) { zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); } jit_set_Z_LVAL(jit, dst, src); jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); return 1; } static int zend_jit_store_const_double(zend_jit_ctx *jit, int var, double val) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); ir_ref src = ir_CONST_DOUBLE(val); if (jit->ra && jit->ra[var].ref == IR_NULL) { zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); } jit_set_Z_DVAL(jit, dst, src); jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); return 1; } static int zend_jit_store_type(zend_jit_ctx *jit, int var, uint8_t type) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); ZEND_ASSERT(type <= IS_DOUBLE); jit_set_Z_TYPE_INFO(jit, dst, type); return 1; } static int zend_jit_store_reg(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, bool in_mem, bool set_type) { zend_jit_addr src; zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); ir_type type; if ((info & MAY_BE_ANY) == MAY_BE_LONG) { type = IR_LONG; src = zend_jit_deopt_rload(jit, type, reg); if (jit->ra && jit->ra[var].ref == IR_NULL) { zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); } else if (!in_mem) { jit_set_Z_LVAL(jit, dst, src); if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); } } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { type = IR_DOUBLE; src = zend_jit_deopt_rload(jit, type, reg); if (jit->ra && jit->ra[var].ref == IR_NULL) { zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); } else if (!in_mem) { jit_set_Z_DVAL(jit, dst, src); if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); } } } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_store_spill_slot(zend_jit_ctx *jit, uint32_t info, int var, int8_t reg, int32_t offset, bool set_type) { zend_jit_addr src; zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); if ((info & MAY_BE_ANY) == MAY_BE_LONG) { src = ir_LOAD_L(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset)); if (jit->ra && jit->ra[var].ref == IR_NULL) { zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); } else { jit_set_Z_LVAL(jit, dst, src); if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_LONG)) { jit_set_Z_TYPE_INFO(jit, dst, IS_LONG); } } } else if ((info & MAY_BE_ANY) == MAY_BE_DOUBLE) { src = ir_LOAD_D(ir_ADD_OFFSET(ir_RLOAD_A(reg), offset)); if (jit->ra && jit->ra[var].ref == IR_NULL) { zend_jit_def_reg(jit, ZEND_ADDR_REG(var), src); } else { jit_set_Z_DVAL(jit, dst, src); if (set_type && (Z_REG(dst) != ZREG_FP || !JIT_G(current_frame) || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(dst))) != IS_DOUBLE)) { jit_set_Z_TYPE_INFO(jit, dst, IS_DOUBLE); } } } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_store_var_type(zend_jit_ctx *jit, int var, uint32_t type) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); jit_set_Z_TYPE_INFO(jit, dst, type); return 1; } static int zend_jit_zval_try_addref(zend_jit_ctx *jit, zend_jit_addr var_addr) { ir_ref if_refcounted, end1; if_refcounted = jit_if_REFCOUNTED(jit, var_addr); ir_IF_FALSE(if_refcounted); end1 = ir_END(); ir_IF_TRUE(if_refcounted); jit_GC_ADDREF(jit, jit_Z_PTR(jit, var_addr)); ir_MERGE_WITH(end1); return 1; } static int zend_jit_store_var_if_necessary(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info) { if (Z_MODE(src) == IS_REG && Z_STORE(src)) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); return zend_jit_spill_store(jit, src, dst, info, 1); } return 1; } static int zend_jit_store_var_if_necessary_ex(zend_jit_ctx *jit, int var, zend_jit_addr src, uint32_t info, zend_jit_addr old, uint32_t old_info) { if (Z_MODE(src) == IS_REG && Z_STORE(src)) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); bool set_type = 1; if ((info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == (old_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF))) { if (Z_MODE(old) != IS_REG || Z_LOAD(old) || Z_STORE(old)) { if (JIT_G(current_frame)) { uint32_t mem_type = STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)); if (mem_type != IS_UNKNOWN && (info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == (1 << mem_type)) { set_type = 0; } } else { set_type = 0; } } } return zend_jit_spill_store(jit, src, dst, info, set_type); } return 1; } static int zend_jit_load_var(zend_jit_ctx *jit, uint32_t info, int var, int ssa_var) { zend_jit_addr src = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); zend_jit_addr dst = ZEND_ADDR_REG(ssa_var); return zend_jit_load_reg(jit, src, dst, info); } static int zend_jit_invalidate_var_if_necessary(zend_jit_ctx *jit, uint8_t op_type, zend_jit_addr addr, znode_op op) { if ((op_type & (IS_TMP_VAR|IS_VAR)) && Z_MODE(addr) == IS_REG && !Z_LOAD(addr) && !Z_STORE(addr)) { /* Invalidate operand type to prevent incorrect destuction by exception_handler_free_op1_op2() */ zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op.var); jit_set_Z_TYPE_INFO(jit, dst, IS_UNDEF); } return 1; } static int zend_jit_update_regs(zend_jit_ctx *jit, uint32_t var, zend_jit_addr src, zend_jit_addr dst, uint32_t info) { if (!zend_jit_same_addr(src, dst)) { if (Z_MODE(src) == IS_REG) { if (Z_MODE(dst) == IS_REG) { zend_jit_def_reg(jit, dst, zend_jit_use_reg(jit, src)); if (!Z_LOAD(src) && !Z_STORE(src) && Z_STORE(dst)) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!zend_jit_spill_store(jit, dst, var_addr, info, JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) )) { return 0; } } } else if (Z_MODE(dst) == IS_MEM_ZVAL) { if (!Z_LOAD(src) && !Z_STORE(src)) { if (!zend_jit_spill_store(jit, src, dst, info, JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) )) { return 0; } } } else { ZEND_UNREACHABLE(); } } else if (Z_MODE(src) == IS_MEM_ZVAL) { if (Z_MODE(dst) == IS_REG) { if (!zend_jit_load_reg(jit, src, dst, info)) { return 0; } } else { ZEND_UNREACHABLE(); } } else { ZEND_UNREACHABLE(); } } else if (Z_MODE(dst) == IS_REG && Z_STORE(dst)) { dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!zend_jit_spill_store(jit, src, dst, info, JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || JIT_G(current_frame) == NULL || STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var)) == IS_UNKNOWN || (1 << STACK_MEM_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(var))) != (info & MAY_BE_ANY) )) { return 0; } } return 1; } struct jit_observer_fcall_is_unobserved_data { ir_ref if_unobserved; ir_ref ir_end_inputs; }; static struct jit_observer_fcall_is_unobserved_data jit_observer_fcall_is_unobserved_start(zend_jit_ctx *jit, const zend_function *func, ir_ref *observer_handler, ir_ref rx, ir_ref func_ref) { ir_ref run_time_cache; struct jit_observer_fcall_is_unobserved_data data = { .ir_end_inputs = IR_UNUSED }; if (func) { ZEND_ASSERT((func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0); } else { // JIT: if (function->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) { ZEND_ASSERT(rx != IR_UNUSED); ir_ref if_trampoline_or_generator = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))), ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR))); ir_IF_TRUE(if_trampoline_or_generator); ir_END_list(data.ir_end_inputs); ir_IF_FALSE(if_trampoline_or_generator); } if (func && (func->common.fn_flags & ZEND_ACC_CLOSURE) == 0 && ZEND_MAP_PTR_IS_OFFSET(func->common.run_time_cache)) { // JIT: ZEND_MAP_PTR_GET_IMM(func->common.runtime_cache) run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)), (uintptr_t)ZEND_MAP_PTR(func->common.run_time_cache))); #ifndef ZTS } else if (func && rx == IS_UNUSED) { // happens for internal functions only ZEND_ASSERT(!ZEND_USER_CODE(func->type)); run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_CONST_ADDR(func), offsetof(zend_op_array, run_time_cache__ptr))); #endif } else { // Closures may be duplicated and have a different runtime cache. Use the regular run_time_cache access pattern for these if (func && ZEND_USER_CODE(func->type)) { // not a closure and definitely not an internal function run_time_cache = ir_LOAD_A(jit_CALL(rx, run_time_cache)); } else { // JIT: ZEND_MAP_PTR_GET(func->common.runtime_cache) run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_op_array, run_time_cache__ptr))); ir_ref if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1))); ir_IF_TRUE(if_odd); ir_ref run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base)))); ir_ref if_odd_end = ir_END(); ir_IF_FALSE(if_odd); // JIT: if (func->common.runtime_cache != NULL) { ir_ref if_rt_cache = ir_IF(ir_EQ(run_time_cache, IR_NULL)); ir_IF_TRUE(if_rt_cache); ir_END_list(data.ir_end_inputs); ir_IF_FALSE(if_rt_cache); ir_MERGE_WITH(if_odd_end); run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache, run_time_cache2); } } // JIT: observer_handler = runtime_cache + ZEND_OBSERVER_HANDLE(function) if (func) { *observer_handler = ir_ADD_OFFSET(run_time_cache, ZEND_OBSERVER_HANDLE(func) * sizeof(void *)); } else { // JIT: (func->type == ZEND_INTERNAL_FUNCTION ? zend_observer_fcall_internal_function_extension : zend_observer_fcall_op_array_extension) * sizeof(void *) ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type))); ir_ref if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(ZEND_INTERNAL_FUNCTION))); ir_IF_TRUE(if_internal_func); ir_ref observer_handler_internal = ir_ADD_OFFSET(run_time_cache, zend_observer_fcall_internal_function_extension * sizeof(void *)); ir_ref if_internal_func_end = ir_END(); ir_IF_FALSE(if_internal_func); ir_ref observer_handler_user = ir_ADD_OFFSET(run_time_cache, zend_observer_fcall_op_array_extension * sizeof(void *)); ir_MERGE_WITH(if_internal_func_end); *observer_handler = ir_PHI_2(IR_ADDR, observer_handler_internal, observer_handler_user); } // JIT: if (*observer_handler == ZEND_OBSERVER_NONE_OBSERVED) { data.if_unobserved = ir_IF(ir_EQ(ir_LOAD_A(*observer_handler), ir_CONST_ADDR(ZEND_OBSERVER_NONE_OBSERVED))); ir_IF_FALSE(data.if_unobserved); return data; } /* For frameless the true branch of if_unobserved is used and this function not called. */ static void jit_observer_fcall_is_unobserved_end(zend_jit_ctx *jit, struct jit_observer_fcall_is_unobserved_data *data) { ir_END_list(data->ir_end_inputs); ir_IF_TRUE(data->if_unobserved); ir_END_list(data->ir_end_inputs); ir_MERGE_list(data->ir_end_inputs); } static void jit_observer_fcall_begin(zend_jit_ctx *jit, ir_ref rx, ir_ref observer_handler) { ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_begin_prechecked), rx, observer_handler); } static void jit_observer_fcall_end(zend_jit_ctx *jit, ir_ref rx, ir_ref res_ref) { // JIT: if (execute_data == EG(current_observed_frame)) { ir_ref has_end_observer = ir_IF(ir_EQ(rx, ir_LOAD_A(jit_EG(current_observed_frame)))); ir_IF_TRUE(has_end_observer); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_observer_fcall_end_prechecked), rx, res_ref); ir_MERGE_WITH_EMPTY_FALSE(has_end_observer); } static int zend_jit_inc_dec(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) { ir_ref if_long = IR_UNUSED; ir_ref op1_lval_ref = IR_UNUSED; ir_ref ref; ir_op op; if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_LONG)) { if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_TRUE(if_long); } if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { op1_lval_ref = jit_Z_LVAL(jit, op1_addr); jit_set_Z_LVAL(jit, res_addr, op1_lval_ref); if (Z_MODE(res_addr) != IS_REG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } } if (Z_MODE(op1_def_addr) == IS_MEM_ZVAL && Z_MODE(op1_addr) == IS_REG && !Z_LOAD(op1_addr) && !Z_STORE(op1_addr)) { jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_LONG); } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { op = may_overflow ? IR_ADD_OV : IR_ADD; } else { op = may_overflow ? IR_SUB_OV : IR_SUB; } if (!op1_lval_ref) { op1_lval_ref = jit_Z_LVAL(jit, op1_addr); } ref = ir_BINARY_OP_L(op, op1_lval_ref, ir_CONST_LONG(1)); if (op1_def_info & MAY_BE_LONG) { jit_set_Z_LVAL(jit, op1_def_addr, ref); } if (may_overflow && (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD)) || ((opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_LONG|MAY_BE_GUARD))))) { int32_t exit_point; const void *exit_addr; zend_jit_trace_stack *stack; uint32_t old_op1_info, old_res_info = 0; stack = JIT_G(current_frame)->stack; old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_DOUBLE, 0); if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); } else { SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); } if (opline->result_type != IS_UNUSED) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); if (opline->opcode == ZEND_PRE_INC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); } else if (opline->opcode == ZEND_PRE_DEC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); } else if (opline->opcode == ZEND_POST_INC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MAX)); } else if (opline->opcode == ZEND_POST_DEC) { SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_LONG(ZEND_LONG_MIN)); } } exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr)); if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { jit_set_Z_LVAL(jit, res_addr, ref); if (Z_MODE(res_addr) != IS_REG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } } SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); if (opline->result_type != IS_UNUSED) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } } else if (may_overflow) { ir_ref if_overflow; ir_ref merge_inputs = IR_UNUSED; if (((op1_def_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD)) || (opline->result_type != IS_UNUSED && (res_info & (MAY_BE_ANY|MAY_BE_GUARD)) == (MAY_BE_DOUBLE|MAY_BE_GUARD))) { int32_t exit_point; const void *exit_addr; zend_jit_trace_stack *stack; uint32_t old_res_info = 0, old_op1_info = 0; stack = JIT_G(current_frame)->stack; if (opline->result_type != IS_UNUSED) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_LONG, 0); if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) { SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ref); } else { SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), op1_lval_ref); } } old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_LONG, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->op1.var), ref); exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr)); if (opline->result_type != IS_UNUSED) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); } else { if_overflow = ir_IF(ir_OVERFLOW(ref)); ir_IF_FALSE(if_overflow); if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { jit_set_Z_LVAL(jit, res_addr, ref); if (Z_MODE(res_addr) != IS_REG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } } ir_END_list(merge_inputs); /* overflow => cold path */ ir_IF_TRUE_cold(if_overflow); } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { if (Z_MODE(op1_def_addr) == IS_REG) { jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); } else { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0)); jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0x41e00000)); #else jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x43e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE); } } else { if (Z_MODE(op1_def_addr) == IS_REG) { jit_set_Z_DVAL(jit, op1_def_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); } else { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0x00200000)); jit_set_Z_W2(jit, op1_def_addr, ir_CONST_U32(0xc1e00000)); #else jit_set_Z_LVAL(jit, op1_def_addr, ir_CONST_LONG(0xc3e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_DOUBLE); } } if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { if (opline->opcode == ZEND_PRE_INC) { if (Z_MODE(res_addr) == IS_REG) { jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); } else { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0)); jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000)); #else jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } } else { if (Z_MODE(res_addr) == IS_REG) { jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); } else { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000)); jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000)); #else jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } } } if (merge_inputs) { ir_END_list(merge_inputs); ir_MERGE_list(merge_inputs); } } else { if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { jit_set_Z_LVAL(jit, res_addr, ref); if (Z_MODE(res_addr) != IS_REG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { ir_ref merge_inputs = ir_END(); /* !is_long => cold path */ ir_IF_FALSE_cold(if_long); if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { jit_SET_EX_OPLINE(jit, opline); if (op1_info & MAY_BE_UNDEF) { ir_ref if_def; if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); ir_IF_FALSE_cold(if_def); // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var)); jit_set_Z_TYPE_INFO(jit, op1_def_addr, IS_NULL); ir_MERGE_WITH_EMPTY_TRUE(if_def); op1_info |= MAY_BE_NULL; } ref = jit_ZVAL_ADDR(jit, op1_addr); if (op1_info & MAY_BE_REF) { ir_ref if_ref, if_typed, func, ref2, arg2; if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE)); ir_IF_TRUE(if_ref); ref2 = jit_Z_PTR_ref(jit, ref); if_typed = jit_if_TYPED_REF(jit, ref2); ir_IF_TRUE(if_typed); if (RETURN_VALUE_USED(opline)) { ZEND_ASSERT(Z_MODE(res_addr) != IS_REG); arg2 = jit_ZVAL_ADDR(jit, res_addr); } else { arg2 = IR_NULL; } if (opline->opcode == ZEND_PRE_INC) { func = ir_CONST_FC_FUNC(zend_jit_pre_inc_typed_ref); } else if (opline->opcode == ZEND_PRE_DEC) { func = ir_CONST_FC_FUNC(zend_jit_pre_dec_typed_ref); } else if (opline->opcode == ZEND_POST_INC) { func = ir_CONST_FC_FUNC(zend_jit_post_inc_typed_ref); } else if (opline->opcode == ZEND_POST_DEC) { func = ir_CONST_FC_FUNC(zend_jit_post_dec_typed_ref); } else { ZEND_UNREACHABLE(); } ir_CALL_2(IR_VOID, func, ref2, arg2); zend_jit_check_exception(jit); ir_END_list(merge_inputs); ir_IF_FALSE(if_typed); ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val)); ir_MERGE_WITH_EMPTY_FALSE(if_ref); ref = ir_PHI_2(IR_ADDR, ref2, ref); } if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { jit_ZVAL_COPY(jit, res_addr, res_use_info, ZEND_ADDR_REF_ZVAL(ref), op1_info, 1); } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { if (opline->opcode == ZEND_PRE_INC && opline->result_type != IS_UNUSED) { ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc), ref, arg2); } else { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function), ref); } } else { if (opline->opcode == ZEND_PRE_DEC && opline->result_type != IS_UNUSED) { ir_ref arg2 = jit_ZVAL_ADDR(jit, res_addr); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec), ref, arg2); } else { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function), ref); } } if (may_throw) { zend_jit_check_exception(jit); } } else { ref = jit_Z_DVAL(jit, op1_addr); if (opline->opcode == ZEND_POST_INC || opline->opcode == ZEND_POST_DEC) { jit_set_Z_DVAL(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } if (opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_POST_INC) { op = IR_ADD; } else { op = IR_SUB; } ref = ir_BINARY_OP_D(op, ref, ir_CONST_DOUBLE(1.0)); jit_set_Z_DVAL(jit, op1_def_addr, ref); if ((opline->opcode == ZEND_PRE_INC || opline->opcode == ZEND_PRE_DEC) && opline->result_type != IS_UNUSED) { jit_set_Z_DVAL(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } } ir_END_list(merge_inputs); ir_MERGE_list(merge_inputs); } if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { return 0; } if (opline->result_type != IS_UNUSED) { if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { return 0; } } return 1; } static int zend_jit_math_long_long(zend_jit_ctx *jit, const zend_op *opline, uint8_t opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, int may_overflow) { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); ir_op op; ir_ref op1, op2, ref, if_overflow = IR_UNUSED; if (opcode == ZEND_ADD) { op = may_overflow ? IR_ADD_OV : IR_ADD; } else if (opcode == ZEND_SUB) { op = may_overflow ? IR_SUB_OV : IR_SUB; } else if (opcode == ZEND_MUL) { op = may_overflow ? IR_MUL_OV : IR_MUL; } else { ZEND_UNREACHABLE(); } op1 = jit_Z_LVAL(jit, op1_addr); op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr); ref = ir_BINARY_OP_L(op, op1, op2); if (may_overflow) { if (res_info & MAY_BE_GUARD) { if ((res_info & MAY_BE_ANY) == MAY_BE_LONG) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_res_info; int32_t exit_point; const void *exit_addr; if (opline->opcode == ZEND_ADD && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } else if (opline->opcode == ZEND_SUB && Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } else { exit_point = zend_jit_trace_get_exit_point(opline, 0); } exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD_NOT(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr)); may_overflow = 0; } else if ((res_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(ir_OVERFLOW(ref), ir_CONST_ADDR(exit_addr)); } else { ZEND_UNREACHABLE(); } } else { if_overflow = ir_IF(ir_OVERFLOW(ref)); ir_IF_FALSE(if_overflow); } } if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) { jit_set_Z_LVAL(jit, res_addr, ref); if (Z_MODE(res_addr) != IS_REG) { if (!zend_jit_same_addr(op1_addr, res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } } } } if (may_overflow) { ir_ref fast_path = IR_UNUSED; if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) { fast_path = ir_END(); ir_IF_TRUE_cold(if_overflow); } if (opcode == ZEND_ADD) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) { if (Z_MODE(res_addr) == IS_REG) { jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MAX + 1.0)); } else { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0)); jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000)); #else jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) { ir_MERGE_WITH(fast_path); } return 1; } op = IR_ADD; } else if (opcode == ZEND_SUB) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_LVAL_P(Z_ZV(op2_addr)) == 1) { if (Z_MODE(res_addr) == IS_REG) { jit_set_Z_DVAL(jit, res_addr, ir_CONST_DOUBLE((double)ZEND_LONG_MIN - 1.0)); } else { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000)); jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000)); #else jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) { ir_MERGE_WITH(fast_path); } return 1; } op = IR_SUB; } else if (opcode == ZEND_MUL) { op = IR_MUL; } else { ZEND_UNREACHABLE(); } #if 1 /* reload */ op1 = jit_Z_LVAL(jit, op1_addr); op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr); #endif #if 1 /* disable CSE */ ir_ref old_cse_limit = jit->ctx.fold_cse_limit; jit->ctx.fold_cse_limit = 0x7fffffff; #endif op1 = ir_INT2D(op1); op2 = ir_INT2D(op2); #if 1 jit->ctx.fold_cse_limit = old_cse_limit; #endif ref = ir_BINARY_OP_D(op, op1, op2); jit_set_Z_DVAL(jit, res_addr, ref); if (Z_MODE(res_addr) != IS_REG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } if ((res_info & MAY_BE_ANY) != MAY_BE_DOUBLE) { ir_MERGE_WITH(fast_path); } } return 1; } static int zend_jit_math_long_double(zend_jit_ctx *jit, uint8_t opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_use_info) { ir_op op; ir_ref op1, op2, ref; if (opcode == ZEND_ADD) { op = IR_ADD; } else if (opcode == ZEND_SUB) { op = IR_SUB; } else if (opcode == ZEND_MUL) { op = IR_MUL; } else if (opcode == ZEND_DIV) { op = IR_DIV; } else { ZEND_UNREACHABLE(); } op1 = jit_Z_LVAL(jit, op1_addr); op2 = jit_Z_DVAL(jit, op2_addr); ref = ir_BINARY_OP_D(op, ir_INT2D(op1), op2); jit_set_Z_DVAL(jit, res_addr, ref); if (Z_MODE(res_addr) != IS_REG) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } } return 1; } static int zend_jit_math_double_long(zend_jit_ctx *jit, uint8_t opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_use_info) { ir_op op; ir_ref op1, op2, ref; if (opcode == ZEND_ADD) { op = IR_ADD; } else if (opcode == ZEND_SUB) { op = IR_SUB; } else if (opcode == ZEND_MUL) { op = IR_MUL; } else if (opcode == ZEND_DIV) { op = IR_DIV; } else { ZEND_UNREACHABLE(); } op1 = jit_Z_DVAL(jit, op1_addr); op2 = jit_Z_LVAL(jit, op2_addr); ref = ir_BINARY_OP_D(op, op1, ir_INT2D(op2)); jit_set_Z_DVAL(jit, res_addr, ref); if (Z_MODE(res_addr) != IS_REG) { if (!zend_jit_same_addr(op1_addr, res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } } } return 1; } static int zend_jit_math_double_double(zend_jit_ctx *jit, uint8_t opcode, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint32_t res_use_info) { bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); ir_op op; ir_ref op1, op2, ref; if (opcode == ZEND_ADD) { op = IR_ADD; } else if (opcode == ZEND_SUB) { op = IR_SUB; } else if (opcode == ZEND_MUL) { op = IR_MUL; } else if (opcode == ZEND_DIV) { op = IR_DIV; } else { ZEND_UNREACHABLE(); } op1 = jit_Z_DVAL(jit, op1_addr); op2 = (same_ops) ? op1 : jit_Z_DVAL(jit, op2_addr); ref = ir_BINARY_OP_D(op, op1, op2); jit_set_Z_DVAL(jit, res_addr, ref); if (Z_MODE(res_addr) != IS_REG) { if (!zend_jit_same_addr(op1_addr, res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_DOUBLE) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } } } return 1; } static int zend_jit_math_helper(zend_jit_ctx *jit, const zend_op *opline, uint8_t opcode, uint8_t op1_type, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, uint8_t op2_type, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, int may_overflow, int may_throw) { ir_ref if_op1_long = IR_UNUSED; ir_ref if_op1_double = IR_UNUSED; ir_ref if_op2_double = IR_UNUSED; ir_ref if_op1_long_op2_long = IR_UNUSED; ir_ref if_op1_long_op2_double = IR_UNUSED; ir_ref if_op1_double_op2_double = IR_UNUSED; ir_ref if_op1_double_op2_long = IR_UNUSED; ir_ref slow_inputs = IR_UNUSED; bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); ir_refs *end_inputs; ir_refs *res_inputs; ir_refs_init(end_inputs, 6); ir_refs_init(res_inputs, 6); if (Z_MODE(op1_addr) == IS_REG) { if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, op1_addr); } } else if (Z_MODE(op2_addr) == IS_REG) { if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, op2_addr); } } if (Z_MODE(res_addr) == IS_REG) { jit->delay_var = Z_SSA_VAR(res_addr); jit->delay_refs = res_inputs; } if ((res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG) && (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_TRUE(if_op1_long); } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); ir_IF_TRUE(if_op1_long_op2_long); } if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { return 0; } ir_refs_add(end_inputs, ir_END()); if (if_op1_long) { ir_IF_FALSE_cold(if_op1_long); ir_END_list(slow_inputs); } if (if_op1_long_op2_long) { ir_IF_FALSE_cold(if_op1_long_op2_long); ir_END_list(slow_inputs); } } else if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG) && (res_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & (MAY_BE_ANY-MAY_BE_LONG)) { if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_TRUE(if_op1_long); } if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_LONG))) { if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); ir_IF_FALSE_cold(if_op1_long_op2_long); if (op2_info & MAY_BE_DOUBLE) { if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); ir_IF_FALSE_cold(if_op1_long_op2_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_long_op2_double); } if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } ir_refs_add(end_inputs, ir_END()); } else { ir_END_list(slow_inputs); } ir_IF_TRUE(if_op1_long_op2_long); } if (!zend_jit_math_long_long(jit, opline, opcode, op1_addr, op2_addr, res_addr, res_info, res_use_info, may_overflow)) { return 0; } ir_refs_add(end_inputs, ir_END()); if (if_op1_long) { ir_IF_FALSE_cold(if_op1_long); } if (op1_info & MAY_BE_DOUBLE) { if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); ir_IF_FALSE_cold(if_op1_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_double); } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); ir_IF_TRUE(if_op1_double_op2_double); } if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } ir_refs_add(end_inputs, ir_END()); if (if_op1_double_op2_double) { ir_IF_FALSE_cold(if_op1_double_op2_double); } } if (!same_ops) { if (op2_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_DOUBLE))) { if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); ir_IF_FALSE_cold(if_op1_double_op2_long); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_double_op2_long); } if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } ir_refs_add(end_inputs, ir_END()); } else if (if_op1_double_op2_double) { ir_END_list(slow_inputs); } } else if (if_op1_long) { ir_END_list(slow_inputs); } } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (res_info & MAY_BE_DOUBLE)) { if (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); ir_IF_FALSE_cold(if_op1_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_double); } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); ir_IF_TRUE(if_op1_double_op2_double); } if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } ir_refs_add(end_inputs, ir_END()); if (if_op1_double_op2_double) { ir_IF_FALSE_cold(if_op1_double_op2_double); } } if (!same_ops && (op2_info & MAY_BE_LONG)) { if (op2_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); ir_IF_FALSE_cold(if_op1_double_op2_long); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_double_op2_long); } if (!zend_jit_math_double_long(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } ir_refs_add(end_inputs, ir_END()); } else if (if_op1_double_op2_double) { ir_END_list(slow_inputs); } } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (res_info & MAY_BE_DOUBLE)) { if (op2_info & (MAY_BE_ANY-MAY_BE_DOUBLE)) { if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); ir_IF_FALSE_cold(if_op2_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op2_double); } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & (MAY_BE_ANY-MAY_BE_DOUBLE))) { if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); ir_IF_TRUE(if_op1_double_op2_double); } if (!zend_jit_math_double_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } ir_refs_add(end_inputs, ir_END()); if (if_op1_double_op2_double) { ir_IF_FALSE_cold(if_op1_double_op2_double); } } if (!same_ops && (op1_info & MAY_BE_LONG)) { if (op1_info & (MAY_BE_ANY-(MAY_BE_DOUBLE|MAY_BE_LONG))) { if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_FALSE_cold(if_op1_long_op2_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_long_op2_double); } if (!zend_jit_math_long_double(jit, opcode, op1_addr, op2_addr, res_addr, res_use_info)) { return 0; } ir_refs_add(end_inputs, ir_END()); } else if (if_op1_double_op2_double) { ir_END_list(slow_inputs); } } if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { ir_ref func, arg1, arg2, arg3; if (slow_inputs) { ir_MERGE_list(slow_inputs); } if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { return 0; } op1_addr = real_addr; } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { return 0; } op2_addr = real_addr; } if (Z_MODE(res_addr) == IS_REG) { arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var)); } else { arg1 = jit_ZVAL_ADDR(jit, res_addr); } arg2 = jit_ZVAL_ADDR(jit, op1_addr); arg3 = jit_ZVAL_ADDR(jit, op2_addr); jit_SET_EX_OPLINE(jit, opline); if (opcode == ZEND_ADD) { func = ir_CONST_FC_FUNC(add_function); } else if (opcode == ZEND_SUB) { func = ir_CONST_FC_FUNC(sub_function); } else if (opcode == ZEND_MUL) { func = ir_CONST_FC_FUNC(mul_function); } else if (opcode == ZEND_DIV) { func = ir_CONST_FC_FUNC(div_function); } else { ZEND_UNREACHABLE(); } ir_CALL_3(IR_VOID, func, arg1, arg2, arg3); jit_FREE_OP(jit, op1_type, op1, op1_info, NULL); jit_FREE_OP(jit, op2_type, op2, op2_info, NULL); if (may_throw) { if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); } } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) { return 0; } } ir_refs_add(end_inputs, ir_END()); } if (end_inputs->count) { ir_MERGE_N(end_inputs->count, end_inputs->refs); } if (Z_MODE(res_addr) == IS_REG) { ZEND_ASSERT(jit->delay_refs == res_inputs); ZEND_ASSERT(end_inputs->count == res_inputs->count); jit->delay_var = -1; jit->delay_refs = NULL; if (res_inputs->count == 1) { zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]); } else { ir_ref phi = ir_PHI_N((res_info & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE, res_inputs->count, res_inputs->refs); zend_jit_def_reg(jit, res_addr, phi); } } return 1; } static int zend_jit_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_overflow, int may_throw) { ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); if (!zend_jit_math_helper(jit, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->result.var, res_addr, res_info, res_use_info, may_overflow, may_throw)) { return 0; } if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { return 0; } return 1; } static int zend_jit_add_arrays(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr res_addr) { ir_ref ref; ir_ref arg1 = jit_Z_PTR(jit, op1_addr); ir_ref arg2 = jit_Z_PTR(jit, op2_addr); ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_add_arrays_helper), arg1, arg2); jit_set_Z_PTR(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_ARRAY_EX); jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); return 1; } static int zend_jit_long_math_helper(zend_jit_ctx *jit, const zend_op *opline, uint8_t opcode, uint8_t op1_type, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, zend_ssa_range *op1_range, uint8_t op2_type, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, zend_ssa_range *op2_range, uint32_t res_var, zend_jit_addr res_addr, uint32_t res_info, uint32_t res_use_info, int may_throw) { ir_ref ref = IR_UNUSED; ir_ref if_long1 = IR_UNUSED; ir_ref if_long2 = IR_UNUSED; bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); ir_refs *res_inputs; ir_refs_init(res_inputs, 2); if (Z_MODE(op1_addr) == IS_REG && Z_LOAD(op1_addr) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, op1_addr); } if (Z_MODE(op2_addr) == IS_REG && Z_LOAD(op2_addr) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, op2_addr); } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if_long1 = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_TRUE(if_long1); } if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { if_long2 = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); ir_IF_TRUE(if_long2); } if (opcode == ZEND_SL) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { if (EXPECTED(op2_lval > 0)) { ref = ir_CONST_LONG(0); } else { zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); jit_SET_EX_OPLINE(jit, opline); ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift)); if (Z_MODE(res_addr) == IS_REG) { ref = ir_CONST_LONG(0); // dead code } } } else { ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval)); } } else { ref = jit_Z_LVAL(jit, op2_addr); if (!op2_range || op2_range->min < 0 || op2_range->max >= SIZEOF_ZEND_LONG * 8) { ir_ref if_wrong, cold_path, ref2, if_ok; ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr); if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1))); ir_IF_TRUE_cold(if_wrong); if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0))); ir_IF_FALSE(if_ok); jit_SET_EX_OPLINE(jit, opline); zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift)); ir_IF_TRUE(if_ok); ref2 = ir_CONST_LONG(0); cold_path = ir_END(); ir_IF_FALSE(if_wrong); ref = ir_SHL_L(op1_ref, ref); ir_MERGE_WITH(cold_path); ref = ir_PHI_2(IR_LONG, ref, ref2); } else { ref = ir_SHL_L(jit_Z_LVAL(jit, op1_addr), ref); } } } else if (opcode == ZEND_SR) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (UNEXPECTED((zend_ulong)op2_lval >= SIZEOF_ZEND_LONG * 8)) { if (EXPECTED(op2_lval > 0)) { ref = ir_SAR_L( jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1)); } else { zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); jit_SET_EX_OPLINE(jit, opline); ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_negative_shift)); if (Z_MODE(res_addr) == IS_REG) { ref = ir_CONST_LONG(0); // dead code } } } else { ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval)); } } else { ref = jit_Z_LVAL(jit, op2_addr); if (!op2_range || op2_range->min < 0 || op2_range->max >= SIZEOF_ZEND_LONG * 8) { ir_ref if_wrong, cold_path, ref2, if_ok; if_wrong = ir_IF(ir_UGT(ref, ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1))); ir_IF_TRUE_cold(if_wrong); if_ok = ir_IF(ir_GE(ref, ir_CONST_LONG(0))); ir_IF_FALSE(if_ok); jit_SET_EX_OPLINE(jit, opline); zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_negative_shift)); ir_IF_TRUE(if_ok); ref2 = ir_CONST_LONG((SIZEOF_ZEND_LONG * 8) - 1); cold_path = ir_END(); ir_IF_FALSE(if_wrong); ir_MERGE_WITH(cold_path); ref = ir_PHI_2(IR_LONG, ref, ref2); } ref = ir_SAR_L(jit_Z_LVAL(jit, op1_addr), ref); } } else if (opcode == ZEND_MOD) { if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { zend_long op2_lval = Z_LVAL_P(Z_ZV(op2_addr)); if (op2_lval == 0) { zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); jit_SET_EX_OPLINE(jit, opline); ir_GUARD(IR_FALSE, jit_STUB_ADDR(jit, jit_stub_mod_by_zero)); if (Z_MODE(res_addr) == IS_REG) { ref = ir_CONST_LONG(0); // dead code } } else if (zend_long_is_power_of_two(op2_lval) && op1_range && op1_range->min >= 0) { ref = ir_AND_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval - 1)); } else { ref = ir_MOD_L(jit_Z_LVAL(jit, op1_addr), ir_CONST_LONG(op2_lval)); } } else { ir_ref zero_path = 0; ir_ref op1_ref = jit_Z_LVAL(jit, op1_addr); ref = jit_Z_LVAL(jit, op2_addr); if ((op2_type & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) || !op2_range || (op2_range->min <= 0 && op2_range->max >= 0)) { ir_ref if_ok = ir_IF(ref); ir_IF_FALSE(if_ok); jit_SET_EX_OPLINE(jit, opline); zend_jit_invalidate_var_if_necessary(jit, op1_type, op1_addr, op1); zend_jit_invalidate_var_if_necessary(jit, op2_type, op2_addr, op2); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_mod_by_zero)); ir_IF_TRUE(if_ok); } /* Prevent overflow error/crash if op1 == LONG_MIN and op2 == -1 */ if (!op2_range || (op2_range->min <= -1 && op2_range->max >= -1)) { ir_ref if_minus_one = ir_IF(ir_EQ(ref, ir_CONST_LONG(-1))); ir_IF_TRUE_cold(if_minus_one); zero_path = ir_END(); ir_IF_FALSE(if_minus_one); } ref = ir_MOD_L(op1_ref, ref); if (zero_path) { ir_MERGE_WITH(zero_path); ref = ir_PHI_2(IR_LONG, ref, ir_CONST_LONG(0)); } } } else { ir_op op; ir_ref op1, op2; if (opcode == ZEND_BW_OR) { op = IR_OR; } else if (opcode == ZEND_BW_AND) { op = IR_AND; } else if (opcode == ZEND_BW_XOR) { op = IR_XOR; } else { ZEND_UNREACHABLE(); } op1 = jit_Z_LVAL(jit, op1_addr); op2 = (same_ops) ? op1 : jit_Z_LVAL(jit, op2_addr); ref = ir_BINARY_OP_L(op, op1, op2); } if (ref) { if (Z_MODE(res_addr) == IS_REG && ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { jit->delay_var = Z_SSA_VAR(res_addr); jit->delay_refs = res_inputs; } jit_set_Z_LVAL(jit, res_addr, ref); if (Z_MODE(res_addr) != IS_REG) { if (!zend_jit_same_addr(op1_addr, res_addr)) { if ((res_use_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF|MAY_BE_GUARD)) != MAY_BE_LONG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } } } } if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { ir_ref fast_path = ir_END(); ir_ref func, arg1, arg2, arg3; if (if_long2 && if_long1) { ir_ref ref; ir_IF_FALSE_cold(if_long2); ref = ir_END(); ir_IF_FALSE_cold(if_long1); ir_MERGE_2(ref, ir_END()); } else if (if_long1) { ir_IF_FALSE_cold(if_long1); } else if (if_long2) { ir_IF_FALSE_cold(if_long2); } if (op1_info & MAY_BE_UNDEF) { ir_ref if_def, ref, ref2; ref = jit_ZVAL_ADDR(jit, op1_addr); if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); ir_IF_FALSE_cold(if_def); // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))); jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var)); ref2 = jit_EG(uninitialized_zval); ir_MERGE_WITH_EMPTY_TRUE(if_def); ref = ir_PHI_2(IR_ADDR, ref2, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } if (op2_info & MAY_BE_UNDEF) { ir_ref if_def, ref, ref2; ref = jit_ZVAL_ADDR(jit, op2_addr); if_def = jit_if_not_Z_TYPE(jit, op2_addr, IS_UNDEF); ir_IF_FALSE_cold(if_def); // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op2.var))); jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var)); ref2 = jit_EG(uninitialized_zval); ir_MERGE_WITH_EMPTY_TRUE(if_def); ref = ir_PHI_2(IR_ADDR, ref2, ref); op2_addr = ZEND_ADDR_REF_ZVAL(ref); } if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op1.var); if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { return 0; } op1_addr = real_addr; } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, op2.var); if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { return 0; } op2_addr = real_addr; } if (Z_MODE(res_addr) == IS_REG) { arg1 = jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var)); } else { arg1 = jit_ZVAL_ADDR(jit, res_addr); } arg2 = jit_ZVAL_ADDR(jit, op1_addr); arg3 = jit_ZVAL_ADDR(jit, op2_addr); jit_SET_EX_OPLINE(jit, opline); if (opcode == ZEND_BW_OR) { func = ir_CONST_FC_FUNC(bitwise_or_function); } else if (opcode == ZEND_BW_AND) { func = ir_CONST_FC_FUNC(bitwise_and_function); } else if (opcode == ZEND_BW_XOR) { func = ir_CONST_FC_FUNC(bitwise_xor_function); } else if (opcode == ZEND_SL) { func = ir_CONST_FC_FUNC(shift_left_function); } else if (opcode == ZEND_SR) { func = ir_CONST_FC_FUNC(shift_right_function); } else if (opcode == ZEND_MOD) { func = ir_CONST_FC_FUNC(mod_function); } else { ZEND_UNREACHABLE(); } ir_CALL_3(IR_VOID, func, arg1, arg2, arg3); if (op1_addr == res_addr && (op2_info & MAY_BE_RCN)) { /* compound assignment may decrement "op2" refcount */ op2_info |= MAY_BE_RC1; } jit_FREE_OP(jit, op1_type, op1, op1_info, NULL); jit_FREE_OP(jit, op2_type, op2, op2_info, NULL); if (may_throw) { if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); } } if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, res_var); if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) { return 0; } } ir_MERGE_2(fast_path, ir_END()); if (Z_MODE(res_addr) == IS_REG) { ZEND_ASSERT(jit->delay_refs == res_inputs); ZEND_ASSERT(res_inputs->count == 2); jit->delay_var = -1; jit->delay_refs = NULL; if (res_inputs->count == 1) { zend_jit_def_reg(jit, res_addr, res_inputs->refs[0]); } else { ir_ref phi = ir_PHI_N(IR_LONG, res_inputs->count, res_inputs->refs); zend_jit_def_reg(jit, res_addr, phi); } } } return 1; } static int zend_jit_long_math(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr, int may_throw) { ZEND_ASSERT((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)); if (!zend_jit_long_math_helper(jit, opline, opline->opcode, opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, opline->result.var, res_addr, res_info, res_use_info, may_throw)) { return 0; } if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { return 0; } return 1; } static int zend_jit_concat_helper(zend_jit_ctx *jit, const zend_op *opline, uint8_t op1_type, znode_op op1, zend_jit_addr op1_addr, uint32_t op1_info, uint8_t op2_type, znode_op op2, zend_jit_addr op2_addr, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) { ir_ref if_op1_string = IR_UNUSED; ir_ref if_op2_string = IR_UNUSED; ir_ref fast_path = IR_UNUSED; if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { if_op1_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); ir_IF_TRUE(if_op1_string); } if (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) { if_op2_string = jit_if_Z_TYPE(jit, op2_addr, IS_STRING); ir_IF_TRUE(if_op2_string); } if (zend_jit_same_addr(op1_addr, res_addr)) { ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr); ir_ref arg2 = jit_ZVAL_ADDR(jit, op2_addr); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_assign_concat_helper), arg1, arg2); /* concatenation with itself may reduce refcount */ op2_info |= MAY_BE_RC1; } else { ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr); ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr); ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr); if (op1_type == IS_CV || op1_type == IS_CONST) { ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_helper), arg1, arg2, arg3); } else { ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fast_concat_tmp_helper), arg1, arg2, arg3); } } /* concatenation with empty string may increase refcount */ op2_info |= MAY_BE_RCN; jit_FREE_OP(jit, op2_type, op2, op2_info, opline); if (if_op1_string || if_op2_string) { fast_path = ir_END(); } } if ((op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING)) || (op2_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF) - MAY_BE_STRING))) { if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { if (if_op1_string && if_op2_string) { ir_IF_FALSE(if_op1_string); ir_MERGE_WITH_EMPTY_FALSE(if_op2_string); } else if (if_op1_string) { ir_IF_FALSE_cold(if_op1_string); } else if (if_op2_string) { ir_IF_FALSE_cold(if_op2_string); } } ir_ref arg1 = jit_ZVAL_ADDR(jit, res_addr); ir_ref arg2 = jit_ZVAL_ADDR(jit, op1_addr); ir_ref arg3 = jit_ZVAL_ADDR(jit, op2_addr); jit_SET_EX_OPLINE(jit, opline); ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(concat_function), arg1, arg2, arg3); /* concatenation with empty string may increase refcount */ op1_info |= MAY_BE_RCN; op2_info |= MAY_BE_RCN; jit_FREE_OP(jit, op1_type, op1, op1_info, NULL); jit_FREE_OP(jit, op2_type, op2, op2_info, NULL); if (may_throw) { if (opline->opcode == ZEND_ASSIGN_DIM_OP && (opline->op2_type & (IS_VAR|IS_TMP_VAR))) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_exception_handler_free_op2)); } else if (Z_MODE(res_addr) == IS_MEM_ZVAL && Z_REG(res_addr) == ZREG_RX) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); } } if ((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)) { ir_MERGE_WITH(fast_path); } } return 1; } static int zend_jit_concat(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, zend_jit_addr res_addr, int may_throw) { zend_jit_addr op1_addr, op2_addr; ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); ZEND_ASSERT((op1_info & MAY_BE_STRING) && (op2_info & MAY_BE_STRING)); op1_addr = OP1_ADDR(); op2_addr = OP2_ADDR(); return zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, res_addr, may_throw); } static int zend_jit_assign_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_ssa_range *op1_range, uint32_t op1_def_info, zend_jit_addr op1_def_addr, uint32_t op1_mem_info, uint32_t op2_info, zend_jit_addr op2_addr, zend_ssa_range *op2_range, int may_overflow, int may_throw) { int result = 1; ir_ref slow_path = IR_UNUSED; ZEND_ASSERT(opline->op1_type == IS_CV && opline->result_type == IS_UNUSED); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF)); if (op1_info & MAY_BE_REF) { ir_ref ref, ref2, arg2, op1_noref_path; ir_ref if_op1_ref = IR_UNUSED; ir_ref if_op1_typed = IR_UNUSED; binary_op_type binary_op = get_binary_op(opline->extended_value); ref = jit_ZVAL_ADDR(jit, op1_addr); if_op1_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE)); ir_IF_FALSE(if_op1_ref); op1_noref_path = ir_END(); ir_IF_TRUE(if_op1_ref); ref2 = jit_Z_PTR_ref(jit, ref); if_op1_typed = jit_if_TYPED_REF(jit, ref2); ir_IF_TRUE_cold(if_op1_typed); if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { return 0; } arg2 = jit_ZVAL_ADDR(jit, real_addr); } else { arg2 = jit_ZVAL_ADDR(jit, op2_addr); } jit_SET_EX_OPLINE(jit, opline); if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref_tmp), ref2, arg2, ir_CONST_FC_FUNC(binary_op)); } else { ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref), ref2, arg2, ir_CONST_FC_FUNC(binary_op)); } zend_jit_check_exception(jit); slow_path = ir_END(); ir_IF_FALSE(if_op1_typed); ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val)); ir_MERGE_WITH(op1_noref_path); ref = ir_PHI_2(IR_ADDR, ref2, ref); ZEND_ASSERT(op1_addr == op1_def_addr); op1_def_addr = op1_addr = ZEND_ADDR_REF_ZVAL(ref); } switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: case ZEND_DIV: result = zend_jit_math_helper(jit, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_overflow, may_throw); break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: result = zend_jit_long_math_helper(jit, opline, opline->extended_value, opline->op1_type, opline->op1, op1_addr, op1_info, op1_range, opline->op2_type, opline->op2, op2_addr, op2_info, op2_range, opline->op1.var, op1_def_addr, op1_def_info, op1_mem_info, may_throw); break; case ZEND_CONCAT: result = zend_jit_concat_helper(jit, opline, opline->op1_type, opline->op1, op1_addr, op1_info, opline->op2_type, opline->op2, op2_addr, op2_info, op1_def_addr, may_throw); break; default: ZEND_UNREACHABLE(); } if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_def_addr, op1_def_info, op1_addr, op1_info)) { return 0; } if (op1_info & MAY_BE_REF) { ir_MERGE_WITH(slow_path); } return result; } static ir_ref jit_ZVAL_DEREF_ref(zend_jit_ctx *jit, ir_ref ref) { ir_ref if_ref, ref2; if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_REFERENCE))); ir_IF_TRUE(if_ref); ref2 = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val)); ir_MERGE_WITH_EMPTY_FALSE(if_ref); return ir_PHI_2(IR_ADDR, ref2, ref); } static zend_jit_addr jit_ZVAL_DEREF(zend_jit_ctx *jit, zend_jit_addr addr) { ir_ref ref = jit_ZVAL_ADDR(jit, addr); ref = jit_ZVAL_DEREF_ref(jit, ref); return ZEND_ADDR_REF_ZVAL(ref); } static ir_ref jit_ZVAL_INDIRECT_DEREF_ref(zend_jit_ctx *jit, ir_ref ref) { ir_ref if_ref, ref2; if_ref = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_INDIRECT))); ir_IF_TRUE(if_ref); ref2 = jit_Z_PTR_ref(jit, ref); ir_MERGE_WITH_EMPTY_FALSE(if_ref); return ir_PHI_2(IR_ADDR, ref2, ref); } static zend_jit_addr jit_ZVAL_INDIRECT_DEREF(zend_jit_ctx *jit, zend_jit_addr addr) { ir_ref ref = jit_ZVAL_ADDR(jit, addr); ref = jit_ZVAL_INDIRECT_DEREF_ref(jit, ref); return ZEND_ADDR_REF_ZVAL(ref); } static int zend_jit_simple_assign(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr, uint32_t var_info, uint32_t var_def_info, uint8_t val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr res_addr, bool check_exception) { ir_ref end_inputs = IR_UNUSED; if (Z_MODE(val_addr) == IS_CONST_ZVAL) { zval *zv = Z_ZV(val_addr); if (!res_addr) { jit_ZVAL_COPY_CONST(jit, var_addr, var_info, var_def_info, zv, 1); } else { jit_ZVAL_COPY_CONST(jit, var_addr, var_info, var_def_info, zv, 1); jit_ZVAL_COPY_CONST(jit, res_addr, -1, var_def_info, zv, 1); } } else { if (val_info & MAY_BE_UNDEF) { ir_ref if_def, ret; if_def = jit_if_not_Z_TYPE(jit, val_addr, IS_UNDEF); ir_IF_FALSE_cold(if_def); jit_set_Z_TYPE_INFO(jit, var_addr, IS_NULL); if (res_addr) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); } jit_SET_EX_OPLINE(jit, opline); ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL); // zend_error_unchecked(E_WARNING, "Undefined variable $%S", CV_DEF_OF(EX_VAR_TO_NUM(opline->op1.var))); ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr))); if (check_exception) { ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); } ir_END_list(end_inputs); ir_IF_TRUE(if_def); } if (val_info & MAY_BE_REF) { if (val_type == IS_CV) { ir_ref ref = jit_ZVAL_ADDR(jit, val_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); val_addr = ZEND_ADDR_REF_ZVAL(ref); } else { ir_ref ref, type, if_ref, ref2, refcount, if_not_zero; ref = jit_ZVAL_ADDR(jit, val_addr); type = jit_Z_TYPE_ref(jit, ref); if_ref = ir_IF(ir_EQ(type, ir_CONST_U8(IS_REFERENCE))); ir_IF_TRUE_cold(if_ref); ref = jit_Z_PTR_ref(jit, ref); ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val)); if (!res_addr) { jit_ZVAL_COPY(jit, var_addr, var_info, ZEND_ADDR_REF_ZVAL(ref2), val_info, 1); } else { jit_ZVAL_COPY_2(jit, res_addr, var_addr, var_info, ZEND_ADDR_REF_ZVAL(ref2), val_info, 2); } refcount = jit_GC_DELREF(jit, ref); if_not_zero = ir_IF(refcount); ir_IF_FALSE(if_not_zero); // TODO: instead of dtor() call and ADDREF above, we may call efree() and move addref at "true" path ??? // This is related to GH-10168 (keep this before GH-10168 is completely closed) // jit_EFREE(jit, ref, sizeof(zend_reference), NULL, NULL); jit_ZVAL_DTOR(jit, ref, val_info, opline); ir_END_list(end_inputs); ir_IF_TRUE(if_not_zero); ir_END_list(end_inputs); ir_IF_FALSE(if_ref); } } if (!res_addr) { jit_ZVAL_COPY(jit, var_addr, var_info, val_addr, val_info, val_type == IS_CV); } else { jit_ZVAL_COPY_2(jit, res_addr, var_addr, var_info, val_addr, val_info, val_type == IS_CV ? 2 : 1); } } if (end_inputs) { ir_END_list(end_inputs); ir_MERGE_list(end_inputs); } return 1; } static int zend_jit_assign_to_variable_call(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr __var_use_addr, zend_jit_addr var_addr, uint32_t __var_info, uint32_t __var_def_info, uint8_t val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr __res_addr, bool __check_exception) { jit_stub_id func; ir_ref undef_path = IR_UNUSED; if (val_info & MAY_BE_UNDEF) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_not_Z_TYPE(jit, val_addr, IS_UNDEF, exit_addr); } else { ir_ref if_def; ZEND_ASSERT(Z_MODE(val_addr) == IS_MEM_ZVAL && Z_REG(val_addr) == ZREG_FP); if_def = ir_IF(jit_Z_TYPE(jit, val_addr)); ir_IF_FALSE_cold(if_def); jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(Z_OFFSET(val_addr))); ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_assign_const, IR_FASTCALL_FUNC), jit_ZVAL_ADDR(jit, var_addr), jit_EG(uninitialized_zval)); undef_path = ir_END(); ir_IF_TRUE(if_def); } } if (!(val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF))) { func = jit_stub_assign_tmp; } else if (val_type == IS_CONST) { func = jit_stub_assign_const; } else if (val_type == IS_TMP_VAR) { func = jit_stub_assign_tmp; } else if (val_type == IS_VAR) { if (!(val_info & MAY_BE_REF)) { func = jit_stub_assign_tmp; } else { func = jit_stub_assign_var; } } else if (val_type == IS_CV) { if (!(val_info & MAY_BE_REF)) { func = jit_stub_assign_cv_noref; } else { func = jit_stub_assign_cv; } } else { ZEND_UNREACHABLE(); } if (opline) { jit_SET_EX_OPLINE(jit, opline); } ir_CALL_2(IR_VOID, jit_STUB_FUNC_ADDR(jit, func, IR_FASTCALL_FUNC), jit_ZVAL_ADDR(jit, var_addr), jit_ZVAL_ADDR(jit, val_addr)); if (undef_path) { ir_MERGE_WITH(undef_path); } return 1; } static int zend_jit_assign_to_variable(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_use_addr, zend_jit_addr var_addr, uint32_t var_info, uint32_t var_def_info, uint8_t val_type, zend_jit_addr val_addr, uint32_t val_info, zend_jit_addr res_addr, zend_jit_addr ref_addr, bool check_exception) { ir_ref if_refcounted = IR_UNUSED; ir_ref simple_inputs = IR_UNUSED; bool done = 0; zend_jit_addr real_res_addr = 0; ir_refs *end_inputs; ir_refs *res_inputs; ir_refs_init(end_inputs, 6); ir_refs_init(res_inputs, 6); if (Z_MODE(val_addr) == IS_REG && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, val_addr); } if (Z_MODE(var_addr) == IS_REG) { jit->delay_var = Z_SSA_VAR(var_addr); jit->delay_refs = res_inputs; if (Z_MODE(res_addr) == IS_REG) { real_res_addr = res_addr; res_addr = 0; } } else if (Z_MODE(res_addr) == IS_REG) { jit->delay_var = Z_SSA_VAR(res_addr); jit->delay_refs = res_inputs; } if ((var_info & MAY_BE_REF) || ref_addr) { ir_ref ref = 0, if_ref = 0, ref2, arg2, if_typed, non_ref_path; uintptr_t func; if (!ref_addr) { ref = jit_ZVAL_ADDR(jit, var_use_addr); if_ref = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(IS_REFERENCE)); ir_IF_TRUE(if_ref); ref2 = jit_Z_PTR_ref(jit, ref); } else { ref2 = jit_ZVAL_ADDR(jit, ref_addr); } if_typed = jit_if_TYPED_REF(jit, ref2); ir_IF_TRUE_cold(if_typed); jit_SET_EX_OPLINE(jit, opline); if (Z_MODE(val_addr) == IS_REG) { zend_jit_addr real_addr; if (opline->opcode == ZEND_ASSIGN_DIM || opline->opcode == ZEND_ASSIGN_OBJ) { real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var); } else { ZEND_ASSERT(opline->opcode == ZEND_ASSIGN); real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); } if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) { return 0; } arg2 = jit_ZVAL_ADDR(jit, real_addr); } else { arg2 = jit_ZVAL_ADDR(jit, val_addr); } if (!res_addr) { if (val_type == IS_CONST) { func = (uintptr_t)zend_jit_assign_const_to_typed_ref; } else if (val_type == IS_TMP_VAR) { func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref; } else if (val_type == IS_VAR) { func = (uintptr_t)zend_jit_assign_var_to_typed_ref; } else if (val_type == IS_CV) { func = (uintptr_t)zend_jit_assign_cv_to_typed_ref; } else { ZEND_UNREACHABLE(); } ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2); } else { if (val_type == IS_CONST) { func = (uintptr_t)zend_jit_assign_const_to_typed_ref2; } else if (val_type == IS_TMP_VAR) { func = (uintptr_t)zend_jit_assign_tmp_to_typed_ref2; } else if (val_type == IS_VAR) { func = (uintptr_t)zend_jit_assign_var_to_typed_ref2; } else if (val_type == IS_CV) { func = (uintptr_t)zend_jit_assign_cv_to_typed_ref2; } else { ZEND_UNREACHABLE(); } ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(func), ref2, arg2, jit_ZVAL_ADDR(jit, res_addr)); } if (check_exception) { zend_jit_check_exception(jit); } ir_refs_add(end_inputs, ir_END()); if (!ref_addr) { ir_IF_FALSE(if_ref); non_ref_path = ir_END(); ir_IF_FALSE(if_typed); ref2 = ir_ADD_OFFSET(ref2, offsetof(zend_reference, val)); ir_MERGE_WITH(non_ref_path); ref = ir_PHI_2(IR_ADDR, ref2, ref); var_addr = var_use_addr = ZEND_ADDR_REF_ZVAL(ref); } else { ir_IF_FALSE(if_typed); } } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { ir_ref ref, counter, if_not_zero; if (var_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if_refcounted = jit_if_REFCOUNTED(jit, var_use_addr); ir_IF_FALSE(if_refcounted); ir_END_list(simple_inputs); ir_IF_TRUE_cold(if_refcounted); } else if (RC_MAY_BE_1(var_info)) { done = 1; } ref = jit_Z_PTR(jit, var_use_addr); if (RC_MAY_BE_1(var_info)) { if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, 0)) { return 0; } counter = jit_GC_DELREF(jit, ref); if_not_zero = ir_IF(counter); ir_IF_FALSE(if_not_zero); jit_ZVAL_DTOR(jit, ref, var_info, opline); if (check_exception) { zend_jit_check_exception(jit); } ir_refs_add(end_inputs, ir_END()); ir_IF_TRUE(if_not_zero); if (RC_MAY_BE_N(var_info) && (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) != 0) { ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); ir_IF_FALSE(if_may_leak); if (opline) { jit_SET_EX_OPLINE(jit, opline); } ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref); if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) { ZEND_ASSERT(jit->delay_refs == res_inputs); ZEND_ASSERT(res_inputs->count > 0); ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]); } if (check_exception && (val_info & MAY_BE_UNDEF)) { zend_jit_check_exception(jit); } ir_refs_add(end_inputs, ir_END()); ir_IF_TRUE(if_may_leak); } if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) { ZEND_ASSERT(jit->delay_refs == res_inputs); ZEND_ASSERT(res_inputs->count > 0); ir_refs_add(res_inputs, res_inputs->refs[res_inputs->count - 1]); } if (check_exception && (val_info & MAY_BE_UNDEF)) { zend_jit_check_exception(jit); } ir_refs_add(end_inputs, ir_END()); } else /* if (RC_MAY_BE_N(var_info)) */ { jit_GC_DELREF(jit, ref); if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT)) { ir_ref if_may_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref); ir_IF_FALSE(if_may_leak); if (opline) { jit_SET_EX_OPLINE(jit, opline); } ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref); ir_END_list(simple_inputs); ir_IF_TRUE(if_may_leak); } ir_END_list(simple_inputs); } } if (simple_inputs) { ir_MERGE_list(simple_inputs); } if (!done) { if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, var_def_info, val_type, val_addr, val_info, res_addr, check_exception)) { return 0; } if (end_inputs->count) { ir_refs_add(end_inputs, ir_END()); } } if (end_inputs->count) { ir_MERGE_N(end_inputs->count, end_inputs->refs); } if (Z_MODE(var_addr) == IS_REG || Z_MODE(res_addr) == IS_REG) { ir_ref phi; ZEND_ASSERT(jit->delay_refs == res_inputs); ZEND_ASSERT(end_inputs->count == res_inputs->count || (end_inputs->count == 0 && res_inputs->count == 1)); jit->delay_var = -1; jit->delay_refs = NULL; if (res_inputs->count == 1) { phi = res_inputs->refs[0]; } else { phi = ir_PHI_N((var_def_info & MAY_BE_LONG & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE, res_inputs->count, res_inputs->refs); } if (Z_MODE(var_addr) == IS_REG) { if ((var_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || ref_addr) { phi = ir_emit2(&jit->ctx, IR_OPT(IR_COPY, jit->ctx.ir_base[phi].type), phi, 1); } zend_jit_def_reg(jit, var_addr, phi); if (real_res_addr) { if (var_def_info & MAY_BE_LONG) { jit_set_Z_LVAL(jit, real_res_addr, jit_Z_LVAL(jit, var_addr)); } else { jit_set_Z_DVAL(jit, real_res_addr, jit_Z_DVAL(jit, var_addr)); } } } else { zend_jit_def_reg(jit, res_addr, phi); } } return 1; } static int zend_jit_qm_assign(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr, uint32_t res_use_info, uint32_t res_info, zend_jit_addr res_addr) { if (op1_addr != op1_def_addr) { if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { return 0; } if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { op1_addr = op1_def_addr; } } if (!zend_jit_simple_assign(jit, opline, res_addr, res_use_info, res_info, opline->op1_type, op1_addr, op1_info, 0, 1)) { return 0; } if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { return 0; } return 1; } static int zend_jit_assign(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_use_addr, uint32_t op1_def_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_jit_addr op2_def_addr, uint32_t res_info, zend_jit_addr res_addr, zend_jit_addr ref_addr, int may_throw) { ZEND_ASSERT(opline->op1_type == IS_CV); if (op2_addr != op2_def_addr) { if (!zend_jit_update_regs(jit, opline->op2.var, op2_addr, op2_def_addr, op2_info)) { return 0; } if (Z_MODE(op2_def_addr) == IS_REG && Z_MODE(op2_addr) != IS_REG) { op2_addr = op2_def_addr; } } if (Z_MODE(op1_addr) != IS_REG && Z_MODE(op1_use_addr) == IS_REG && !Z_LOAD(op1_use_addr) && !Z_STORE(op1_use_addr)) { /* Force type update */ op1_info |= MAY_BE_UNDEF; } if (!zend_jit_assign_to_variable(jit, opline, op1_use_addr, op1_addr, op1_info, op1_def_info, opline->op2_type, op2_addr, op2_info, res_addr, ref_addr, may_throw)) { return 0; } if (Z_MODE(op1_addr) == IS_REG) { if (Z_STORE(op1_addr)) { if (!zend_jit_store_var_if_necessary_ex(jit, opline->op1.var, op1_addr, op1_def_info, op1_use_addr, op1_info)) { return 0; } } else if ((op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) && Z_MODE(op1_use_addr) == IS_MEM_ZVAL && Z_REG(op1_use_addr) == ZREG_FP && EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)) < jit->current_op_array->last_var) { /* We have to update type of CV because it may be captured by exception backtrace or released on RETURN */ if ((op1_def_info & MAY_BE_ANY) == MAY_BE_LONG) { jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_LONG); if (JIT_G(current_frame)) { SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_LONG, 1); } } else if ((op1_def_info & MAY_BE_ANY) == MAY_BE_DOUBLE) { jit_set_Z_TYPE_INFO(jit, op1_use_addr, IS_DOUBLE); if (JIT_G(current_frame)) { SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(Z_OFFSET(op1_use_addr)), IS_DOUBLE, 1); } } else { ZEND_UNREACHABLE(); } } } if (opline->result_type != IS_UNUSED) { if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { return 0; } } return 1; } static ir_op zend_jit_cmp_op(const zend_op *opline) { ir_op op; switch (opline->opcode) { case ZEND_IS_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_CASE: case ZEND_CASE_STRICT: op = IR_EQ; break; case ZEND_IS_NOT_EQUAL: case ZEND_IS_NOT_IDENTICAL: op = IR_NE; break; case ZEND_IS_SMALLER: op = IR_LT; break; case ZEND_IS_SMALLER_OR_EQUAL: op = IR_LE; break; default: ZEND_UNREACHABLE(); } return op; } static ir_ref zend_jit_cmp_long_long(zend_jit_ctx *jit, const zend_op *opline, zend_ssa_range *op1_range, zend_jit_addr op1_addr, zend_ssa_range *op2_range, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr, bool skip_comparison) { ir_ref ref; bool result; if (zend_jit_is_constant_cmp_long_long(opline, op1_range, op1_addr, op2_range, op2_addr, &result)) { if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { jit_set_Z_TYPE_INFO(jit, res_addr, result ? IS_TRUE : IS_FALSE); } if (smart_branch_opcode && !exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { return jit_IF_ex(jit, IR_FALSE, result ? target_label : target_label2); } else if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { return jit_IF_ex(jit, IR_TRUE, result ? target_label : target_label2); } else { ZEND_UNREACHABLE(); } } if (opline->opcode != ZEND_IS_IDENTICAL && opline->opcode != ZEND_IS_NOT_IDENTICAL && opline->opcode != ZEND_CASE_STRICT) { return ir_END(); } else { return IR_NULL; /* success */ } } ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_LVAL(jit, op1_addr), jit_Z_LVAL(jit, op2_addr)); if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } } else { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } } } else if (smart_branch_opcode) { return jit_IF_ex(jit, ref, (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label); } if (opline->opcode != ZEND_IS_IDENTICAL && opline->opcode != ZEND_IS_NOT_IDENTICAL && opline->opcode != ZEND_CASE_STRICT) { return ir_END(); } else { return IR_NULL; /* success */ } } static ir_ref zend_jit_cmp_long_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), ir_INT2D(jit_Z_LVAL(jit, op1_addr)), jit_Z_DVAL(jit, op2_addr)); if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } } else if (smart_branch_opcode) { return jit_IF_ex(jit, ref, (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label); } return ir_END(); } static ir_ref zend_jit_cmp_double_long(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), ir_INT2D(jit_Z_LVAL(jit, op2_addr))); if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } } else if (smart_branch_opcode) { return jit_IF_ex(jit, ref, (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label); } return ir_END(); } static ir_ref zend_jit_cmp_double_double(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr op1_addr, zend_jit_addr op2_addr, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { ir_ref ref = ir_CMP_OP(zend_jit_cmp_op(opline), jit_Z_DVAL(jit, op1_addr), jit_Z_DVAL(jit, op2_addr)); if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } } else { if (opline->opcode != ZEND_IS_NOT_IDENTICAL) { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } } } else if (smart_branch_opcode) { return jit_IF_ex(jit, ref, (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label); } if (opline->opcode != ZEND_IS_IDENTICAL && opline->opcode != ZEND_IS_NOT_IDENTICAL && opline->opcode != ZEND_CASE_STRICT) { return ir_END(); } else { return IR_NULL; /* success */ } } static ir_ref zend_jit_cmp_slow(zend_jit_ctx *jit, ir_ref ref, const zend_op *opline, zend_jit_addr res_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { ref = ir_CMP_OP(zend_jit_cmp_op(opline), ref, ir_CONST_I32(0)); if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } } else if (smart_branch_opcode) { return jit_IF_ex(jit, ref, (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label); } return ir_END(); } static int zend_jit_cmp(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr, bool skip_comparison) { ir_ref ref = IR_UNUSED; ir_ref if_op1_long = IR_UNUSED; ir_ref if_op1_double = IR_UNUSED; ir_ref if_op2_double = IR_UNUSED; ir_ref if_op1_long_op2_long = IR_UNUSED; ir_ref if_op1_long_op2_double = IR_UNUSED; ir_ref if_op1_double_op2_double = IR_UNUSED; ir_ref if_op1_double_op2_long = IR_UNUSED; ir_ref slow_inputs = IR_UNUSED; bool same_ops = zend_jit_same_addr(op1_addr, op2_addr); bool has_slow = (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))); ir_refs *end_inputs; ir_refs_init(end_inputs, 8); if (Z_MODE(op1_addr) == IS_REG) { if (!has_concrete_type(op2_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op1_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, op1_addr); } } else if (Z_MODE(op2_addr) == IS_REG) { if (!has_concrete_type(op1_info & MAY_BE_ANY) && jit->ra[Z_SSA_VAR(op2_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, op2_addr); } } if ((op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG)) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if_op1_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_TRUE(if_op1_long); } if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG))) { if_op1_long_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); ir_IF_FALSE_cold(if_op1_long_op2_long); if (op2_info & MAY_BE_DOUBLE) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { if_op1_long_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); ir_IF_FALSE_cold(if_op1_long_op2_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_long_op2_double); } ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr); if (!ref) { return 0; } ir_refs_add(end_inputs, ref); } else { ir_END_list(slow_inputs); } ir_IF_TRUE(if_op1_long_op2_long); } ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison); if (!ref) { return 0; } ir_refs_add(end_inputs, ref); if (if_op1_long) { ir_IF_FALSE_cold(if_op1_long); } if (op1_info & MAY_BE_DOUBLE) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); ir_IF_FALSE_cold(if_op1_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_double); } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); ir_IF_TRUE(if_op1_double_op2_double); } ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr); if (!ref) { return 0; } ir_refs_add(end_inputs, ref); if (if_op1_double_op2_double) { ir_IF_FALSE_cold(if_op1_double_op2_double); } } if (!same_ops) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) { if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); ir_IF_FALSE_cold(if_op1_double_op2_long); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_double_op2_long); } ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr); if (!ref) { return 0; } ir_refs_add(end_inputs, ref); } else if (if_op1_double_op2_double) { ir_END_list(slow_inputs); } } else if (if_op1_long) { ir_END_list(slow_inputs); } } else if ((op1_info & MAY_BE_DOUBLE) && !(op1_info & MAY_BE_LONG) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { if_op1_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); ir_IF_FALSE_cold(if_op1_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_double); } if (op2_info & MAY_BE_DOUBLE) { if (!same_ops && (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { if_op1_double_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); ir_IF_TRUE(if_op1_double_op2_double); } ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr); if (!ref) { return 0; } ir_refs_add(end_inputs, ref); if (if_op1_double_op2_double) { ir_IF_FALSE_cold(if_op1_double_op2_double); } } if (!same_ops && (op2_info & MAY_BE_LONG)) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { if_op1_double_op2_long = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); ir_IF_FALSE_cold(if_op1_double_op2_long); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_double_op2_long); } ref = zend_jit_cmp_double_long(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr); if (!ref) { return 0; } ir_refs_add(end_inputs, ref); } else if (if_op1_double_op2_double) { ir_END_list(slow_inputs); } } else if ((op2_info & MAY_BE_DOUBLE) && !(op2_info & MAY_BE_LONG) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE))) { if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE)) { if_op2_double = jit_if_Z_TYPE(jit, op2_addr, IS_DOUBLE); ir_IF_FALSE_cold(if_op2_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op2_double); } if (op1_info & MAY_BE_DOUBLE) { if (!same_ops && (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_DOUBLE))) { if_op1_double_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_DOUBLE); ir_IF_TRUE(if_op1_double_op2_double); } ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr); if (!ref) { return 0; } ir_refs_add(end_inputs, ref); if (if_op1_double_op2_double) { ir_IF_FALSE_cold(if_op1_double_op2_double); } } if (!same_ops && (op1_info & MAY_BE_LONG)) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_DOUBLE|MAY_BE_LONG))) { if_op1_long_op2_double = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_FALSE_cold(if_op1_long_op2_double); ir_END_list(slow_inputs); ir_IF_TRUE(if_op1_long_op2_double); } ref = zend_jit_cmp_long_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr); if (!ref) { return 0; } ir_refs_add(end_inputs, ref); } else if (if_op1_double_op2_double) { ir_END_list(slow_inputs); } } if (has_slow || (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE))) || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_DOUBLE)))) { ir_ref op1, op2, ref; if (slow_inputs) { ir_MERGE_list(slow_inputs); } jit_SET_EX_OPLINE(jit, opline); if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { return 0; } op1_addr = real_addr; } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { return 0; } op2_addr = real_addr; } op1 = jit_ZVAL_ADDR(jit, op1_addr); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, NULL, 0); } op2 = jit_ZVAL_ADDR(jit, op2_addr); if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, NULL, 0); } ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_compare), op1, op2); if (opline->opcode != ZEND_CASE) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); } jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL); if (may_throw) { zend_jit_check_exception_undef_result(jit, opline); } ref = zend_jit_cmp_slow(jit, ref, opline, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr); if (!ref) { return 0; } ir_refs_add(end_inputs, ref); } if (end_inputs->count) { uint32_t n = end_inputs->count; if (smart_branch_opcode && !exit_addr) { zend_basic_block *bb; ir_ref ref; uint32_t label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label; uint32_t label2 = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label : target_label2; ZEND_ASSERT(jit->b >= 0); bb = &jit->ssa->cfg.blocks[jit->b]; ZEND_ASSERT(bb->successors_count == 2); if (UNEXPECTED(bb->successors[0] == bb->successors[1])) { ir_ref merge_inputs = IR_UNUSED; while (n) { n--; ir_IF_TRUE(end_inputs->refs[n]); ir_END_list(merge_inputs); ir_IF_FALSE(end_inputs->refs[n]); ir_END_list(merge_inputs); } ir_MERGE_list(merge_inputs); _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); } else if (n == 1) { ref = end_inputs->refs[0]; _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); } else { ir_ref true_inputs = IR_UNUSED, false_inputs = IR_UNUSED; while (n) { n--; jit_IF_TRUE_FALSE_ex(jit, end_inputs->refs[n], label); ir_END_list(true_inputs); jit_IF_TRUE_FALSE_ex(jit, end_inputs->refs[n], label2); ir_END_list(false_inputs); } ir_MERGE_list(true_inputs); _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); ir_MERGE_list(false_inputs); _zend_jit_add_predecessor_ref(jit, label2, jit->b, ir_END()); } jit->b = -1; } else { ir_MERGE_N(n, end_inputs->refs); } } else if (smart_branch_opcode && !exit_addr) { /* dead code */ _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END()); jit->b = -1; } return 1; } static int zend_jit_identical(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_ssa_range *op1_range, zend_jit_addr op1_addr, uint32_t op2_info, zend_ssa_range *op2_range, zend_jit_addr op2_addr, zend_jit_addr res_addr, int may_throw, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr, bool skip_comparison) { bool always_false = 0, always_true = 0; ir_ref ref = IR_UNUSED; if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { ir_ref op1 = jit_ZVAL_ADDR(jit, op1_addr); op1 = zend_jit_zval_check_undef(jit, op1, opline->op1.var, opline, 0); op1_info |= MAY_BE_NULL; op1_addr = ZEND_ADDR_REF_ZVAL(op1); } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { ir_ref op2 = jit_ZVAL_ADDR(jit, op2_addr); op2 = zend_jit_zval_check_undef(jit, op2, opline->op2.var, opline, 0); op2_info |= MAY_BE_NULL; op2_addr = ZEND_ADDR_REF_ZVAL(op2); } if ((op1_info & op2_info & MAY_BE_ANY) == 0) { always_false = 1; } else if (has_concrete_type(op1_info) && has_concrete_type(op2_info) && concrete_type(op1_info) == concrete_type(op2_info) && concrete_type(op1_info) <= IS_TRUE) { always_true = 1; } else if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_MODE(op2_addr) == IS_CONST_ZVAL) { if (zend_is_identical(Z_ZV(op1_addr), Z_ZV(op2_addr))) { always_true = 1; } else { always_false = 1; } } if (always_true) { if (opline->opcode != ZEND_CASE_STRICT) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_TRUE : IS_FALSE); } if (may_throw) { zend_jit_check_exception(jit); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ || smart_branch_opcode == ZEND_JMPNZ_EX) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } } else if (smart_branch_opcode) { uint32_t label; if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label : target_label2; } else { label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label; } _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); jit->b = -1; } return 1; } else if (always_false) { if (opline->opcode != ZEND_CASE_STRICT) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPZ_EX || smart_branch_opcode == ZEND_JMPNZ_EX) { jit_set_Z_TYPE_INFO(jit, res_addr, opline->opcode != ZEND_IS_NOT_IDENTICAL ? IS_FALSE : IS_TRUE); } if (may_throw) { zend_jit_check_exception(jit); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } } else if (smart_branch_opcode) { uint32_t label; if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label; } else { label = (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label : target_label2; } _zend_jit_add_predecessor_ref(jit, label, jit->b, ir_END()); jit->b = -1; } return 1; } if ((opline->op1_type & (IS_CV|IS_VAR)) && (op1_info & MAY_BE_REF)) { ref = jit_ZVAL_ADDR(jit, op1_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } if ((opline->op2_type & (IS_CV|IS_VAR)) && (op2_info & MAY_BE_REF)) { ref = jit_ZVAL_ADDR(jit, op2_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op2_addr = ZEND_ADDR_REF_ZVAL(ref); } if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG && (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) { ref = zend_jit_cmp_long_long(jit, opline, op1_range, op1_addr, op2_range, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr, skip_comparison); if (!ref) { return 0; } } else if ((op1_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE && (op2_info & (MAY_BE_REF|MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_DOUBLE) { ref = zend_jit_cmp_double_double(jit, opline, op1_addr, op2_addr, res_addr, smart_branch_opcode, target_label, target_label2, exit_addr); if (!ref) { return 0; } } else { if (opline->op1_type != IS_CONST) { if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { return 0; } op1_addr = real_addr; } } if (opline->op2_type != IS_CONST) { if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { return 0; } } } if (Z_MODE(op1_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op1_addr)) <= IS_TRUE) { zval *val = Z_ZV(op1_addr); ref = ir_EQ(jit_Z_TYPE(jit, op2_addr), ir_CONST_U8(Z_TYPE_P(val))); } else if (Z_MODE(op2_addr) == IS_CONST_ZVAL && Z_TYPE_P(Z_ZV(op2_addr)) <= IS_TRUE) { zval *val = Z_ZV(op2_addr); ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(Z_TYPE_P(val))); } else { if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store_inv(jit, op1_addr, real_addr, op1_info)) { return 0; } op1_addr = real_addr; } if (Z_MODE(op2_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); if (!zend_jit_spill_store_inv(jit, op2_addr, real_addr, op2_info)) { return 0; } op2_addr = real_addr; } if (may_throw) { jit_SET_EX_OPLINE(jit, opline); } ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_is_identical), jit_ZVAL_ADDR(jit, op1_addr), jit_ZVAL_ADDR(jit, op2_addr)); } if (!smart_branch_opcode || smart_branch_opcode == ZEND_JMPNZ_EX || smart_branch_opcode == ZEND_JMPZ_EX) { if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref))); } else { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } } if (opline->opcode != ZEND_CASE_STRICT) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); } jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL); if (may_throw) { zend_jit_check_exception_undef_result(jit, opline); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } } else if (smart_branch_opcode) { if (opline->opcode == ZEND_IS_NOT_IDENTICAL) { /* swap labels */ uint32_t tmp = target_label; target_label = target_label2; target_label2 = tmp; } ref = jit_IF_ex(jit, ref, (smart_branch_opcode == ZEND_JMPZ || smart_branch_opcode == ZEND_JMPZ_EX) ? target_label2 : target_label); } } if (smart_branch_opcode && !exit_addr) { zend_basic_block *bb; ZEND_ASSERT(jit->b >= 0); bb = &jit->ssa->cfg.blocks[jit->b]; ZEND_ASSERT(bb->successors_count == 2); if (bb->successors_count == 2 && bb->successors[0] == bb->successors[1]) { ir_IF_TRUE(ref); ir_MERGE_WITH_EMPTY_FALSE(ref); _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END()); } else { ZEND_ASSERT(bb->successors_count == 2); _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); } jit->b = -1; } return 1; } static int zend_jit_bool_jmpznz(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, uint32_t target_label, uint32_t target_label2, int may_throw, uint8_t branch_opcode, const void *exit_addr) { uint32_t true_label = -1; uint32_t false_label = -1; bool set_bool = 0; bool set_bool_not = 0; bool always_true = 0, always_false = 0; ir_ref ref, end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED; ir_type type = IR_UNUSED; if (branch_opcode == ZEND_BOOL) { set_bool = 1; } else if (branch_opcode == ZEND_BOOL_NOT) { set_bool = 1; set_bool_not = 1; } else if (branch_opcode == ZEND_JMPZ) { true_label = target_label2; false_label = target_label; } else if (branch_opcode == ZEND_JMPNZ) { true_label = target_label; false_label = target_label2; } else if (branch_opcode == ZEND_JMPZ_EX) { set_bool = 1; true_label = target_label2; false_label = target_label; } else if (branch_opcode == ZEND_JMPNZ_EX) { set_bool = 1; true_label = target_label; false_label = target_label2; } else { ZEND_UNREACHABLE(); } if (opline->op1_type == IS_CV && (op1_info & MAY_BE_REF)) { ref = jit_ZVAL_ADDR(jit, op1_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } if (Z_MODE(op1_addr) == IS_CONST_ZVAL /* NAN Value must cause a warning to be emitted */ && (Z_TYPE_P(Z_ZV(op1_addr)) != IS_DOUBLE || !zend_isnan(Z_DVAL_P(Z_ZV(op1_addr))))) { if (zend_is_true(Z_ZV(op1_addr))) { always_true = 1; } else { always_false = 1; } } else if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { if (!(op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)-MAY_BE_TRUE))) { always_true = 1; } else if (!(op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE)))) { if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { ref = jit_ZVAL_ADDR(jit, op1_addr); zend_jit_zval_check_undef(jit, ref, opline->op1.var, opline, 0); } always_false = 1; } } if (always_true) { if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); if (may_throw) { zend_jit_check_exception(jit); } if (true_label != (uint32_t)-1) { ZEND_ASSERT(exit_addr == NULL); _zend_jit_add_predecessor_ref(jit, true_label, jit->b, ir_END()); jit->b = -1; } return 1; } else if (always_false) { if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); if (may_throw) { zend_jit_check_exception(jit); } if (false_label != (uint32_t)-1) { ZEND_ASSERT(exit_addr == NULL); _zend_jit_add_predecessor_ref(jit, false_label, jit->b, ir_END()); jit->b = -1; } return 1; } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE)) { type = jit_Z_TYPE(jit, op1_addr); if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE)) { ir_ref if_type = ir_IF(ir_LT(type, ir_CONST_U8(IS_TRUE))); ir_IF_TRUE_cold(if_type); if (op1_info & MAY_BE_UNDEF) { zend_jit_type_check_undef(jit, type, opline->op1.var, opline, 1, 0, 1); } if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); } if (exit_addr) { if (branch_opcode == ZEND_JMPNZ || branch_opcode == ZEND_JMPNZ_EX) { ir_END_list(end_inputs); } else { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } } else if (false_label != (uint32_t)-1) { ir_END_list(false_inputs); } else { ir_END_list(end_inputs); } ir_IF_FALSE(if_type); } if (op1_info & MAY_BE_TRUE) { ir_ref if_type = IR_UNUSED; if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE))) { if_type = ir_IF(ir_EQ(type, ir_CONST_U8(IS_TRUE))); ir_IF_TRUE(if_type); } if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); } if (exit_addr) { if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { ir_END_list(end_inputs); } else { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } } else if (true_label != (uint32_t)-1) { ir_END_list(true_inputs); } else { ir_END_list(end_inputs); } if (if_type) { ir_IF_FALSE(if_type); } } } if (op1_info & MAY_BE_LONG) { ir_ref if_long = IR_UNUSED; ir_ref ref; if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG))) { if (!type) { type = jit_Z_TYPE(jit, op1_addr); } if_long = ir_IF(ir_EQ(type, ir_CONST_U8(IS_LONG))); ir_IF_TRUE(if_long); } ref = jit_Z_LVAL(jit, op1_addr); if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) { ref = ir_NE(ref, ir_CONST_LONG(0)); if (set_bool_not) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref))); } else { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } ir_END_list(end_inputs); } else if (exit_addr) { if (set_bool) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, ir_CONST_LONG(0))), ir_CONST_U32(IS_FALSE))); } if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } ir_END_list(end_inputs); } else { ir_ref if_val = ir_IF(ref); ir_IF_TRUE(if_val); if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); } ir_END_list(true_inputs); ir_IF_FALSE(if_val); if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); } ir_END_list(false_inputs); } if (if_long) { ir_IF_FALSE(if_long); } } if (op1_info & MAY_BE_DOUBLE) { ir_ref if_double = IR_UNUSED; ir_ref ref; if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) { if (!type) { type = jit_Z_TYPE(jit, op1_addr); } if_double = ir_IF(ir_EQ(type, ir_CONST_U8(IS_DOUBLE))); ir_IF_TRUE(if_double); } ir_ref dval = jit_Z_DVAL(jit, op1_addr); ir_ref is_nan = ir_NE(dval, dval); ir_ref if_val = ir_IF(is_nan); ir_IF_TRUE_cold(if_val); jit_SET_EX_OPLINE(jit, opline); ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_nan_coerced_to_type_warning)); ir_MERGE_WITH_EMPTY_FALSE(if_val); ref = ir_NE(dval, ir_CONST_DOUBLE(0.0)); if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) { if (set_bool_not) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref))); } else { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } ir_END_list(end_inputs); } else if (exit_addr) { if (set_bool) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } ir_END_list(end_inputs); } else { ir_ref if_val = ir_IF(ref); ir_IF_TRUE(if_val); if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); } ir_END_list(true_inputs); ir_IF_FALSE(if_val); if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); } ir_END_list(false_inputs); } if (if_double) { ir_IF_FALSE(if_double); } } if (op1_info & (MAY_BE_ANY - (MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_TRUE|MAY_BE_LONG|MAY_BE_DOUBLE))) { jit_SET_EX_OPLINE(jit, opline); ref = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_is_true), jit_ZVAL_ADDR(jit, op1_addr)); jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); if (may_throw) { zend_jit_check_exception_undef_result(jit, opline); } if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT) { if (set_bool_not) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_SUB_U32(ir_CONST_U32(IS_TRUE), ir_ZEXT_U32(ref))); } else { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } if (end_inputs) { ir_END_list(end_inputs); } } else if (exit_addr) { if (set_bool) { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); } if (branch_opcode == ZEND_JMPZ || branch_opcode == ZEND_JMPZ_EX) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } if (end_inputs) { ir_END_list(end_inputs); } } else { ir_ref if_val = ir_IF(ref); ir_IF_TRUE(if_val); if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_FALSE : IS_TRUE); } ir_END_list(true_inputs); ir_IF_FALSE(if_val); if (set_bool) { jit_set_Z_TYPE_INFO(jit, res_addr, set_bool_not ? IS_TRUE : IS_FALSE); } ir_END_list(false_inputs); } } if (branch_opcode == ZEND_BOOL || branch_opcode == ZEND_BOOL_NOT || exit_addr) { if (end_inputs) { ir_MERGE_list(end_inputs); } } else { _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs); } return 1; } static int zend_jit_defined(zend_jit_ctx *jit, const zend_op *opline, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { uint32_t defined_label = (uint32_t)-1; uint32_t undefined_label = (uint32_t)-1; zval *zv = RT_CONSTANT(opline, opline->op1); zend_jit_addr res_addr = 0; ir_ref ref, ref2, if_set, if_zero, if_set2; ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED; if (smart_branch_opcode && !exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { defined_label = target_label2; undefined_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { defined_label = target_label; undefined_label = target_label2; } else { ZEND_UNREACHABLE(); } } else { res_addr = RES_ADDR(); } // if (CACHED_PTR(opline->extended_value)) { ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value)); if_set = ir_IF(ref); ir_IF_FALSE_cold(if_set); if_zero = ir_END(); ir_IF_TRUE(if_set); if_set2 = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL))); ir_IF_FALSE(if_set2); if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else { ir_END_list(end_inputs); } } else if (smart_branch_opcode) { ir_END_list(true_inputs); } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); ir_END_list(end_inputs); } ir_IF_TRUE_cold(if_set2); ref2 = jit_EG(zend_constants); ref = ir_SHR_A(ref, ir_CONST_ADDR(1)); if (sizeof(void*) == 8) { ref = ir_TRUNC_U32(ref); } ref2 = ir_EQ(ref, ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(ref2), offsetof(HashTable, nNumOfElements)))); ref2 = ir_IF(ref2); ir_IF_TRUE(ref2); if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else { ir_END_list(end_inputs); } } else if (smart_branch_opcode) { ir_END_list(false_inputs); } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); ir_END_list(end_inputs); } ir_IF_FALSE(ref2); ir_MERGE_2(if_zero, ir_END()); jit_SET_EX_OPLINE(jit, opline); ref2 = ir_NE(ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_check_constant), ir_CONST_ADDR(zv)), IR_NULL); if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { ir_GUARD(ref2, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref2, ir_CONST_ADDR(exit_addr)); } ir_END_list(end_inputs); } else if (smart_branch_opcode) { ref2 = ir_IF(ref2); ir_IF_TRUE(ref2); ir_END_list(true_inputs); ir_IF_FALSE(ref2); ir_END_list(false_inputs); } else { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref2), ir_CONST_U32(IS_FALSE))); ir_END_list(end_inputs); } if (!smart_branch_opcode || exit_addr) { if (end_inputs) { ir_MERGE_list(end_inputs); } } else { _zend_jit_merge_smart_branch_inputs(jit, defined_label, undefined_label, true_inputs, false_inputs); } return 1; } static int zend_jit_escape_if_undef(zend_jit_ctx *jit, int var, uint32_t flags, const zend_op *opline, int8_t reg) { zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg)); ir_ref if_def = ir_IF(jit_Z_TYPE(jit, reg_addr)); ir_IF_FALSE_cold(if_def); if (flags & ZEND_JIT_EXIT_RESTORE_CALL) { if (!zend_jit_save_call_chain(jit, -1)) { return 0; } } if ((opline-1)->opcode != ZEND_FETCH_CONSTANT && (opline-1)->opcode != ZEND_FETCH_LIST_R && ((opline-1)->op1_type & (IS_VAR|IS_TMP_VAR)) && !(flags & ZEND_JIT_EXIT_FREE_OP1)) { zend_jit_addr val_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline-1)->op1.var); zend_jit_zval_try_addref(jit, val_addr); } jit_LOAD_IP_ADDR(jit, opline - 1); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape)); ir_IF_TRUE(if_def); return 1; } static int zend_jit_restore_zval(zend_jit_ctx *jit, int var, int8_t reg) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); zend_jit_addr reg_addr = ZEND_ADDR_REF_ZVAL(zend_jit_deopt_rload(jit, IR_ADDR, reg)); // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) jit_ZVAL_COPY(jit, var_addr, MAY_BE_ANY, reg_addr, MAY_BE_ANY, 1); return 1; } static zend_jit_addr zend_jit_guard_fetch_result_type(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr val_addr, uint8_t type, bool deref, uint32_t flags, bool op1_avoid_refcounting) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; int32_t exit_point; const void *res_exit_addr = NULL; ir_ref end1 = IR_UNUSED, ref1 = IR_UNUSED; ir_ref ref = jit_ZVAL_ADDR(jit, val_addr); uint32_t old_op1_info = 0; uint32_t old_info; ir_ref old_ref; if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) { old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); if (op1_avoid_refcounting || ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && STACK_FLAGS(stack, EX_VAR_TO_NUM(opline->op1.var)) & (ZREG_ZVAL_ADDREF|ZREG_THIS))) { SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } } old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); old_ref = STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var)); CLEAR_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); if (deref) { ir_ref if_type; if (type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) { if_type = ir_IF(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type))); } else { if_type = jit_if_Z_TYPE(jit, val_addr, type); } ir_IF_TRUE(if_type); end1 = ir_END(); ref1 = ref; ir_IF_FALSE_cold(if_type); SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY); exit_point = zend_jit_trace_get_exit_point(opline+1, flags); res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!res_exit_addr) { return 0; } jit_guard_Z_TYPE(jit, val_addr, IS_REFERENCE, res_exit_addr); ref = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val)); val_addr = ZEND_ADDR_REF_ZVAL(ref); } SET_STACK_REF_EX(stack, EX_VAR_TO_NUM(opline->result.var), ref, ZREG_ZVAL_COPY); exit_point = zend_jit_trace_get_exit_point(opline+1, flags); res_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!res_exit_addr) { return 0; } if (!deref && type == IS_NULL && (opline->opcode == ZEND_FETCH_DIM_IS || opline->opcode == ZEND_FETCH_OBJ_IS)) { ir_GUARD(ir_ULE(jit_Z_TYPE(jit, val_addr), ir_CONST_U8(type)), ir_CONST_ADDR(res_exit_addr)); } else { jit_guard_Z_TYPE(jit, val_addr, type, res_exit_addr); } if (deref) { ir_MERGE_WITH(end1); ref = ir_PHI_2(IR_ADDR, ref, ref1); } val_addr = ZEND_ADDR_REF_ZVAL(ref); SET_STACK_REF(stack, EX_VAR_TO_NUM(opline->result.var), old_ref); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); if (opline->op1_type & (IS_VAR|IS_TMP_VAR|IS_CV)) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); } return val_addr; } static int zend_jit_fetch_constant(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, zend_jit_addr res_addr) { zval *zv = RT_CONSTANT(opline, opline->op2) + 1; uint32_t res_info = RES_INFO(); ir_ref ref, ref2, if_set, if_special, not_set_path, special_path, fast_path; // JIT: c = CACHED_PTR(opline->extended_value); ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value)); // JIT: if (c != NULL) if_set = ir_IF(ref); if (!zend_jit_is_persistent_constant(zv, opline->op1.num)) { // JIT: if (!IS_SPECIAL_CACHE_VAL(c)) ir_IF_FALSE_cold(if_set); not_set_path = ir_END(); ir_IF_TRUE(if_set); if_special = ir_IF(ir_AND_A(ref, ir_CONST_ADDR(CACHE_SPECIAL))); ir_IF_TRUE_cold(if_special); special_path = ir_END(); ir_IF_FALSE(if_special); fast_path = ir_END(); ir_MERGE_2(not_set_path, special_path); } else { ir_IF_TRUE(if_set); fast_path = ir_END(); ir_IF_FALSE_cold(if_set); } // JIT: zend_jit_get_constant(RT_CONSTANT(opline, opline->op2) + 1, opline->op1.num); jit_SET_EX_OPLINE(jit, opline); ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_get_constant), ir_CONST_ADDR(zv), ir_CONST_U32(opline->op1.num)); ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_MERGE_WITH(fast_path); ref = ir_PHI_2(IR_ADDR, ref2, ref); if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { uint8_t type = concrete_type(res_info); zend_jit_addr const_addr = ZEND_ADDR_REF_ZVAL(ref); const_addr = zend_jit_guard_fetch_result_type(jit, opline, const_addr, type, 0, 0, 0); if (!const_addr) { return 0; } res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, res_info, 1); if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { return 0; } } else { ir_ref const_addr = ZEND_ADDR_REF_ZVAL(ref); // JIT: ZVAL_COPY_OR_DUP(EX_VAR(opline->result.var), &c->value); (no dup) jit_ZVAL_COPY(jit, res_addr, MAY_BE_ANY, const_addr, MAY_BE_ANY, 1); } return 1; } static int zend_jit_type_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { uint32_t mask; zend_jit_addr op1_addr = OP1_ADDR(); zend_jit_addr res_addr = 0; uint32_t true_label = -1, false_label = -1; ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED; // TODO: support for is_resource() ??? ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); if (smart_branch_opcode && !exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { true_label = target_label2; false_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { true_label = target_label; false_label = target_label2; } else { ZEND_UNREACHABLE(); } } else { res_addr = RES_ADDR(); } if (op1_info & MAY_BE_UNDEF) { ir_ref if_def = IR_UNUSED; if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); ir_IF_FALSE_cold(if_def); } jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var)); zend_jit_check_exception_undef_result(jit, opline); if (opline->extended_value & MAY_BE_NULL) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else { ir_END_list(end_inputs); } } else if (smart_branch_opcode) { ir_END_list(true_inputs); } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); ir_END_list(end_inputs); } } else { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else { ir_END_list(end_inputs); } } else if (smart_branch_opcode) { ir_END_list(false_inputs); } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); if (if_def) { ir_END_list(end_inputs); } } } if (if_def) { ir_IF_TRUE(if_def); op1_info |= MAY_BE_NULL; } } if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { mask = opline->extended_value; if (!(op1_info & MAY_BE_GUARD) && !(op1_info & (MAY_BE_ANY - mask))) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else if (end_inputs) { ir_END_list(end_inputs); } } else if (smart_branch_opcode) { ir_END_list(true_inputs); } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); ir_END_list(end_inputs); } } else if (!(op1_info & MAY_BE_GUARD) && !(op1_info & mask)) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else if (end_inputs) { ir_END_list(end_inputs); } } else if (smart_branch_opcode) { ir_END_list(false_inputs); } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); ir_END_list(end_inputs); } } else { ir_ref ref; bool invert = 0; uint8_t type; switch (mask) { case MAY_BE_NULL: type = IS_NULL; break; case MAY_BE_FALSE: type = IS_FALSE; break; case MAY_BE_TRUE: type = IS_TRUE; break; case MAY_BE_LONG: type = IS_LONG; break; case MAY_BE_DOUBLE: type = IS_DOUBLE; break; case MAY_BE_STRING: type = IS_STRING; break; case MAY_BE_ARRAY: type = IS_ARRAY; break; case MAY_BE_OBJECT: type = IS_OBJECT; break; case MAY_BE_ANY - MAY_BE_NULL: type = IS_NULL; invert = 1; break; case MAY_BE_ANY - MAY_BE_FALSE: type = IS_FALSE; invert = 1; break; case MAY_BE_ANY - MAY_BE_TRUE: type = IS_TRUE; invert = 1; break; case MAY_BE_ANY - MAY_BE_LONG: type = IS_LONG; invert = 1; break; case MAY_BE_ANY - MAY_BE_DOUBLE: type = IS_DOUBLE; invert = 1; break; case MAY_BE_ANY - MAY_BE_STRING: type = IS_STRING; invert = 1; break; case MAY_BE_ANY - MAY_BE_ARRAY: type = IS_ARRAY; invert = 1; break; case MAY_BE_ANY - MAY_BE_OBJECT: type = IS_OBJECT; invert = 1; break; case MAY_BE_ANY - MAY_BE_RESOURCE: type = IS_OBJECT; invert = 1; break; default: type = 0; } if (op1_info & MAY_BE_REF) { ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } if (type == 0) { ref = ir_AND_U32(ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)), ir_CONST_U32(mask)); if (!smart_branch_opcode) { ref = ir_NE(ref, ir_CONST_U32(0)); } } else if (invert) { ref = ir_NE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type)); } else { ref = ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(type)); } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } if (end_inputs) { ir_END_list(end_inputs); } } else if (smart_branch_opcode) { ir_ref if_val = ir_IF(ref); ir_IF_TRUE(if_val); ir_END_list(true_inputs); ir_IF_FALSE(if_val); ir_END_list(false_inputs); } else { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); ir_END_list(end_inputs); } } } if (!smart_branch_opcode || exit_addr) { if (end_inputs) { ir_MERGE_list(end_inputs); } else if (exit_addr && !jit->ctx.control) { ir_BEGIN(IR_UNUSED); /* unreachable block */ } } else { _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs); } return 1; } static int zend_jit_isset_isempty_cv(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_jit_addr res_addr = RES_ADDR(); uint32_t true_label = -1, false_label = -1; ir_ref end_inputs = IR_UNUSED, true_inputs = IR_UNUSED, false_inputs = IR_UNUSED; // TODO: support for empty() ??? ZEND_ASSERT(opline->extended_value != MAY_BE_RESOURCE); if (smart_branch_opcode && !exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { true_label = target_label2; false_label = target_label; } else if (smart_branch_opcode == ZEND_JMPNZ) { true_label = target_label; false_label = target_label2; } else { ZEND_UNREACHABLE(); } } else { res_addr = RES_ADDR(); } if (op1_info & MAY_BE_REF) { ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } if (!(op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { if (exit_addr) { ZEND_ASSERT(smart_branch_opcode == ZEND_JMPZ); } else if (smart_branch_opcode) { ir_END_list(true_inputs); } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); ir_END_list(end_inputs); } } else if (!(op1_info & (MAY_BE_ANY - MAY_BE_NULL))) { if (exit_addr) { ZEND_ASSERT(smart_branch_opcode == ZEND_JMPNZ); } else if (smart_branch_opcode) { ir_END_list(false_inputs); } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); ir_END_list(end_inputs); } } else { ir_ref ref = ir_GT(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL)); if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } } else if (smart_branch_opcode) { ir_ref if_val = ir_IF(ref); ir_IF_TRUE(if_val); ir_END_list(true_inputs); ir_IF_FALSE(if_val); ir_END_list(false_inputs); } else { jit_set_Z_TYPE_INFO_ref(jit, jit_ZVAL_ADDR(jit, res_addr), ir_ADD_U32(ir_ZEXT_U32(ref), ir_CONST_U32(IS_FALSE))); ir_END_list(end_inputs); } } if (!smart_branch_opcode || exit_addr) { if (end_inputs) { ir_MERGE_list(end_inputs); } } else { _zend_jit_merge_smart_branch_inputs(jit, true_label, false_label, true_inputs, false_inputs); } return 1; } /* copy of hidden zend_closure */ typedef struct _zend_closure { zend_object std; zend_function func; zval this_ptr; zend_class_entry *called_scope; zif_handler orig_internal_handler; } zend_closure; static int zend_jit_stack_check(zend_jit_ctx *jit, const zend_op *opline, uint32_t used_stack) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } // JIT: if (EG(vm_stack_end) - EG(vm_stack_top) < used_stack) ir_GUARD( ir_UGE( ir_SUB_A(ir_LOAD_A(jit_EG(vm_stack_end)), ir_LOAD_A(jit_EG(vm_stack_top))), ir_CONST_ADDR(used_stack)), ir_CONST_ADDR(exit_addr)); return 1; } static int zend_jit_free_trampoline(zend_jit_ctx *jit, ir_ref func) { // JIT: if (UNEXPECTED(func->common.fn_flags & ZEND_ACC_CALL_VIA_TRAMPOLINE)) ir_ref if_trampoline = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, common.fn_flags))), ir_CONST_U32(ZEND_ACC_CALL_VIA_TRAMPOLINE))); ir_IF_TRUE(if_trampoline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_trampoline_helper), func); ir_MERGE_WITH_EMPTY_FALSE(if_trampoline); return 1; } static int zend_jit_push_call_frame(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_function *func, bool is_closure, bool delayed_fetch_this, int checked_stack, ir_ref func_ref, ir_ref this_ref) { uint32_t used_stack; ir_ref used_stack_ref = IR_UNUSED; bool stack_check = 1; ir_ref rx, ref, top, if_enough_stack, cold_path = IR_UNUSED; ZEND_ASSERT(func_ref != IR_NULL); if (func) { used_stack = zend_vm_calc_used_stack(opline->extended_value, func); if ((int)used_stack <= checked_stack) { stack_check = 0; } used_stack_ref = ir_CONST_ADDR(used_stack); } else { ir_ref num_args_ref; ir_ref if_internal_func = IR_UNUSED; const size_t func_type_offset = is_closure ? offsetof(zend_closure, func.type) : offsetof(zend_function, type); used_stack = (ZEND_CALL_FRAME_SLOT + opline->extended_value + ZEND_OBSERVER_ENABLED) * sizeof(zval); used_stack_ref = ir_CONST_ADDR(used_stack); used_stack_ref = ir_HARD_COPY_A(used_stack_ref); /* load constant once */ // JIT: if (EXPECTED(ZEND_USER_CODE(func->type))) { ir_ref tmp = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, func_type_offset)); if_internal_func = ir_IF(ir_AND_U8(tmp, ir_CONST_U8(1))); ir_IF_FALSE(if_internal_func); // JIT: used_stack += (func->op_array.last_var + func->op_array.T - MIN(func->op_array.num_args, num_args)) * sizeof(zval); num_args_ref = ir_CONST_U32(opline->extended_value); if (!is_closure) { ref = ir_SUB_U32( ir_SUB_U32( ir_MIN_U32( num_args_ref, ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.num_args)))), ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.last_var)))), ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, op_array.T)))); } else { ref = ir_SUB_U32( ir_SUB_U32( ir_MIN_U32( num_args_ref, ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.num_args)))), ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.last_var)))), ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.op_array.T)))); } ref = ir_MUL_U32(ref, ir_CONST_U32(sizeof(zval))); if (sizeof(void*) == 8) { ref = ir_SEXT_A(ref); } ref = ir_SUB_A(used_stack_ref, ref); ir_MERGE_WITH_EMPTY_TRUE(if_internal_func); used_stack_ref = ir_PHI_2(IR_ADDR, ref, used_stack_ref); } zend_jit_start_reuse_ip(jit); // JIT: if (UNEXPECTED(used_stack > (size_t)(((char*)EG(vm_stack_end)) - (char*)call))) { jit_STORE_IP(jit, ir_LOAD_A(jit_EG(vm_stack_top))); if (stack_check) { // JIT: Check Stack Overflow ref = ir_UGE( ir_SUB_A( ir_LOAD_A(jit_EG(vm_stack_end)), jit_IP(jit)), used_stack_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { bool may_be_trampoline = !func && (opline->opcode == ZEND_INIT_METHOD_CALL); int32_t exit_point = zend_jit_trace_get_exit_point(opline, may_be_trampoline ? (ZEND_JIT_EXIT_TO_VM | ZEND_JIT_EXIT_METHOD_CALL) : ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } if (may_be_trampoline) { jit->trace->exit_info[exit_point].poly_func.ref = func_ref; jit->trace->exit_info[exit_point].poly_this.ref = this_ref; } ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { if_enough_stack = ir_IF(ref); ir_IF_FALSE_cold(if_enough_stack); #ifdef _WIN32 if (0) { #else if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { #endif jit_SET_EX_OPLINE(jit, opline); ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_int_extend_stack_helper), used_stack_ref); } else { if (!is_closure) { ref = func_ref; } else { ref = ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func)); } jit_SET_EX_OPLINE(jit, opline); ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_extend_stack_helper), used_stack_ref, ref); } jit_STORE_IP(jit, ref); cold_path = ir_END(); ir_IF_TRUE(if_enough_stack); } } ref = jit_EG(vm_stack_top); rx = jit_IP(jit); #if !OPTIMIZE_FOR_SIZE /* JIT: EG(vm_stack_top) = (zval*)((char*)call + used_stack); * This vesions is longer but faster * mov EG(vm_stack_top), %CALL * lea size(%call), %tmp * mov %tmp, EG(vm_stack_top) */ top = rx; #else /* JIT: EG(vm_stack_top) += used_stack; * Use ir_emit() because ir_LOAD() makes load forwarding and doesn't allow load/store fusion * mov EG(vm_stack_top), %CALL * add $size, EG(vm_stack_top) */ top = jit->ctx.control = ir_emit2(&jit->ctx, IR_OPT(IR_LOAD, IR_ADDR), jit->ctx.control, ref); #endif ir_STORE(ref, ir_ADD_A(top, used_stack_ref)); // JIT: zend_vm_init_call_frame(call, call_info, func, num_args, called_scope, object); if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || opline->opcode != ZEND_INIT_METHOD_CALL) { // JIT: ZEND_SET_CALL_INFO(call, 0, call_info); ir_STORE(jit_CALL(rx, This.u1.type_info), ir_CONST_U32(IS_UNDEF | ZEND_CALL_NESTED_FUNCTION)); } #ifdef _WIN32 if (0) { #else if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { #endif if (cold_path) { ir_MERGE_WITH(cold_path); rx = jit_IP(jit); } // JIT: call->func = func; ir_STORE(jit_CALL(rx, func), func_ref); } else { if (!is_closure) { // JIT: call->func = func; ir_STORE(jit_CALL(rx, func), func_ref); } else { // JIT: call->func = &closure->func; ir_STORE(jit_CALL(rx, func), ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func))); } if (cold_path) { ir_MERGE_WITH(cold_path); rx = jit_IP(jit); } } if (opline->opcode == ZEND_INIT_METHOD_CALL) { // JIT: Z_PTR(call->This) = obj; ZEND_ASSERT(this_ref != IR_NULL); ir_STORE(jit_CALL(rx, This.value.ptr), this_ref); if (opline->op1_type == IS_UNUSED || delayed_fetch_this) { // JIT: call->call_info |= ZEND_CALL_HAS_THIS; ref = jit_CALL(rx, This.u1.type_info); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS)); } else { ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_HAS_THIS))); } } else { if (opline->op1_type == IS_CV) { // JIT: GC_ADDREF(obj); jit_GC_ADDREF(jit, this_ref); } // JIT: call->call_info |= ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS; ref = jit_CALL(rx, This.u1.type_info); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { ir_STORE(ref, ir_CONST_U32( ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS)); } else { ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_HAS_THIS | ZEND_CALL_RELEASE_THIS))); } } } else if (opline->opcode == ZEND_INIT_STATIC_METHOD_CALL) { // JIT: Z_CE(call->This) = called_scope; ir_STORE(jit_CALL(rx, This), this_ref); } else if (!is_closure) { // JIT: Z_CE(call->This) = called_scope; ir_STORE(jit_CALL(rx, This), IR_NULL); } else { ir_ref object_or_called_scope, call_info, call_info2, object, if_cond; ir_ref if_cond_user = IR_UNUSED; if (opline->op2_type == IS_CV) { // JIT: GC_ADDREF(closure); jit_GC_ADDREF(jit, func_ref); } // JIT: RX(object_or_called_scope) = closure->called_scope; object_or_called_scope = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, called_scope))); // JIT: call_info = ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE | // (closure->func->common.fn_flags & ZEND_ACC_FAKE_CLOSURE); call_info = ir_OR_U32( ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.common.fn_flags))), ir_CONST_U32(ZEND_ACC_FAKE_CLOSURE)), ir_CONST_U32(ZEND_CALL_NESTED_FUNCTION | ZEND_CALL_DYNAMIC | ZEND_CALL_CLOSURE)); // JIT: if (Z_TYPE(closure->this_ptr) != IS_UNDEF) { if_cond = ir_IF(ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.u1.v.type)))); ir_IF_TRUE(if_cond); // JIT: call_info |= ZEND_CALL_HAS_THIS; call_info2 = ir_OR_U32(call_info, ir_CONST_U32(ZEND_CALL_HAS_THIS)); // JIT: object_or_called_scope = Z_OBJ(closure->this_ptr); object = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, this_ptr.value.ptr))); ir_MERGE_WITH_EMPTY_FALSE(if_cond); call_info = ir_PHI_2(IR_U32, call_info2, call_info); object_or_called_scope = ir_PHI_2(IR_ADDR, object, object_or_called_scope); // JIT: ZEND_SET_CALL_INFO(call, 0, call_info); ref = jit_CALL(rx, This.u1.type_info); ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), call_info)); // JIT: Z_PTR(call->This) = object_or_called_scope; ir_STORE(jit_CALL(rx, This.value.ptr), object_or_called_scope); if (!func) { // JIT: if (closure->func.common.type & ZEND_USER_FUNCTION) ir_ref type = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func.type))); if_cond_user = ir_IF(ir_AND_U8(type, ir_CONST_U8(ZEND_USER_FUNCTION))); ir_IF_TRUE(if_cond_user); } if (!func || func->common.type == ZEND_USER_FUNCTION) { // JIT: zend_jit_init_func_run_time_cache_helper(closure->func); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper), ir_ADD_OFFSET(func_ref, offsetof(zend_closure, func))); } if (!func) { ir_MERGE_WITH_EMPTY_FALSE(if_cond_user); } } // JIT: ZEND_CALL_NUM_ARGS(call) = num_args; ir_STORE(jit_CALL(rx, This.u2.num_args), ir_CONST_U32(opline->extended_value)); return 1; } static int zend_jit_func_guard(zend_jit_ctx *jit, ir_ref func_ref, const zend_function *func, const void *exit_addr) { if (func->type == ZEND_USER_FUNCTION && (!(func->common.fn_flags & ZEND_ACC_IMMUTABLE) || (func->common.fn_flags & ZEND_ACC_CLOSURE) || !func->common.function_name)) { const zend_op *opcodes = func->op_array.opcodes; // JIT: if (call->func.op_array.opcodes != opcodes) goto exit_addr; ir_GUARD( ir_EQ( ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))), ir_CONST_ADDR(opcodes)), ir_CONST_ADDR(exit_addr)); #ifdef ZEND_WIN32 } else if (func->type == ZEND_INTERNAL_FUNCTION) { // ASLR may cause different addresses in different workers. Check for the internal function handler. // JIT: if (call->func.internal_function.handler != handler) goto exit_addr; ir_GUARD( ir_EQ( ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_internal_function, handler))), ir_CONST_FC_FUNC(func->internal_function.handler)), ir_CONST_ADDR(exit_addr)); #endif } else { // JIT: if (call->func != func) goto exit_addr; ir_GUARD(ir_EQ(func_ref, ir_CONST_ADDR(func)), ir_CONST_ADDR(exit_addr)); } return 1; } static int zend_jit_init_fcall_guard(zend_jit_ctx *jit, uint32_t level, const zend_function *func, const zend_op *to_opline) { int32_t exit_point; const void *exit_addr; ir_ref call; if (func->type == ZEND_USER_FUNCTION && !zend_accel_in_shm(func->op_array.opcodes)) { /* op_array and op_array->opcodes are not persistent. We can't link. */ return 0; } exit_point = zend_jit_trace_get_exit_point(to_opline, ZEND_JIT_EXIT_POLYMORPHISM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } // call = EX(call); call = ir_LOAD_A(jit_EX(call)); while (level > 0) { // call = call->prev_execute_data call = ir_LOAD_A(jit_CALL(call, prev_execute_data)); level--; } return zend_jit_func_guard(jit, ir_LOAD_A(jit_CALL(call, func)), func, exit_addr); } static int zend_jit_init_fcall(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; ir_ref func_ref = IR_UNUSED; if (jit->delayed_call_level) { if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } if (info) { call_info = info->callee_info; while (call_info && call_info->caller_init_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } } if (!func && trace && trace->op == ZEND_JIT_TRACE_INIT_CALL) { #ifdef _WIN32 /* ASLR */ if (trace->func->type != ZEND_INTERNAL_FUNCTION) { func = (zend_function*)trace->func; } #else func = (zend_function*)trace->func; #endif } #ifdef _WIN32 if (0) { #else if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_INTERNAL_FUNCTION) { #endif /* load constant address later */ func_ref = ir_CONST_ADDR(func); } else if (func && op_array == &func->op_array) { /* recursive call */ if (!(func->op_array.fn_flags & ZEND_ACC_IMMUTABLE) || zend_jit_prefer_const_addr_load(jit, (uintptr_t)func)) { func_ref = ir_LOAD_A(jit_EX(func)); } else { func_ref = ir_CONST_ADDR(func); } } else { ir_ref if_func, cache_slot_ref, ref; // JIT: if (CACHED_PTR(opline->result.num)) cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num); func_ref = ir_LOAD_A(cache_slot_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) && opline->opcode != ZEND_INIT_FCALL) { /* Called func may be changed because of recompilation. See ext/opcache/tests/jit/init_fcall_003.phpt */ if_func = ir_IF(ir_EQ(func_ref, ir_CONST_ADDR(func))); } else { if_func = ir_IF(func_ref); } ir_IF_FALSE_cold(if_func); if (opline->opcode == ZEND_INIT_FCALL && func && func->type == ZEND_USER_FUNCTION && (func->op_array.fn_flags & ZEND_ACC_IMMUTABLE)) { ref = ir_HARD_COPY_A(ir_CONST_ADDR(func)); /* load constant once */ ir_STORE(cache_slot_ref, ref); ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_init_func_run_time_cache_helper), ref); } else { zval *zv = RT_CONSTANT(opline, opline->op2); if (opline->opcode == ZEND_INIT_FCALL) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper), ir_CONST_ADDR(Z_STR_P(zv)), cache_slot_ref); } else if (opline->opcode == ZEND_INIT_FCALL_BY_NAME) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_func_helper), ir_CONST_ADDR(Z_STR_P(zv + 1)), cache_slot_ref); } else if (opline->opcode == ZEND_INIT_NS_FCALL_BY_NAME) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_ns_func_helper), ir_CONST_ADDR(zv), cache_slot_ref); } else { ZEND_UNREACHABLE(); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, func && (func->common.fn_flags & ZEND_ACC_IMMUTABLE) ? ZEND_JIT_EXIT_INVALIDATE : 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } if (!func || opline->opcode == ZEND_INIT_FCALL) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else if (!zend_jit_func_guard(jit, ref, func, exit_addr)) { return 0; } } else { jit_SET_EX_OPLINE(jit, opline); ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_undefined_function)); } } ir_MERGE_WITH_EMPTY_TRUE(if_func); func_ref = ir_PHI_2(IR_ADDR, ref, func_ref); } if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, IR_UNUSED)) { return 0; } if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { if (!zend_jit_save_call_chain(jit, call_level)) { return 0; } } else { ZEND_ASSERT(call_level > 0); jit->delayed_call_level = call_level; delayed_call_chain = 1; } if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { if (!zend_jit_set_ip(jit, opline + 1)) { return 0; } } return 1; } static int zend_jit_init_method_call(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, uint32_t op1_info, zend_jit_addr op1_addr, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, zend_class_entry *trace_ce, zend_jit_trace_rec *trace, int checked_stack, ir_ref func_ref, ir_ref this_ref, bool polymorphic_side_trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_function *func = NULL; zval *function_name; ir_ref if_static = IR_UNUSED, cold_path; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); function_name = RT_CONSTANT(opline, opline->op2); if (info) { call_info = info->callee_info; while (call_info && call_info->caller_init_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } } if (polymorphic_side_trace) { /* function is passed from parent snapshot */ ZEND_ASSERT(func_ref != IR_UNUSED && this_ref != IR_UNUSED); } else { ir_ref ref, ref2, if_found, fast_path, run_time_cache, this_ref2; if (on_this) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); this_ref = jit_Z_PTR(jit, this_addr); } else { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { // JIT: ZVAL_DEREF(op1) ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } else { ir_ref if_ref; /* Hack: Convert reference to regular value to simplify JIT code */ ZEND_ASSERT(Z_REG(op1_addr) == ZREG_FP); if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE); ir_IF_TRUE(if_ref); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper), jit_ZVAL_ADDR(jit, op1_addr)); ir_MERGE_WITH_EMPTY_FALSE(if_ref); } } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_OBJECT)), ir_CONST_ADDR(exit_addr)); } else { ir_ref if_object = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT); ir_IF_FALSE_cold(if_object); jit_SET_EX_OPLINE(jit, opline); if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call_tmp), jit_ZVAL_ADDR(jit, op1_addr)); } else { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_method_call), jit_ZVAL_ADDR(jit, op1_addr)); } ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_IF_TRUE(if_object); } } this_ref = jit_Z_PTR(jit, op1_addr); } if (jit->delayed_call_level) { if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } if (func) { // JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*)); ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num + sizeof(void*))); if_found = ir_IF(ref); ir_IF_TRUE(if_found); fast_path = ir_END(); } else { // JIT: if (CACHED_PTR(opline->result.num) == obj->ce)) { run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); ref = ir_EQ( ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num)), ir_LOAD_A(ir_ADD_OFFSET(this_ref, offsetof(zend_object, ce)))); if_found = ir_IF(ref); ir_IF_TRUE(if_found); // JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*)); ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->result.num + sizeof(void*))); fast_path = ir_END(); } ir_IF_FALSE_cold(if_found); jit_SET_EX_OPLINE(jit, opline); if (!jit->ctx.fixed_call_stack_size) { // JIT: alloca(sizeof(void*)); this_ref2 = ir_ALLOCA(ir_CONST_ADDR(0x10)); } else { #ifdef _WIN64 this_ref2 = ir_HARD_COPY_A(jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS)); #else this_ref2 = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP)); #endif } ir_STORE(this_ref2, this_ref); if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_tmp_helper), this_ref, ir_CONST_ADDR(function_name), this_ref2); } else { ref2 = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_method_helper), this_ref, ir_CONST_ADDR(function_name), this_ref2); } if (!jit->ctx.fixed_call_stack_size) { this_ref2 = ir_LOAD_A(ir_RLOAD_A(IR_REG_SP)); // JIT: revert alloca ir_AFREE(ir_CONST_ADDR(0x10)); } else { #ifdef _WIN64 this_ref2 = ir_LOAD_A(jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS)); #else this_ref2 = ir_LOAD_A(ir_RLOAD_A(IR_REG_SP)); #endif } ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_MERGE_WITH(fast_path); func_ref = ir_PHI_2(IR_ADDR, ref2, ref); this_ref = ir_PHI_2(IR_ADDR, this_ref2, this_ref); } if ((!func || zend_jit_may_be_modified(func, op_array)) && trace && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func) { int32_t exit_point; const void *exit_addr; exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : ZEND_JIT_EXIT_METHOD_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit->trace->exit_info[exit_point].poly_func.ref = func_ref; jit->trace->exit_info[exit_point].poly_this.ref = this_ref; func = (zend_function*)trace->func; if (!zend_jit_func_guard(jit, func_ref, func, exit_addr)) { return 0; } } if (!func) { // JIT: if (fbc->common.fn_flags & ZEND_ACC_STATIC) { if_static = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))), ir_CONST_U32(ZEND_ACC_STATIC))); ir_IF_TRUE_cold(if_static); } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) != 0) { ir_ref ret; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this) { ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame_tmp), this_ref, func_ref, ir_CONST_U32(opline->extended_value)); } else { ret = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_static_method_call_frame), this_ref, func_ref, ir_CONST_U32(opline->extended_value)); } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR) && !delayed_fetch_this)) { ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } jit_STORE_IP(jit, ret); } if (!func) { cold_path = ir_END(); ir_IF_FALSE(if_static); } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC) == 0) { if (!zend_jit_push_call_frame(jit, opline, NULL, func, 0, delayed_fetch_this, checked_stack, func_ref, this_ref)) { return 0; } } if (!func) { ir_MERGE_WITH(cold_path); } zend_jit_start_reuse_ip(jit); if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { if (!zend_jit_save_call_chain(jit, call_level)) { return 0; } } else { ZEND_ASSERT(call_level > 0); delayed_call_chain = 1; jit->delayed_call_level = call_level; } if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { if (!zend_jit_set_ip(jit, opline + 1)) { return 0; } } return 1; } static int zend_jit_init_static_method_call(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; zend_class_entry *ce; zend_function *func = NULL; ir_ref func_ref, func_ref2, scope_ref, scope_ref2, if_cached, cold_path, ref; ir_ref if_static = IR_UNUSED; if (info) { call_info = info->callee_info; while (call_info && call_info->caller_init_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } } ce = zend_get_known_class(op_array, opline, opline->op1_type, opline->op1); if (!func && ce && (opline->op1_type == IS_CONST || !(ce->ce_flags & ZEND_ACC_TRAIT))) { zval *zv = RT_CONSTANT(opline, opline->op2); zend_string *method_name; ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); method_name = Z_STR_P(zv); zv = zend_hash_find(&ce->function_table, method_name); if (zv) { zend_function *fn = Z_PTR_P(zv); if (fn->common.scope == op_array->scope || (fn->common.fn_flags & ZEND_ACC_PUBLIC) || ((fn->common.fn_flags & ZEND_ACC_PROTECTED) && op_array->scope && instanceof_function_slow(op_array->scope, fn->common.scope))) { func = fn; } } } if (jit->delayed_call_level) { if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } // JIT: fbc = CACHED_PTR(opline->result.num + sizeof(void*)); func_ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num + sizeof(void*))); // JIT: if (fbc) if_cached = ir_IF(func_ref); ir_IF_FALSE_cold(if_cached); jit_SET_EX_OPLINE(jit, opline); scope_ref2 = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_class_helper), jit_FP(jit)); ir_GUARD(scope_ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler)); func_ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_find_static_method_helper), jit_FP(jit), scope_ref2); ir_GUARD(func_ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler)); cold_path = ir_END(); ir_IF_TRUE(if_cached); if (ce && (ce->ce_flags & ZEND_ACC_IMMUTABLE) && (ce->ce_flags & ZEND_ACC_LINKED)) { scope_ref = ir_CONST_ADDR(ce); } else { scope_ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->result.num)); } ir_MERGE_2(cold_path, ir_END()); func_ref = ir_PHI_2(IR_ADDR, func_ref2, func_ref); scope_ref = ir_PHI_2(IR_ADDR, scope_ref2, scope_ref); if ((!func || zend_jit_may_be_modified(func, op_array)) && trace && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func) { int32_t exit_point; const void *exit_addr; exit_point = zend_jit_trace_get_exit_point(opline, func ? ZEND_JIT_EXIT_INVALIDATE : 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } // jit->trace->exit_info[exit_point].poly_func_ref = func_ref; // jit->trace->exit_info[exit_point].poly_this_ref = scope_ref; func = (zend_function*)trace->func; if (!zend_jit_func_guard(jit, func_ref, func, exit_addr)) { return 0; } } if (!func || !(func->common.fn_flags & ZEND_ACC_STATIC)) { if (!func) { // JIT: if (fbc->common.fn_flags & ZEND_ACC_STATIC) { if_static = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_function, common.fn_flags))), ir_CONST_U32(ZEND_ACC_STATIC))); ir_IF_FALSE_cold(if_static); } jit_SET_EX_OPLINE(jit, opline); ref = ir_CALL_3(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_push_this_method_call_frame), scope_ref, func_ref, ir_CONST_U32(opline->extended_value)); ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler)); jit_STORE_IP(jit, ref); if (!func) { cold_path = ir_END(); ir_IF_TRUE(if_static); } } if (!func || (func->common.fn_flags & ZEND_ACC_STATIC)) { if (opline->op1_type == IS_UNUSED && ((opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_PARENT || (opline->op1.num & ZEND_FETCH_CLASS_MASK) == ZEND_FETCH_CLASS_SELF)) { if (op_array->fn_flags & ZEND_ACC_STATIC) { scope_ref = ir_LOAD_A(jit_EX(This.value.ref)); } else if (op_array->fn_flags & ZEND_ACC_CLOSURE) { ir_ref if_object, values = IR_UNUSED; if_object = ir_IF(ir_EQ(jit_Z_TYPE_ref(jit, jit_EX(This)), ir_CONST_U8(IS_OBJECT))); ir_IF_TRUE(if_object); ir_END_PHI_list(values, ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(This.value.ref)), offsetof(zend_object, ce)))); ir_IF_FALSE(if_object); ir_END_PHI_list(values, ir_LOAD_A(jit_EX(This.value.ref))); ir_PHI_list(values); } else { scope_ref = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(This.value.ref)), offsetof(zend_object, ce))); } } if (!zend_jit_push_call_frame(jit, opline, op_array, func, 0, 0, checked_stack, func_ref, scope_ref)) { return 0; } if (!func) { ir_MERGE_2(cold_path, ir_END()); } } zend_jit_start_reuse_ip(jit); if (zend_jit_needs_call_chain(call_info, b, op_array, ssa, ssa_op, opline, call_level, trace)) { if (!zend_jit_save_call_chain(jit, call_level)) { return 0; } } else { ZEND_ASSERT(call_level > 0); jit->delayed_call_level = call_level; delayed_call_chain = 1; } if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { if (!zend_jit_set_ip(jit, opline + 1)) { return 0; } } return 1; } static int zend_jit_init_closure_call(zend_jit_ctx *jit, const zend_op *opline, uint32_t b, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, int call_level, zend_jit_trace_rec *trace, int checked_stack) { zend_function *func = NULL; zend_jit_addr op2_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); ir_ref ref; ref = jit_Z_PTR(jit, op2_addr); if (ssa->var_info[ssa_op->op2_use].ce != zend_ce_closure && !(ssa->var_info[ssa_op->op2_use].type & MAY_BE_CLASS_GUARD)) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD( ir_EQ( ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_object, ce))), ir_CONST_ADDR(zend_ce_closure)), ir_CONST_ADDR(exit_addr)); if (ssa->var_info && ssa_op->op2_use >= 0) { ssa->var_info[ssa_op->op2_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op2_use].ce = zend_ce_closure; ssa->var_info[ssa_op->op2_use].is_instanceof = 0; } } if (trace && trace->op == ZEND_JIT_TRACE_INIT_CALL && trace->func && trace->func->type == ZEND_USER_FUNCTION) { const zend_op *opcodes; int32_t exit_point; const void *exit_addr; func = (zend_function*)trace->func; opcodes = func->op_array.opcodes; exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_CLOSURE_CALL); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD( ir_EQ( ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_closure, func.op_array.opcodes))), ir_CONST_ADDR(opcodes)), ir_CONST_ADDR(exit_addr)); } if (jit->delayed_call_level) { if (!zend_jit_save_call_chain(jit, jit->delayed_call_level)) { return 0; } } if (!zend_jit_push_call_frame(jit, opline, NULL, func, 1, 0, checked_stack, ref, IR_UNUSED)) { return 0; } if (zend_jit_needs_call_chain(NULL, b, op_array, ssa, ssa_op, opline, call_level, trace)) { if (!zend_jit_save_call_chain(jit, call_level)) { return 0; } } else { ZEND_ASSERT(call_level > 0); delayed_call_chain = 1; jit->delayed_call_level = call_level; } if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { if (!zend_jit_set_ip(jit, opline + 1)) { return 0; } } return 1; } static int zend_jit_send_val(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr) { uint32_t arg_num = opline->op2.num; zend_jit_addr arg_addr; ZEND_ASSERT(opline->opcode == ZEND_SEND_VAL || arg_num <= MAX_ARG_FLAG_NUM); if (!zend_jit_reuse_ip(jit)) { return 0; } if (opline->opcode == ZEND_SEND_VAL_EX) { uint32_t mask = ZEND_SEND_BY_REF << ((arg_num + 3) * 2); ZEND_ASSERT(arg_num <= MAX_ARG_FLAG_NUM); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_MUST_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { /* Don't generate code that always throws exception */ return 0; } } else { ir_ref cond = ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask)); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr)); } else { ir_ref if_pass_by_ref; if_pass_by_ref = ir_IF(cond); ir_IF_TRUE_cold(if_pass_by_ref); if (Z_MODE(op1_addr) == IS_REG) { /* set type to avoid zval_ptr_dtor() on uninitialized value */ zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); jit_set_Z_TYPE_INFO(jit, addr, IS_UNDEF); } jit_SET_EX_OPLINE(jit, opline); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_throw_cannot_pass_by_ref)); ir_IF_FALSE(if_pass_by_ref); } } } arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); jit_ZVAL_COPY_CONST(jit, arg_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1); } else { jit_ZVAL_COPY(jit, arg_addr, MAY_BE_ANY, op1_addr, op1_info, 0); } return 1; } static int zend_jit_send_ref(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, int cold) { zend_jit_addr op1_addr, arg_addr, ref_addr; ir_ref ref_path = IR_UNUSED; op1_addr = OP1_ADDR(); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(jit)) { return 0; } if (opline->op1_type == IS_VAR) { if (op1_info & MAY_BE_INDIRECT) { op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr); } } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_UNDEF) { if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { // JIT: if (Z_TYPE_P(op1) == IS_UNDEF) ir_ref if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); ir_IF_FALSE(if_def); // JIT: ZVAL_NULL(op1) jit_set_Z_TYPE_INFO(jit,op1_addr, IS_NULL); ir_MERGE_WITH_EMPTY_TRUE(if_def); } op1_info &= ~MAY_BE_UNDEF; op1_info |= MAY_BE_NULL; } } else { ZEND_UNREACHABLE(); } if (op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) { ir_ref ref, ref2; if (op1_info & MAY_BE_REF) { ir_ref if_ref; // JIT: if (Z_TYPE_P(op1) == IS_UNDEF) if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE); ir_IF_TRUE(if_ref); // JIT: ref = Z_PTR_P(op1) ref = jit_Z_PTR(jit, op1_addr); // JIT: GC_ADDREF(ref) jit_GC_ADDREF(jit, ref); // JIT: ZVAL_REFERENCE(arg, ref) jit_set_Z_PTR(jit, arg_addr, ref); jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX); ref_path = ir_END(); ir_IF_FALSE(if_ref); } // JIT: ZVAL_NEW_REF(arg, varptr); // JIT: ref = emalloc(sizeof(zend_reference)); ref = jit_EMALLOC(jit, sizeof(zend_reference), op_array, opline); // JIT: GC_REFCOUNT(ref) = 2 jit_set_GC_REFCOUNT(jit, ref, 2); // JIT: GC_TYPE(ref) = GC_REFERENCE ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, gc.u.type_info)), ir_CONST_U32(GC_REFERENCE)); ir_STORE(ir_ADD_OFFSET(ref, offsetof(zend_reference, sources.ptr)), IR_NULL); ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val)); ref_addr = ZEND_ADDR_REF_ZVAL(ref2); // JIT: ZVAL_COPY_VALUE(&ref->val, op1) jit_ZVAL_COPY(jit, ref_addr, MAY_BE_ANY, op1_addr, op1_info, 0); // JIT: ZVAL_REFERENCE(arg, ref) jit_set_Z_PTR(jit, op1_addr, ref); jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX); // JIT: ZVAL_REFERENCE(arg, ref) jit_set_Z_PTR(jit, arg_addr, ref); jit_set_Z_TYPE_INFO(jit, arg_addr, IS_REFERENCE_EX); } if (ref_path) { ir_MERGE_WITH(ref_path); } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); return 1; } static int zend_jit_send_var(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr op1_def_addr) { uint32_t arg_num = opline->op2.num; zend_jit_addr arg_addr; ir_ref end_inputs = IR_UNUSED; ZEND_ASSERT((opline->opcode != ZEND_SEND_VAR_EX && opline->opcode != ZEND_SEND_VAR_NO_REF_EX) || arg_num <= MAX_ARG_FLAG_NUM); arg_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, opline->result.var); if (!zend_jit_reuse_ip(jit)) { return 0; } if (opline->opcode == ZEND_SEND_VAR_EX) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) { return 0; } return 1; } } else { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); // JIT: if (RX->func->quick_arg_flags & mask) ir_ref if_send_by_ref = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_RX(func)), offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask))); ir_IF_TRUE_cold(if_send_by_ref); if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) { return 0; } ir_END_list(end_inputs); ir_IF_FALSE(if_send_by_ref); } } else if (opline->opcode == ZEND_SEND_VAR_NO_REF_EX) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { // JIT: ZVAL_COPY_VALUE(arg, op1) jit_ZVAL_COPY(jit, arg_addr, MAY_BE_ANY, op1_addr, op1_info, 0); if (!ARG_MAY_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!(op1_info & MAY_BE_REF)) { /* Don't generate code that always throws exception */ return 0; } else { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } // JIT: if (Z_TYPE_P(op1) != IS_REFERENCE) ir_GUARD(ir_EQ(jit_Z_TYPE(jit, op1_addr), ir_CONST_U32(IS_REFERENCE)), ir_CONST_ADDR(exit_addr)); } } return 1; } } else { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); ir_ref func, if_send_by_ref, if_prefer_ref; // JIT: if (RX->func->quick_arg_flags & mask) func = ir_LOAD_A(jit_RX(func)); if_send_by_ref = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask))); ir_IF_TRUE_cold(if_send_by_ref); mask = ZEND_SEND_PREFER_REF << ((arg_num + 3) * 2); // JIT: ZVAL_COPY_VALUE(arg, op1) jit_ZVAL_COPY(jit, arg_addr, MAY_BE_ANY, op1_addr, op1_info, 0); if (op1_info & MAY_BE_REF) { ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE); ir_IF_TRUE(if_ref); ir_END_list(end_inputs); ir_IF_FALSE(if_ref); } // JIT: if (RX->func->quick_arg_flags & mask) if_prefer_ref = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func, offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask))); ir_IF_TRUE(if_prefer_ref); ir_END_list(end_inputs); ir_IF_FALSE(if_prefer_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else { jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference), jit_ZVAL_ADDR(jit, arg_addr)); zend_jit_check_exception(jit); ir_END_list(end_inputs); } ir_IF_FALSE(if_send_by_ref); } } else if (opline->opcode == ZEND_SEND_FUNC_ARG) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 0)) { return 0; } return 1; } } else { // JIT: if (RX->This.u1.type_info & ZEND_CALL_SEND_ARG_BY_REF) ir_ref if_send_by_ref = ir_IF(ir_AND_U32( ir_LOAD_U32(jit_RX(This.u1.type_info)), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF))); ir_IF_TRUE_cold(if_send_by_ref); if (!zend_jit_send_ref(jit, opline, op_array, op1_info, 1)) { return 0; } ir_END_list(end_inputs); ir_IF_FALSE(if_send_by_ref); } } if (op1_info & MAY_BE_UNDEF) { ir_ref ref, if_def = IR_UNUSED; if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { if_def = jit_if_not_Z_TYPE(jit, op1_addr, IS_UNDEF); ir_IF_FALSE_cold(if_def); } // JIT: zend_jit_undefined_op_helper(opline->op1.var) jit_SET_EX_OPLINE(jit, opline); ref = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var)); // JIT: ZVAL_NULL(arg) jit_set_Z_TYPE_INFO(jit, arg_addr, IS_NULL); // JIT: check_exception ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler)); if (op1_info & (MAY_BE_ANY|MAY_BE_REF)) { ir_END_list(end_inputs); ir_IF_TRUE(if_def); } else { if (end_inputs) { ir_END_list(end_inputs); ir_MERGE_list(end_inputs); } return 1; } } if (opline->opcode == ZEND_SEND_VAR_NO_REF) { // JIT: ZVAL_COPY_VALUE(arg, op1) jit_ZVAL_COPY(jit, arg_addr, MAY_BE_ANY, op1_addr, op1_info, 0); if (op1_info & MAY_BE_REF) { // JIT: if (Z_TYPE_P(arg) == IS_REFERENCE) ir_ref if_ref = jit_if_Z_TYPE(jit, arg_addr, IS_REFERENCE); ir_IF_TRUE(if_ref); ir_END_list(end_inputs); ir_IF_FALSE(if_ref); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(IR_FALSE, ir_CONST_ADDR(exit_addr)); } else { jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_only_vars_by_reference), jit_ZVAL_ADDR(jit, arg_addr)); zend_jit_check_exception(jit); } } else { if (op1_info & MAY_BE_REF) { if (opline->op1_type == IS_CV) { ir_ref ref; // JIT: ZVAL_DEREF(op1) ref = jit_ZVAL_ADDR(jit, op1_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); // JIT: ZVAL_COPY(arg, op1) jit_ZVAL_COPY(jit, arg_addr, MAY_BE_ANY, op1_addr, op1_info, 1); } else { ir_ref if_ref, ref, ref2, refcount, if_not_zero, if_refcounted; zend_jit_addr ref_addr; // JIT: if (Z_TYPE_P(op1) == IS_REFERENCE) if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE); ir_IF_TRUE_cold(if_ref); // JIT: ref = Z_COUNTED_P(op1); ref = jit_Z_PTR(jit, op1_addr); ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val)); ref_addr = ZEND_ADDR_REF_ZVAL(ref2); // JIT: ZVAL_COPY_VALUE(arg, op1); jit_ZVAL_COPY(jit, arg_addr, MAY_BE_ANY, ref_addr, op1_info, 0); // JIT: if (GC_DELREF(ref) != 0) refcount = jit_GC_DELREF(jit, ref); if_not_zero = ir_IF(refcount); ir_IF_TRUE(if_not_zero); // JIT: if (Z_REFCOUNTED_P(arg) if_refcounted = jit_if_REFCOUNTED(jit, arg_addr); ir_IF_TRUE(if_refcounted); // JIT: Z_ADDREF_P(arg) jit_GC_ADDREF(jit, jit_Z_PTR(jit, arg_addr)); ir_END_list(end_inputs); ir_IF_FALSE(if_refcounted); ir_END_list(end_inputs); ir_IF_FALSE(if_not_zero); // JIT: efree(ref) jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline); ir_END_list(end_inputs); ir_IF_FALSE(if_ref); // JIT: ZVAL_COPY_VALUE(arg, op1); jit_ZVAL_COPY(jit, arg_addr, MAY_BE_ANY, op1_addr, op1_info, 0); } } else { if (op1_addr != op1_def_addr) { if (!zend_jit_update_regs(jit, opline->op1.var, op1_addr, op1_def_addr, op1_info)) { return 0; } if (Z_MODE(op1_def_addr) == IS_REG && Z_MODE(op1_addr) != IS_REG) { op1_addr = op1_def_addr; } } // JIT: ZVAL_COPY_VALUE(arg, op1) jit_ZVAL_COPY(jit, arg_addr, MAY_BE_ANY, op1_addr, op1_info, opline->op1_type == IS_CV); } } if (end_inputs) { ir_END_list(end_inputs); ir_MERGE_list(end_inputs); } return 1; } static int zend_jit_check_func_arg(zend_jit_ctx *jit, const zend_op *opline) { uint32_t arg_num = opline->op2.num; ir_ref ref; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->call && JIT_G(current_frame)->call->func) { if (ARG_SHOULD_BE_SENT_BY_REF(JIT_G(current_frame)->call->func, arg_num)) { if (!TRACE_FRAME_IS_LAST_SEND_BY_REF(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_REF(JIT_G(current_frame)->call); // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); if (jit->reuse_ip) { ref = jit_IP(jit); } else { ref = ir_LOAD_A(jit_EX(call)); } ref = jit_CALL(ref, This.u1.type_info); ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF))); } } else { if (!TRACE_FRAME_IS_LAST_SEND_BY_VAL(JIT_G(current_frame)->call)) { TRACE_FRAME_SET_LAST_SEND_BY_VAL(JIT_G(current_frame)->call); // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); if (jit->reuse_ip) { ref = jit_IP(jit); } else { ref = ir_LOAD_A(jit_EX(call)); } ref = jit_CALL(ref, This.u1.type_info); ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF))); } } } else { // JIT: if (QUICK_ARG_SHOULD_BE_SENT_BY_REF(EX(call)->func, arg_num)) { uint32_t mask = (ZEND_SEND_BY_REF|ZEND_SEND_PREFER_REF) << ((arg_num + 3) * 2); ir_ref rx, if_ref, cold_path; if (!zend_jit_reuse_ip(jit)) { return 0; } rx = jit_IP(jit); ref = ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(ir_LOAD_A(jit_CALL(rx, func)), offsetof(zend_function, quick_arg_flags))), ir_CONST_U32(mask)); if_ref = ir_IF(ref); ir_IF_TRUE_cold(if_ref); // JIT: ZEND_ADD_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); ref = jit_CALL(rx, This.u1.type_info); ir_STORE(ref, ir_OR_U32(ir_LOAD_U32(ref), ir_CONST_U32(ZEND_CALL_SEND_ARG_BY_REF))); cold_path = ir_END(); ir_IF_FALSE(if_ref); // JIT: ZEND_DEL_CALL_FLAG(EX(call), ZEND_CALL_SEND_ARG_BY_REF); ref = jit_CALL(rx, This.u1.type_info); ir_STORE(ref, ir_AND_U32(ir_LOAD_U32(ref), ir_CONST_U32(~ZEND_CALL_SEND_ARG_BY_REF))); ir_MERGE_WITH(cold_path); } return 1; } static int zend_jit_check_undef_args(zend_jit_ctx *jit, const zend_op *opline) { ir_ref call, if_may_have_undef, ret; if (jit->reuse_ip) { call = jit_IP(jit); } else { call = ir_LOAD_A(jit_EX(call)); } if_may_have_undef = ir_IF(ir_AND_U8( ir_LOAD_U8(ir_ADD_OFFSET(call, offsetof(zend_execute_data, This.u1.type_info) + 3)), ir_CONST_U8(ZEND_CALL_MAY_HAVE_UNDEF >> 24))); ir_IF_TRUE_cold(if_may_have_undef); jit_SET_EX_OPLINE(jit, opline); ret = ir_CALL_1(IR_I32, ir_CONST_FC_FUNC(zend_handle_undef_args), call); ir_GUARD_NOT(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_MERGE_WITH_EMPTY_FALSE(if_may_have_undef); return 1; } static int zend_jit_do_fcall(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, unsigned int next_block, zend_jit_trace_rec *trace) { zend_func_info *info = ZEND_FUNC_INFO(op_array); zend_call_info *call_info = NULL; const zend_function *func = NULL; uint32_t i; uint32_t call_num_args = 0; bool unknown_num_args = 0; const void *exit_addr = NULL; const zend_op *prev_opline; ir_ref rx, func_ref = IR_UNUSED, if_user = IR_UNUSED, user_path = IR_UNUSED; prev_opline = opline - 1; while (prev_opline->opcode == ZEND_EXT_FCALL_BEGIN || prev_opline->opcode == ZEND_TICKS) { prev_opline--; } if (prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { unknown_num_args = 1; } if (info) { call_info = info->callee_info; while (call_info && call_info->caller_call_opline != opline) { call_info = call_info->next_callee; } if (call_info && call_info->callee_func && !call_info->is_prototype) { func = call_info->callee_func; } if ((op_array->fn_flags & ZEND_ACC_TRAIT_CLONE) && (!JIT_G(current_frame) || !JIT_G(current_frame)->call || !JIT_G(current_frame)->call->func)) { call_info = NULL; func = NULL; /* megamorphic call from trait */ } } if (!func) { /* resolve function at run time */ } else if (func->type == ZEND_USER_FUNCTION) { ZEND_ASSERT(opline->opcode != ZEND_DO_ICALL); call_num_args = call_info->num_args; } else if (func->type == ZEND_INTERNAL_FUNCTION) { ZEND_ASSERT(opline->opcode != ZEND_DO_UCALL); call_num_args = call_info->num_args; } else { ZEND_UNREACHABLE(); } if (trace && !func) { if (trace->op == ZEND_JIT_TRACE_DO_ICALL) { ZEND_ASSERT(!trace->func || trace->func->type == ZEND_INTERNAL_FUNCTION); #ifndef ZEND_WIN32 // TODO: ASLR may cause different addresses in different workers ??? func = trace->func; if (JIT_G(current_frame) && JIT_G(current_frame)->call && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); } else { unknown_num_args = 1; } #endif } else if (trace->op == ZEND_JIT_TRACE_ENTER) { ZEND_ASSERT(trace->func->type == ZEND_USER_FUNCTION); if (zend_accel_in_shm(trace->func->op_array.opcodes)) { func = trace->func; if (JIT_G(current_frame) && JIT_G(current_frame)->call && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call) >= 0) { call_num_args = TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)->call); } else { unknown_num_args = 1; } } } } bool may_have_extra_named_params = opline->extended_value == ZEND_FCALL_MAY_HAVE_EXTRA_NAMED_PARAMS && (!func || func->common.fn_flags & ZEND_ACC_VARIADIC); if (!jit->reuse_ip) { zend_jit_start_reuse_ip(jit); // JIT: call = EX(call); jit_STORE_IP(jit, ir_LOAD_A(jit_EX(call))); } rx = jit_IP(jit); zend_jit_stop_reuse_ip(jit); jit_SET_EX_OPLINE(jit, opline); if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { if (trace) { uint32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } func_ref = ir_LOAD_A(jit_CALL(rx, func)); ir_GUARD_NOT( ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))), ir_CONST_U32(ZEND_ACC_DEPRECATED|ZEND_ACC_NODISCARD)), ir_CONST_ADDR(exit_addr)); } } } if (!jit->delayed_call_level) { // JIT: EX(call) = call->prev_execute_data; ir_STORE(jit_EX(call), (call_level == 1) ? IR_NULL : ir_LOAD_A(jit_CALL(rx, prev_execute_data))); } delayed_call_chain = 0; jit->delayed_call_level = 0; // JIT: call->prev_execute_data = execute_data; ir_STORE(jit_CALL(rx, prev_execute_data), jit_FP(jit)); if (!func) { if (!func_ref) { func_ref = ir_LOAD_A(jit_CALL(rx, func)); } } if (opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME) { if (!func) { if (!trace) { ir_ref if_deprecated_nodiscard, ret; uint32_t no_discard = RETURN_VALUE_USED(opline) ? 0 : ZEND_ACC_NODISCARD; if_deprecated_nodiscard = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))), ir_CONST_U32(ZEND_ACC_DEPRECATED|no_discard))); ir_IF_TRUE_cold(if_deprecated_nodiscard); ir_ref helper = ir_CONST_FC_FUNC(no_discard ? zend_jit_deprecated_nodiscard_helper : zend_jit_deprecated_helper); if (GCC_GLOBAL_REGS) { ret = ir_CALL(IR_BOOL, helper); } else { ret = ir_CALL_1(IR_BOOL, helper, rx); } ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_MERGE_WITH_EMPTY_FALSE(if_deprecated_nodiscard); } } else { if (func->common.fn_flags & ZEND_ACC_DEPRECATED) { ir_ref ret; if (GCC_GLOBAL_REGS) { ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper)); } else { ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_deprecated_helper), rx); } ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } if ((func->common.fn_flags & ZEND_ACC_NODISCARD) && !RETURN_VALUE_USED(opline)) { ir_ref ret; if (GCC_GLOBAL_REGS) { ret = ir_CALL(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_nodiscard_helper)); } else { ret = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_nodiscard_helper), rx); } ir_GUARD(ret, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } } } if (!func && opline->opcode != ZEND_DO_UCALL && opline->opcode != ZEND_DO_ICALL) { ir_ref type_ref = ir_LOAD_U8(ir_ADD_OFFSET(func_ref, offsetof(zend_function, type))); if_user = ir_IF(ir_EQ(type_ref, ir_CONST_U8(ZEND_USER_FUNCTION))); ir_IF_TRUE(if_user); } if ((!func || func->type == ZEND_USER_FUNCTION) && opline->opcode != ZEND_DO_ICALL) { bool recursive_call_through_jmp = 0; uint32_t num_args = 0; // JIT: EX(call) = NULL; ir_STORE(jit_CALL(rx, call), IR_NULL); // JIT: EX(return_value) = RETURN_VALUE_USED(opline) ? EX_VAR(opline->result.var) : 0; ir_STORE(jit_CALL(rx, return_value), RETURN_VALUE_USED(opline) ? jit_ZVAL_ADDR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var)) : IR_NULL); // JIT: EX_LOAD_RUN_TIME_CACHE(op_array); if (!func || func->op_array.cache_size) { ir_ref run_time_cache; if (func && op_array == &func->op_array) { /* recursive call */ run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); } else if (func && !(func->op_array.fn_flags & ZEND_ACC_CLOSURE) && ZEND_MAP_PTR_IS_OFFSET(func->op_array.run_time_cache)) { run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_CG(map_ptr_base)), (uintptr_t)ZEND_MAP_PTR(func->op_array.run_time_cache))); } else if ((func && (func->op_array.fn_flags & ZEND_ACC_CLOSURE)) || (JIT_G(current_frame) && JIT_G(current_frame)->call && TRACE_FRAME_IS_CLOSURE_CALL(JIT_G(current_frame)->call))) { /* Closures always use direct pointers */ ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func)); run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr))); } else { ir_ref if_odd, run_time_cache2; ir_ref local_func_ref = func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func)); run_time_cache = ir_LOAD_A(ir_ADD_OFFSET(local_func_ref, offsetof(zend_op_array, run_time_cache__ptr))); if_odd = ir_IF(ir_AND_A(run_time_cache, ir_CONST_ADDR(1))); ir_IF_TRUE(if_odd); run_time_cache2 = ir_LOAD_A(ir_ADD_A(run_time_cache, ir_LOAD_A(jit_CG(map_ptr_base)))); ir_MERGE_WITH_EMPTY_FALSE(if_odd); run_time_cache = ir_PHI_2(IR_ADDR, run_time_cache2, run_time_cache); } ir_STORE(jit_CALL(rx, run_time_cache), run_time_cache); } // JIT: EG(current_execute_data) = execute_data = call; ir_STORE(jit_EG(current_execute_data), rx); jit_STORE_FP(jit, rx); // JIT: opline = op_array->opcodes; if (func && !unknown_num_args) { for (i = call_num_args; i < func->op_array.last_var; i++) { uint32_t n = EX_NUM_TO_VAR(i); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, n); jit_set_Z_TYPE_INFO_ex(jit, var_addr, ir_CONST_U32(IS_UNDEF)); } if (call_num_args <= func->op_array.num_args) { if (!trace || (trace->op == ZEND_JIT_TRACE_END && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) { if ((func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) != 0) { if (trace) { num_args = 0; } else if (call_info) { num_args = skip_valid_arguments(op_array, ssa, call_info); } else { num_args = call_num_args; } } else { num_args = call_num_args; } if (zend_accel_in_shm(func->op_array.opcodes)) { jit_LOAD_IP_ADDR(jit, func->op_array.opcodes + num_args); } else { if (!func_ref) { func_ref = ir_LOAD_A(jit_CALL(rx, func)); } ir_ref ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))); if (num_args) { ip = ir_ADD_OFFSET(ip, num_args * sizeof(zend_op)); } jit_STORE_IP(jit, ip); } if (!trace && op_array == &func->op_array && call_num_args >= op_array->required_num_args) { /* recursive call */ recursive_call_through_jmp = 1; } } } else { ir_ref helper; if (!trace || (trace->op == ZEND_JIT_TRACE_END && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER)) { ir_ref ip; if (zend_accel_in_shm(func->op_array.opcodes)) { ip = ir_CONST_ADDR(func->op_array.opcodes); } else { if (!func_ref) { func_ref = ir_LOAD_A(jit_CALL(rx, func)); } ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))); } jit_STORE_IP(jit, ip); helper = ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper); } else { helper = ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper_no_skip_recv); } if (GCC_GLOBAL_REGS) { ir_CALL(IR_VOID, helper); } else if (ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { ir_CALL_2(IR_ADDR, helper, jit_FP(jit), jit_IP(jit)); } else { ir_ref ref = ir_CALL_2(IR_ADDR, helper, jit_FP(jit), jit_IP(jit)); jit_STORE_IP(jit, ref); } } } else { ir_ref ip; ir_ref merge_inputs = IR_UNUSED; // JIT: opline = op_array->opcodes if (func && zend_accel_in_shm(func->op_array.opcodes)) { ip = ir_CONST_ADDR(func->op_array.opcodes); } else { if (!func_ref) { func_ref = ir_LOAD_A(jit_CALL(rx, func)); } ip = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, opcodes))); } jit_STORE_IP(jit, ip); // JIT: num_args = EX_NUM_ARGS(); ir_ref num_args, first_extra_arg; num_args = ir_LOAD_U32(jit_EX(This.u2.num_args)); if (func) { first_extra_arg = ir_CONST_U32(func->op_array.num_args); } else { // JIT: first_extra_arg = op_array->num_args; ZEND_ASSERT(func_ref); first_extra_arg = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, num_args))); } // JIT: if (UNEXPECTED(num_args > first_extra_arg)) ir_ref if_extra_args = ir_IF(ir_GT(num_args, first_extra_arg)); ir_IF_TRUE_cold(if_extra_args); if (GCC_GLOBAL_REGS) { ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper)); } else { ir_ref ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_copy_extra_args_helper), jit_FP(jit), jit_IP(jit)); jit_STORE_IP(jit, ref); } ir_END_list(merge_inputs); ir_IF_FALSE(if_extra_args); if (!func || (func->op_array.fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0) { if (!func) { // JIT: if (EXPECTED((op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) == 0)) ir_ref if_has_type_hints = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, fn_flags))), ir_CONST_U32(ZEND_ACC_HAS_TYPE_HINTS))); ir_IF_TRUE(if_has_type_hints); ir_END_list(merge_inputs); ir_IF_FALSE(if_has_type_hints); } // JIT: opline += num_args; ir_ref ref = ir_MUL_U32(num_args, ir_CONST_U32(sizeof(zend_op))); if (sizeof(void*) == 8) { ref = ir_ZEXT_A(ref); } jit_STORE_IP(jit, ir_ADD_A(jit_IP(jit), ref)); } ir_END_list(merge_inputs); ir_MERGE_list(merge_inputs); // JIT: if (EXPECTED((int)num_args < op_array->last_var)) { ir_ref last_var; if (func) { last_var = ir_CONST_U32(func->op_array.last_var); } else { ZEND_ASSERT(func_ref); last_var = ir_LOAD_U32(ir_ADD_OFFSET(func_ref, offsetof(zend_op_array, last_var))); } ir_ref idx = ir_SUB_U32(last_var, num_args); ir_ref if_need = ir_IF(ir_GT(idx, ir_CONST_U32(0))); ir_IF_TRUE(if_need); // JIT: zval *var = EX_VAR_NUM(num_args); if (sizeof(void*) == 8) { num_args = ir_ZEXT_A(num_args); } ir_ref var_ref = ir_ADD_OFFSET( ir_ADD_A(jit_FP(jit), ir_MUL_A(num_args, ir_CONST_ADDR(sizeof(zval)))), (ZEND_CALL_FRAME_SLOT * sizeof(zval)) + offsetof(zval, u1.type_info)); ir_ref loop = ir_LOOP_BEGIN(ir_END()); var_ref = ir_PHI_2(IR_ADDR, var_ref, IR_UNUSED); idx = ir_PHI_2(IR_U32, idx, IR_UNUSED); ir_STORE(var_ref, ir_CONST_I32(IS_UNDEF)); ir_PHI_SET_OP(var_ref, 2, ir_ADD_OFFSET(var_ref, sizeof(zval))); ir_ref idx2 = ir_SUB_U32(idx, ir_CONST_U32(1)); ir_PHI_SET_OP(idx, 2, idx2); ir_ref if_not_zero = ir_IF(idx2); ir_IF_TRUE(if_not_zero); ir_MERGE_SET_OP(loop, 2, ir_LOOP_END()); ir_IF_FALSE(if_not_zero); ir_MERGE_WITH_EMPTY_FALSE(if_need); } if (ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0)) { ir_ref observer_handler; ir_ref rx = jit_FP(jit); struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref); if (trace && (trace->op != ZEND_JIT_TRACE_END || trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { ZEND_ASSERT(trace[1].op == ZEND_JIT_TRACE_VM || trace[1].op == ZEND_JIT_TRACE_END); jit_SET_EX_OPLINE(jit, trace[1].opline); } else { // EX(opline) = opline ir_STORE(jit_EX(opline), jit_IP(jit)); } jit_observer_fcall_begin(jit, rx, observer_handler); if (trace) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } else { exit_addr = NULL; } zend_jit_check_timeout(jit, NULL /* we're inside the called function */, exit_addr); jit_observer_fcall_is_unobserved_end(jit, &unobserved_data); } if (trace) { if (!func && (opline->opcode != ZEND_DO_UCALL)) { user_path = ir_END(); } } else { zend_basic_block *bb; do { if (recursive_call_through_jmp) { ir_ref begin, end; ir_insn *insn; /* attempt to convert direct recursive call into loop */ begin = jit->bb_start_ref[num_args]; ZEND_ASSERT(begin != IR_UNUSED); insn = &jit->ctx.ir_base[begin]; if (insn->op == IR_BEGIN) { end = ir_LOOP_END(); insn = &jit->ctx.ir_base[begin]; insn->op = IR_LOOP_BEGIN; insn->inputs_count = 2; insn->op2 = end; break; } else if ((insn->op == IR_MERGE || insn->op == IR_LOOP_BEGIN) && insn->inputs_count == 2) { end = ir_LOOP_END(); insn = &jit->ctx.ir_base[begin]; insn->op = IR_LOOP_BEGIN; insn->inputs_count = 3; insn->op3 = end; break; } else if (insn->op == IR_LOOP_BEGIN && insn->inputs_count == 3) { ZEND_ASSERT(jit->ctx.ir_base[insn->op3].op == IR_LOOP_END); jit->ctx.ir_base[insn->op3].op = IR_END; ir_MERGE_2(insn->op3, ir_END()); end = ir_LOOP_END(); insn = &jit->ctx.ir_base[begin]; insn->op3 = end; break; } } /* fallback to indirect JMP or RETURN */ if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit))); } else { zend_jit_vm_enter(jit, jit_IP(jit)); } } while (0); bb = &jit->ssa->cfg.blocks[jit->b]; if (bb->successors_count > 0) { int succ; ir_ref ref; ZEND_ASSERT(bb->successors_count == 1); succ = bb->successors[0]; /* Add a fake control edge from UNREACHABLE/RETURN to the following ENTRY */ ref = jit->ctx.insns_count - 1; ZEND_ASSERT(jit->ctx.ir_base[ref].op == IR_UNREACHABLE || jit->ctx.ir_base[ref].op == IR_RETURN || jit->ctx.ir_base[ref].op == IR_LOOP_END); ZEND_ASSERT(jit->ssa->cfg.blocks[succ].flags & ZEND_BB_ENTRY); ref = zend_jit_continue_entry(jit, ref, jit->ssa->cfg.blocks[succ].start); if (func || (opline->opcode == ZEND_DO_UCALL)) { _zend_jit_add_predecessor_ref(jit, succ, jit->b, ref); jit->b = -1; } else { user_path = ref; } } } } if ((!func || func->type == ZEND_INTERNAL_FUNCTION) && (opline->opcode != ZEND_DO_UCALL)) { if (!func && (opline->opcode != ZEND_DO_ICALL)) { ir_IF_FALSE(if_user); } // JIT: EG(current_execute_data) = execute_data; ir_STORE(jit_EG(current_execute_data), rx); bool may_have_observer = ZEND_OBSERVER_ENABLED && (!func || (func->common.fn_flags & (ZEND_ACC_CALL_VIA_TRAMPOLINE | ZEND_ACC_GENERATOR)) == 0); if (may_have_observer) { ir_ref observer_handler; struct jit_observer_fcall_is_unobserved_data unobserved_data = jit_observer_fcall_is_unobserved_start(jit, func, &observer_handler, rx, func_ref ? func_ref : ir_LOAD_A(jit_CALL(rx, func))); jit_observer_fcall_begin(jit, rx, observer_handler); jit_observer_fcall_is_unobserved_end(jit, &unobserved_data); } // JIT: ZVAL_NULL(EX_VAR(opline->result.var)); ir_ref res_addr = IR_UNUSED, func_ptr; if (RETURN_VALUE_USED(opline)) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } else { /* CPU stack allocated temporary zval */ ir_ref ptr; if (!jit->ctx.fixed_call_stack_size) { // JIT: alloca(sizeof(void*)); ptr = ir_ALLOCA(ir_CONST_ADDR(sizeof(zval))); } else { #ifdef _WIN64 ptr = ir_HARD_COPY_A(jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS)); #else ptr = ir_HARD_COPY_A(ir_RLOAD_A(IR_REG_SP)); #endif } res_addr = ZEND_ADDR_REF_ZVAL(ptr); } jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); zend_jit_reset_last_valid_opline(jit); // JIT: (zend_execute_internal ? zend_execute_internal : fbc->internal_function.handler)(call, ret); ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr); if (zend_execute_internal) { ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_execute_internal), rx, res_ref); } else { if (func) { func_ptr = ir_CONST_FC_FUNC(func->internal_function.handler); } else { func_ptr = ir_LOAD_A(ir_ADD_OFFSET(func_ref, offsetof(zend_internal_function, handler))); #if defined(IR_TARGET_X86) func_ptr = ir_CAST_FC_FUNC(func_ptr); #endif } ir_CALL_2(IR_VOID, func_ptr, rx, res_ref); } if (may_have_observer) { jit_observer_fcall_end(jit, rx, res_ref); } /* When zend_interrupt_function is set, it gets called while * the frame is still on top. This is less efficient than * doing it later once it's popped off. There is code further * down that handles when there isn't an interrupt function. */ if (zend_interrupt_function) { // JIT: if (EG(vm_interrupt)) zend_fcall_interrupt(execute_data); ir_ref if_interrupt = ir_IF(ir_LOAD_U8(jit_EG(vm_interrupt))); ir_IF_TRUE_cold(if_interrupt); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_fcall_interrupt), rx); ir_MERGE_WITH_EMPTY_FALSE(if_interrupt); } // JIT: EG(current_execute_data) = execute_data; ir_STORE(jit_EG(current_execute_data), jit_FP(jit)); // JIT: zend_vm_stack_free_args(call); if (func && !unknown_num_args) { for (i = 0; i < call_num_args; i++ ) { if (zend_jit_needs_arg_dtor(func, i, call_info)) { uint32_t offset = EX_NUM_TO_VAR(i); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_RX, offset); jit_ZVAL_PTR_DTOR(jit, var_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 0, opline); } } } else { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_vm_stack_free_args_helper), rx); } if (may_have_extra_named_params) { // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_HAS_EXTRA_NAMED_PARAMS)) ir_ref if_has_named = ir_IF(ir_AND_U8( ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 3)), ir_CONST_U8(ZEND_CALL_HAS_EXTRA_NAMED_PARAMS >> 24))); ir_IF_TRUE_cold(if_has_named); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_extra_named_params), ir_LOAD_A(jit_CALL(rx, extra_named_params))); ir_MERGE_WITH_EMPTY_FALSE(if_has_named); } if (opline->opcode == ZEND_DO_FCALL) { // TODO: optimize ??? // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_RELEASE_THIS)) ir_ref if_release_this = ir_IF(ir_AND_U8( ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)), ir_CONST_U8(ZEND_CALL_RELEASE_THIS >> 16))); ir_IF_TRUE_cold(if_release_this); // JIT: OBJ_RELEASE(Z_OBJ(RX->This)); jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_CALL(rx, This.value.obj))); ir_MERGE_WITH_EMPTY_FALSE(if_release_this); } ir_ref allocated_path = IR_UNUSED; if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !JIT_G(current_frame)->call || !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)->call) || prev_opline->opcode == ZEND_SEND_UNPACK || prev_opline->opcode == ZEND_SEND_ARRAY || prev_opline->opcode == ZEND_CHECK_UNDEF_ARGS) { // JIT: zend_vm_stack_free_call_frame(call); // JIT: if (UNEXPECTED(ZEND_CALL_INFO(call) & ZEND_CALL_ALLOCATED)) ir_ref if_allocated = ir_IF(ir_AND_U8( ir_LOAD_U8(ir_ADD_OFFSET(rx, offsetof(zend_execute_data, This.u1.type_info) + 2)), ir_CONST_U8(ZEND_CALL_ALLOCATED >> 16))); ir_IF_TRUE_cold(if_allocated); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_free_call_frame), rx); allocated_path = ir_END(); ir_IF_FALSE(if_allocated); } ir_STORE(jit_EG(vm_stack_top), rx); if (allocated_path) { ir_MERGE_WITH(allocated_path); } if (!RETURN_VALUE_USED(opline)) { zend_class_entry *ce; bool ce_is_instanceof; uint32_t func_info = call_info ? zend_get_func_info(call_info, ssa, &ce, &ce_is_instanceof) : (MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN); /* If an exception is thrown, the return_value may stay at the * original value of null. */ func_info |= MAY_BE_NULL; if (func_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { ir_ref sp; if (!jit->ctx.fixed_call_stack_size) { sp = ir_RLOAD_A(IR_REG_SP); } else { #ifdef _WIN64 sp = jit_ADD_OFFSET(jit, ir_RLOAD_A(IR_REG_SP), IR_SHADOW_ARGS); #else sp = ir_RLOAD_A(IR_REG_SP); #endif } res_addr = ZEND_ADDR_REF_ZVAL(sp); jit_ZVAL_PTR_DTOR(jit, res_addr, func_info, 1, opline); } if (!jit->ctx.fixed_call_stack_size) { // JIT: revert alloca ir_AFREE(ir_CONST_ADDR(sizeof(zval))); } } // JIT: if (UNEXPECTED(EG(exception) != NULL)) { ir_GUARD_NOT(ir_LOAD_A(jit_EG_exception(jit)), jit_STUB_ADDR(jit, jit_stub_icall_throw)); /* If there isn't a zend_interrupt_function, the timeout is * handled here because it's more efficient. */ if (!zend_interrupt_function) { // TODO: Can we avoid checking for interrupts after each call ??? if (trace && jit->last_valid_opline != opline) { int32_t exit_point = zend_jit_trace_get_exit_point(opline + 1, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } else { exit_addr = NULL; } zend_jit_check_timeout(jit, opline + 1, exit_addr); } if ((!trace || !func) && opline->opcode != ZEND_DO_ICALL) { jit_LOAD_IP_ADDR(jit, opline + 1); } else if (trace && trace->op == ZEND_JIT_TRACE_END && trace->stop >= ZEND_JIT_TRACE_STOP_INTERPRETER) { jit_LOAD_IP_ADDR(jit, opline + 1); } } if (user_path) { ir_MERGE_WITH(user_path); } return 1; } static int zend_jit_constructor(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, int call_level, int next_block) { ir_ref if_skip_constructor = jit_IF_ex(jit, jit_CMP_IP(jit, IR_NE, opline), next_block); ir_IF_FALSE(if_skip_constructor); if (JIT_G(opt_level) < ZEND_JIT_LEVEL_INLINE) { if (!zend_jit_tail_handler(jit, opline)) { return 0; } } else { if (!zend_jit_do_fcall(jit, opline, op_array, ssa, call_level, next_block, NULL)) { return 0; } } /* override predecessors of the next block */ ZEND_ASSERT(jit->ssa->cfg.blocks[next_block].predecessors_count == 1); if (!jit->ctx.control) { ZEND_ASSERT(jit->bb_edges[jit->bb_predecessors[next_block]]); ir_IF_TRUE(if_skip_constructor); ir_MERGE_2(jit->bb_edges[jit->bb_predecessors[next_block]], ir_END()); jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END(); } else { ZEND_ASSERT(!jit->bb_edges[jit->bb_predecessors[next_block]]); /* merge current control path with the true branch of constructor skip condition */ ir_MERGE_WITH_EMPTY_TRUE(if_skip_constructor); jit->bb_edges[jit->bb_predecessors[next_block]] = ir_END(); jit->b = -1; } return 1; } static int zend_jit_verify_arg_type(zend_jit_ctx *jit, const zend_op *opline, zend_arg_info *arg_info, bool check_exception) { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; ir_ref ref, fast_path = IR_UNUSED; ref = jit_ZVAL_ADDR(jit, res_addr); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && JIT_G(current_frame)->prev) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint8_t type = STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var)); if (type != IS_UNKNOWN && (type_mask & (1u << type))) { return 1; } } if (ZEND_ARG_SEND_MODE(arg_info)) { if (opline->opcode == ZEND_RECV_INIT) { ref = jit_ZVAL_DEREF_ref(jit, ref); } else { ref = jit_Z_PTR_ref(jit, ref); ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val)); } } if (type_mask != 0) { if (is_power_of_two(type_mask)) { uint32_t type_code = concrete_type(type_mask); ir_ref if_ok = jit_if_Z_TYPE_ref(jit, ref, ir_CONST_U8(type_code)); ir_IF_TRUE(if_ok); fast_path = ir_END(); ir_IF_FALSE_cold(if_ok); } else { ir_ref if_ok = ir_IF(ir_AND_U32( ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE_ref(jit, ref)), ir_CONST_U32(type_mask))); ir_IF_TRUE(if_ok); fast_path = ir_END(); ir_IF_FALSE_cold(if_ok); } } jit_SET_EX_OPLINE(jit, opline); ref = ir_CALL_2(IR_BOOL, ir_CONST_FC_FUNC(zend_jit_verify_arg_slow), ref, ir_CONST_ADDR(arg_info)); if (check_exception) { ir_GUARD(ref, jit_STUB_ADDR(jit, jit_stub_exception_handler)); } if (fast_path) { ir_MERGE_WITH(fast_path); } return 1; } static int zend_jit_recv(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array) { uint32_t arg_num = opline->op1.num; zend_arg_info *arg_info = NULL; if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { if (EXPECTED(arg_num <= op_array->num_args)) { arg_info = &op_array->arg_info[arg_num-1]; } else if (UNEXPECTED(op_array->fn_flags & ZEND_ACC_VARIADIC)) { arg_info = &op_array->arg_info[op_array->num_args]; } if (arg_info && !ZEND_TYPE_IS_SET(arg_info->type)) { arg_info = NULL; } } if (arg_info || (opline+1)->opcode != ZEND_RECV) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (!JIT_G(current_frame) || TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) < 0 || arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num)), ir_CONST_ADDR(exit_addr)); } } else { ir_ref if_ok =ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num))); ir_IF_FALSE_cold(if_ok); jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_missing_arg_error), jit_FP(jit)); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_IF_TRUE(if_ok); } } if (arg_info) { if (!zend_jit_verify_arg_type(jit, opline, arg_info, 1)) { return 0; } } return 1; } static int zend_jit_recv_init(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool is_last, int may_throw) { uint32_t arg_num = opline->op1.num; zval *zv = RT_CONSTANT(opline, opline->op2); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); ir_ref ref, if_fail, skip_path = IR_UNUSED; if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && JIT_G(current_frame) && TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) >= 0) { if (arg_num > TRACE_FRAME_NUM_ARGS(JIT_G(current_frame))) { jit_ZVAL_COPY_CONST(jit, res_addr, -1, -1, zv, 1); } } else { if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { ir_ref if_skip = ir_IF(ir_GE(ir_LOAD_U32(jit_EX(This.u2.num_args)), ir_CONST_U32(arg_num))); ir_IF_TRUE(if_skip); skip_path = ir_END(); ir_IF_FALSE(if_skip); } jit_ZVAL_COPY_CONST(jit, res_addr, -1, -1, zv, 1); } if (Z_CONSTANT_P(zv)) { jit_SET_EX_OPLINE(jit, opline); ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zval_update_constant_ex), jit_ZVAL_ADDR(jit, res_addr), ir_LOAD_A(ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), offsetof(zend_op_array, scope)))); if_fail = ir_IF(ref); ir_IF_TRUE_cold(if_fail); jit_ZVAL_PTR_DTOR(jit, res_addr, MAY_BE_ANY|MAY_BE_RC1|MAY_BE_RCN, 1, opline); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_IF_FALSE(if_fail); } if (skip_path) { ir_MERGE_WITH(skip_path); } if (op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS) { do { zend_arg_info *arg_info; if (arg_num <= op_array->num_args) { arg_info = &op_array->arg_info[arg_num-1]; } else if (op_array->fn_flags & ZEND_ACC_VARIADIC) { arg_info = &op_array->arg_info[op_array->num_args]; } else { break; } if (!ZEND_TYPE_IS_SET(arg_info->type)) { break; } if (!zend_jit_verify_arg_type(jit, opline, arg_info, may_throw)) { return 0; } } while (0); } return 1; } static bool zend_jit_verify_return_type(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info) { zend_arg_info *arg_info = &op_array->arg_info[-1]; ZEND_ASSERT(ZEND_TYPE_IS_SET(arg_info->type)); zend_jit_addr op1_addr = OP1_ADDR(); bool needs_slow_check = 1; uint32_t type_mask = ZEND_TYPE_PURE_MASK(arg_info->type) & MAY_BE_ANY; ir_ref fast_path = IR_UNUSED; if (type_mask != 0) { if (((op1_info & MAY_BE_ANY) & type_mask) == 0) { /* pass */ } else if (((op1_info & MAY_BE_ANY) | type_mask) == type_mask) { needs_slow_check = 0; } else if (is_power_of_two(type_mask)) { uint32_t type_code = concrete_type(type_mask); ir_ref if_ok = jit_if_Z_TYPE(jit, op1_addr, type_code); ir_IF_TRUE(if_ok); fast_path = ir_END(); ir_IF_FALSE_cold(if_ok); } else { ir_ref if_ok = ir_IF(ir_AND_U32( ir_SHL_U32(ir_CONST_U32(1), jit_Z_TYPE(jit, op1_addr)), ir_CONST_U32(type_mask))); ir_IF_TRUE(if_ok); fast_path = ir_END(); ir_IF_FALSE_cold(if_ok); } } if (needs_slow_check) { ir_ref ref; jit_SET_EX_OPLINE(jit, opline); ref = jit_ZVAL_ADDR(jit, op1_addr); if (op1_info & MAY_BE_UNDEF) { ref = zend_jit_zval_check_undef(jit, ref, opline->op1.var, NULL, 1); } ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_verify_return_slow), ref, ir_LOAD_A(jit_EX(func)), ir_CONST_ADDR(arg_info)); zend_jit_check_exception(jit); if (fast_path) { ir_MERGE_WITH(fast_path); } } return 1; } static int zend_jit_leave_frame(zend_jit_ctx *jit) { // JIT: EG(current_execute_data) = EX(prev_execute_data); ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data))); return 1; } static int zend_jit_free_cvs(zend_jit_ctx *jit) { // JIT: EG(current_execute_data) = EX(prev_execute_data); ir_STORE(jit_EG(current_execute_data), ir_LOAD_A(jit_EX(prev_execute_data))); // JIT: zend_free_compiled_variables(execute_data); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_free_compiled_variables), jit_FP(jit)); return 1; } static int zend_jit_free_cv(zend_jit_ctx *jit, uint32_t info, uint32_t var) { if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, EX_NUM_TO_VAR(var)); jit_ZVAL_PTR_DTOR(jit, var_addr, info, 1, NULL); } return 1; } static int zend_jit_free_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t info, uint32_t var_offset) { if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { jit_ZVAL_PTR_DTOR(jit, ZEND_ADDR_MEM_ZVAL(ZREG_FP, var_offset), info, 0, opline); } return 1; } static int zend_jit_leave_func(zend_jit_ctx *jit, const zend_op_array *op_array, const zend_op *opline, uint32_t op1_info, bool left_frame, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info, int indirect_var_access, int may_throw) { bool may_be_top_frame = JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !TRACE_FRAME_IS_NESTED(JIT_G(current_frame)); bool may_need_call_helper = indirect_var_access || /* may have symbol table */ !op_array->function_name || /* may have symbol table */ may_be_top_frame || (op_array->fn_flags & ZEND_ACC_VARIADIC) || /* may have extra named args */ JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) == -1 || /* unknown number of args */ (uint32_t)TRACE_FRAME_NUM_ARGS(JIT_G(current_frame)) > op_array->num_args; /* extra args */ bool may_need_release_this = !(op_array->fn_flags & ZEND_ACC_CLOSURE) && op_array->scope && !(op_array->fn_flags & ZEND_ACC_STATIC) && (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !JIT_G(current_frame) || !TRACE_FRAME_NO_NEED_RELEASE_THIS(JIT_G(current_frame))); ir_ref call_info = IR_UNUSED, ref, cold_path = IR_UNUSED; if (may_need_call_helper) { if (!left_frame) { left_frame = 1; if (!zend_jit_leave_frame(jit)) { return 0; } } /* ZEND_CALL_FAKE_CLOSURE handled on slow path to eliminate check for ZEND_CALL_CLOSURE on fast path */ call_info = ir_LOAD_U32(jit_EX(This.u1.type_info)); ref = ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_TOP|ZEND_CALL_HAS_SYMBOL_TABLE|ZEND_CALL_FREE_EXTRA_ARGS|ZEND_CALL_ALLOCATED|ZEND_CALL_HAS_EXTRA_NAMED_PARAMS|ZEND_CALL_FAKE_CLOSURE)); if (trace && trace->op != ZEND_JIT_TRACE_END) { ir_ref if_slow = ir_IF(ref); ir_IF_TRUE_cold(if_slow); if (!GCC_GLOBAL_REGS) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_leave_func_helper), jit_FP(jit), jit_IP(jit)); } else { ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_leave_func_helper)); } if (may_be_top_frame) { // TODO: try to avoid this check ??? if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) { #if 0 /* this check should be handled by the following OPLINE guard */ | cmp IP, zend_jit_halt_op | je ->trace_halt #endif } else if (GCC_GLOBAL_REGS) { ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt)); } else { ir_GUARD(ir_NE(ref, ir_CONST_ADDR(ZEND_VM_ENTER_BIT)), jit_STUB_ADDR(jit, jit_stub_trace_halt)); jit_STORE_IP(jit, ref); } } if (!GCC_GLOBAL_REGS) { // execute_data = EG(current_execute_data) jit_STORE_FP(jit, ir_LOAD_A(jit_EG(current_execute_data))); } cold_path = ir_END(); ir_IF_FALSE(if_slow); } else { ir_GUARD_NOT(ref, jit_STUB_ADDR(jit, jit_stub_leave_function_handler)); } } if ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_FAKE_CLOSURE)) == ZEND_ACC_CLOSURE) { if (!left_frame) { left_frame = 1; if (!zend_jit_leave_frame(jit)) { return 0; } } // JIT: OBJ_RELEASE(ZEND_CLOSURE_OBJECT(EX(func))); jit_OBJ_RELEASE(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(func)), -sizeof(zend_object))); } else if (may_need_release_this) { ir_ref if_release, fast_path = IR_UNUSED; if (!left_frame) { left_frame = 1; if (!zend_jit_leave_frame(jit)) { return 0; } } if (!JIT_G(current_frame) || !TRACE_FRAME_ALWAYS_RELEASE_THIS(JIT_G(current_frame))) { // JIT: if (call_info & ZEND_CALL_RELEASE_THIS) if (!call_info) { call_info = ir_LOAD_U32(jit_EX(This.u1.type_info)); } if_release = ir_IF(ir_AND_U32(call_info, ir_CONST_U32(ZEND_CALL_RELEASE_THIS))); ir_IF_FALSE(if_release); fast_path = ir_END(); ir_IF_TRUE(if_release); } // JIT: OBJ_RELEASE(execute_data->This)) jit_OBJ_RELEASE(jit, ir_LOAD_A(jit_EX(This.value.obj))); if (fast_path) { ir_MERGE_WITH(fast_path); } // TODO: avoid EG(excption) check for $this->foo() calls may_throw = 1; } // JIT: EG(vm_stack_top) = (zval*)execute_data ir_STORE(jit_EG(vm_stack_top), jit_FP(jit)); // JITL execute_data = EX(prev_execute_data) jit_STORE_FP(jit, ir_LOAD_A(jit_EX(prev_execute_data))); if (!left_frame) { // JIT: EG(current_execute_data) = execute_data ir_STORE(jit_EG(current_execute_data), jit_FP(jit)); } if (trace) { if (trace->op != ZEND_JIT_TRACE_END && (JIT_G(current_frame) && !TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { zend_jit_reset_last_valid_opline(jit); } else { /* We add extra RLOAD and RSTORE to make fusion for persistent register * mov (%FP), %IP * add $0x1c, %IP * The naive (commented) code leads to extra register allocation and move. * mov (%FP), %tmp * add $0x1c, %tmp * mov %tmp, %FP */ #if 0 jit_STORE_IP(jit, ir_ADD_OFFSET(ir_LOAD_A(jit_EX(opline)), sizeof(zend_op))); #else jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op))); #endif } if (cold_path) { ir_MERGE_WITH(cold_path); } if (trace->op == ZEND_JIT_TRACE_BACK && (!JIT_G(current_frame) || TRACE_FRAME_IS_UNKNOWN_RETURN(JIT_G(current_frame)))) { const zend_op *next_opline = trace->opline; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & MAY_BE_RC1) && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { /* exception might be thrown during destruction of unused return value */ // JIT: if (EG(exception)) ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw)); } do { trace++; } while (trace->op == ZEND_JIT_TRACE_INIT_CALL); ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); next_opline = trace->opline; ZEND_ASSERT(next_opline != NULL); if (trace->op == ZEND_JIT_TRACE_END && trace->stop == ZEND_JIT_TRACE_STOP_RECURSIVE_RET) { trace_info->flags |= ZEND_JIT_TRACE_LOOP; ir_ref if_eq = ir_IF(jit_CMP_IP(jit, IR_EQ, next_opline)); ir_IF_TRUE(if_eq); ZEND_ASSERT(jit->trace_loop_ref); ZEND_ASSERT(jit->ctx.ir_base[jit->trace_loop_ref].op2 == IR_UNUSED); ir_MERGE_SET_OP(jit->trace_loop_ref, 2, ir_END()); ir_IF_FALSE(if_eq); #ifdef ZEND_VM_HYBRID_JIT_RED_ZONE_SIZE ir_TAILCALL(IR_VOID, ir_LOAD_A(jit_IP(jit))); #else ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_escape)); #endif } else { ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), jit_STUB_ADDR(jit, jit_stub_trace_escape)); } zend_jit_set_last_valid_opline(jit, trace->opline); return 1; } else if (may_throw || (((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & MAY_BE_RC1) && (op1_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) && (!JIT_G(current_frame) || TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))))) { // JIT: if (EG(exception)) ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw)); } return 1; } else { // JIT: if (EG(exception)) ir_GUARD_NOT(ir_LOAD_A(jit_EG(exception)), jit_STUB_ADDR(jit, jit_stub_leave_throw)); // JIT: opline = EX(opline) + 1 jit_STORE_IP(jit, ir_LOAD_A(jit_EX(opline))); jit_STORE_IP(jit, ir_ADD_OFFSET(jit_IP(jit), sizeof(zend_op))); } if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit))); } else { zend_jit_vm_leave(jit, jit_IP(jit)); } jit->b = -1; return 1; } static void zend_jit_common_return(zend_jit_ctx *jit) { ZEND_ASSERT(jit->return_inputs); ir_MERGE_list(jit->return_inputs); } static int zend_jit_return(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, uint32_t op1_info, zend_jit_addr op1_addr) { zend_jit_addr ret_addr; int8_t return_value_used = -1; ir_ref return_value = IR_UNUSED, ref, refcount, if_return_value_used = IR_UNUSED; ZEND_ASSERT(op_array->type != ZEND_EVAL_CODE && op_array->function_name); ZEND_ASSERT(!(op1_info & MAY_BE_UNDEF)); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { jit->return_inputs = IR_UNUSED; if (JIT_G(current_frame)) { if (TRACE_FRAME_IS_RETURN_VALUE_USED(JIT_G(current_frame))) { return_value_used = 1; } else if (TRACE_FRAME_IS_RETURN_VALUE_UNUSED(JIT_G(current_frame))) { return_value_used = 0; } else { return_value_used = -1; } } } if (ZEND_OBSERVER_ENABLED) { if (Z_MODE(op1_addr) == IS_REG) { zend_jit_addr dst = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); if (!zend_jit_spill_store_inv(jit, op1_addr, dst, op1_info)) { return 0; } op1_addr = dst; } jit_observer_fcall_end(jit, jit_FP(jit), jit_ZVAL_ADDR(jit, op1_addr)); } // JIT: if (!EX(return_value)) return_value = ir_LOAD_A(jit_EX(return_value)); ret_addr = ZEND_ADDR_REF_ZVAL(return_value); if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (return_value_used == -1) { if_return_value_used = ir_IF(return_value); ir_IF_FALSE_cold(if_return_value_used); } if (return_value_used != 1) { if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)-(MAY_BE_OBJECT|MAY_BE_RESOURCE))) { ir_ref if_refcounted = jit_if_REFCOUNTED(jit, op1_addr); ir_IF_FALSE(if_refcounted); ir_END_list(jit->return_inputs); ir_IF_TRUE(if_refcounted); } ref = jit_Z_PTR(jit, op1_addr); refcount = jit_GC_DELREF(jit, ref); if (RC_MAY_BE_1(op1_info)) { if (RC_MAY_BE_N(op1_info)) { ir_ref if_non_zero = ir_IF(refcount); ir_IF_TRUE(if_non_zero); ir_END_list(jit->return_inputs); ir_IF_FALSE(if_non_zero); } jit_ZVAL_DTOR(jit, ref, op1_info, opline); } if (return_value_used == -1) { ir_END_list(jit->return_inputs); } } } else if (return_value_used == -1) { if_return_value_used = ir_IF(return_value); ir_IF_FALSE_cold(if_return_value_used); ir_END_list(jit->return_inputs); } if (if_return_value_used) { ir_IF_TRUE(if_return_value_used); } if (return_value_used == 0) { if (jit->return_inputs) { ZEND_ASSERT(JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE); ir_END_list(jit->return_inputs); ir_MERGE_list(jit->return_inputs); jit->return_inputs = IR_UNUSED; } return 1; } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); jit_ZVAL_COPY_CONST(jit, ret_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1); } else if (opline->op1_type == IS_TMP_VAR) { jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0); } else if (opline->op1_type == IS_CV) { if (op1_info & MAY_BE_REF) { ref = jit_ZVAL_ADDR(jit, op1_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || (op1_info & (MAY_BE_REF|MAY_BE_OBJECT)) || !op_array->function_name) { jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 1); } else if (return_value_used != 1) { jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0); // JIT: if (EXPECTED(!(EX_CALL_INFO() & ZEND_CALL_CODE))) ZVAL_NULL(retval_ptr); jit_set_Z_TYPE_INFO(jit, op1_addr, IS_NULL); } else { jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0); } } else { jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0); } } else { if (op1_info & MAY_BE_REF) { ir_ref if_ref, ref2, if_non_zero; zend_jit_addr ref_addr; if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE); ir_IF_TRUE_cold(if_ref); // JIT: zend_refcounted *ref = Z_COUNTED_P(retval_ptr) ref = jit_Z_PTR(jit, op1_addr); // JIT: ZVAL_COPY_VALUE(return_value, &ref->value) ref2 = ir_ADD_OFFSET(ref, offsetof(zend_reference, val)); ref_addr = ZEND_ADDR_REF_ZVAL(ref2); jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, ref_addr, op1_info, 0); ref2 = jit_GC_DELREF(jit, ref); if_non_zero = ir_IF(ref2); ir_IF_TRUE(if_non_zero); // JIT: if (IS_REFCOUNTED()) ir_ref if_refcounted = jit_if_REFCOUNTED(jit, ret_addr); ir_IF_FALSE(if_refcounted); ir_END_list(jit->return_inputs); ir_IF_TRUE(if_refcounted); // JIT: ADDREF ref2 = jit_Z_PTR(jit, ret_addr); jit_GC_ADDREF(jit, ref2); ir_END_list(jit->return_inputs); ir_IF_FALSE(if_non_zero); jit_EFREE(jit, ref, sizeof(zend_reference), op_array, opline); ir_END_list(jit->return_inputs); ir_IF_FALSE(if_ref); } jit_ZVAL_COPY(jit, ret_addr, MAY_BE_ANY, op1_addr, op1_info, 0); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (jit->return_inputs) { ir_END_list(jit->return_inputs); ir_MERGE_list(jit->return_inputs); jit->return_inputs = IR_UNUSED; } } else { ir_END_list(jit->return_inputs); jit->b = -1; } return 1; } static int zend_jit_bind_global(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) { zend_jit_addr op1_addr = OP1_ADDR(); zend_string *varname = Z_STR_P(RT_CONSTANT(opline, opline->op2)); ir_ref cache_slot_ref, idx_ref, num_used_ref, bucket_ref, ref, ref2; ir_ref if_fit, if_reference, if_same_key, fast_path; ir_ref slow_inputs = IR_UNUSED, end_inputs = IR_UNUSED; // JIT: idx = (uintptr_t)CACHED_PTR(opline->extended_value) - 1; cache_slot_ref = ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), opline->extended_value); idx_ref = ir_SUB_A(ir_LOAD_A(cache_slot_ref), ir_CONST_ADDR(1)); // JIT: if (EXPECTED(idx < EG(symbol_table).nNumUsed * sizeof(Bucket))) num_used_ref = ir_MUL_U32(ir_LOAD_U32(jit_EG(symbol_table.nNumUsed)), ir_CONST_U32(sizeof(Bucket))); if (sizeof(void*) == 8) { num_used_ref = ir_ZEXT_A(num_used_ref); } if_fit = ir_IF(ir_ULT(idx_ref, num_used_ref)); ir_IF_FALSE_cold(if_fit); ir_END_list(slow_inputs); ir_IF_TRUE(if_fit); // JIT: Bucket *p = (Bucket*)((char*)EG(symbol_table).arData + idx); bucket_ref = ir_ADD_A(ir_LOAD_A(jit_EG(symbol_table.arData)), idx_ref); if_reference = jit_if_Z_TYPE_ref(jit, bucket_ref, ir_CONST_U8(IS_REFERENCE)); ir_IF_FALSE_cold(if_reference); ir_END_list(slow_inputs); ir_IF_TRUE(if_reference); // JIT: (EXPECTED(p->key == varname)) if_same_key = ir_IF(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(bucket_ref, offsetof(Bucket, key))), ir_CONST_ADDR(varname))); ir_IF_FALSE_cold(if_same_key); ir_END_list(slow_inputs); ir_IF_TRUE(if_same_key); // JIT: GC_ADDREF(Z_PTR(p->val)) ref = jit_Z_PTR_ref(jit, bucket_ref); jit_GC_ADDREF(jit, ref); fast_path = ir_END(); ir_MERGE_list(slow_inputs); ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_global_helper), ir_CONST_ADDR(varname), cache_slot_ref); ir_MERGE_WITH(fast_path); ref = ir_PHI_2(IR_ADDR, ref2, ref); if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { ir_ref if_refcounted = IR_UNUSED, refcount, if_non_zero, if_may_not_leak; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { // JIT: if (UNEXPECTED(Z_REFCOUNTED_P(variable_ptr))) if_refcounted = jit_if_REFCOUNTED(jit, op1_addr); ir_IF_TRUE_cold(if_refcounted); } // JIT:zend_refcounted *garbage = Z_COUNTED_P(variable_ptr); ref2 = jit_Z_PTR(jit, op1_addr); // JIT: ZVAL_REF(variable_ptr, ref) jit_set_Z_PTR(jit, op1_addr, ref); jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX); // JIT: if (GC_DELREF(garbage) == 0) refcount = jit_GC_DELREF(jit, ref2); if_non_zero = ir_IF(refcount); if (!(op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT))) { ir_IF_TRUE(if_non_zero); ir_END_list(end_inputs); } ir_IF_FALSE(if_non_zero); jit_ZVAL_DTOR(jit, ref2, op1_info, opline); if (op1_info & (MAY_BE_REF|MAY_BE_ARRAY|MAY_BE_OBJECT)) { ir_END_list(end_inputs); ir_IF_TRUE(if_non_zero); // JIT: GC_ZVAL_CHECK_POSSIBLE_ROOT(variable_ptr) if_may_not_leak = jit_if_GC_MAY_NOT_LEAK(jit, ref2); ir_IF_TRUE(if_may_not_leak); ir_END_list(end_inputs); ir_IF_FALSE(if_may_not_leak); if (opline) { jit_SET_EX_OPLINE(jit, opline); } ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(gc_possible_root), ref2); } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { ir_END_list(end_inputs); ir_IF_FALSE(if_refcounted); } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_OBJECT|MAY_BE_RESOURCE))) { // JIT: ZVAL_REF(variable_ptr, ref) jit_set_Z_PTR(jit, op1_addr, ref); jit_set_Z_TYPE_INFO(jit, op1_addr, IS_REFERENCE_EX); } if (end_inputs) { ir_END_list(end_inputs); ir_MERGE_list(end_inputs); } return 1; } static int zend_jit_free(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, int may_throw) { zend_jit_addr op1_addr = OP1_ADDR(); if (op1_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) { if (may_throw) { jit_SET_EX_OPLINE(jit, opline); } if (opline->opcode == ZEND_FE_FREE && (op1_info & (MAY_BE_OBJECT|MAY_BE_REF))) { ir_ref ref, if_array, if_exists, end_inputs = IR_UNUSED; if (op1_info & MAY_BE_ARRAY) { if_array = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY); ir_IF_TRUE(if_array); ir_END_list(end_inputs); ir_IF_FALSE(if_array); } ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_iter_idx))); if_exists = ir_IF(ir_EQ(ref, ir_CONST_U32(-1))); ir_IF_TRUE(if_exists); ir_END_list(end_inputs); ir_IF_FALSE(if_exists); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_hash_iterator_del), ref); ir_END_list(end_inputs); ir_MERGE_list(end_inputs); } jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline); if (may_throw) { zend_jit_check_exception(jit); } } return 1; } static int zend_jit_echo(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) { if (opline->op1_type == IS_CONST) { zval *zv; size_t len; zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); len = Z_STRLEN_P(zv); if (len > 0) { const char *str = Z_STRVAL_P(zv); jit_SET_EX_OPLINE(jit, opline); ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write), ir_CONST_ADDR(str), ir_CONST_ADDR(len)); zend_jit_check_exception(jit); } } else { zend_jit_addr op1_addr = OP1_ADDR(); ir_ref ref; ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); jit_SET_EX_OPLINE(jit, opline); ref = jit_Z_PTR(jit, op1_addr); ir_CALL_2(IR_VOID, ir_CONST_FUNC(zend_write), ir_ADD_OFFSET(ref, offsetof(zend_string, val)), ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_string, len)))); if (opline->op1_type & (IS_VAR|IS_TMP_VAR)) { jit_ZVAL_PTR_DTOR(jit, op1_addr, op1_info, 0, opline); } zend_jit_check_exception(jit); } return 1; } static int zend_jit_strlen(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr) { if (opline->op1_type == IS_CONST) { zval *zv; size_t len; zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); len = Z_STRLEN_P(zv); jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(len)); if (Z_MODE(res_addr) != IS_REG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) { return 0; } } else { ir_ref ref; ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); ref = jit_Z_PTR(jit, op1_addr); ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(zend_string, len))); jit_set_Z_LVAL(jit, res_addr, ref); if (Z_MODE(res_addr) == IS_REG) { if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) { return 0; } } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } return 1; } static int zend_jit_count(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, zend_jit_addr res_addr, int may_throw) { if (opline->op1_type == IS_CONST) { zval *zv; zend_long count; zv = RT_CONSTANT(opline, opline->op1); ZEND_ASSERT(Z_TYPE_P(zv) == IS_ARRAY); count = zend_hash_num_elements(Z_ARRVAL_P(zv)); jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(count)); if (Z_MODE(res_addr) != IS_REG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) { return 0; } } else { ir_ref ref; ZEND_ASSERT((op1_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_ARRAY); // Note: See the implementation of ZEND_COUNT in Zend/zend_vm_def.h - arrays do not contain IS_UNDEF starting in php 8.1+. ref = jit_Z_PTR(jit, op1_addr); if (sizeof(void*) == 8) { ref = ir_LOAD_U32(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements))); ref = ir_ZEXT_L(ref); } else { ref = ir_LOAD_L(ir_ADD_OFFSET(ref, offsetof(HashTable, nNumOfElements))); } jit_set_Z_LVAL(jit, res_addr, ref); if (Z_MODE(res_addr) == IS_REG) { if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, MAY_BE_LONG)) { return 0; } } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } if (may_throw) { zend_jit_check_exception(jit); } return 1; } static int zend_jit_in_array(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { HashTable *ht = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); ir_ref ref; ZEND_ASSERT(opline->op1_type != IS_VAR && opline->op1_type != IS_TMP_VAR); ZEND_ASSERT((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) == MAY_BE_STRING); // JIT: result = zend_hash_find_ex(ht, Z_STR_P(op1), OP1_TYPE == IS_CONST); if (opline->op1_type != IS_CONST) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ir_CONST_ADDR(ht), jit_Z_PTR(jit, op1_addr)); } else { zend_string *str = Z_STR_P(RT_CONSTANT(opline, opline->op1)); ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ir_CONST_ADDR(ht), ir_CONST_ADDR(str)); } if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } } else if (smart_branch_opcode) { zend_basic_block *bb; ZEND_ASSERT(jit->b >= 0); bb = &jit->ssa->cfg.blocks[jit->b]; ZEND_ASSERT(bb->successors_count == 2); ref = jit_IF_ex(jit, ref, (smart_branch_opcode == ZEND_JMPZ) ? target_label2 : target_label); _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ref); jit->b = -1; } else { jit_set_Z_TYPE_INFO_ex(jit, res_addr, ir_ADD_U32(ir_ZEXT_U32(ir_NE(ref, IR_NULL)), ir_CONST_U32(IS_FALSE))); } return 1; } static int zend_jit_rope(zend_jit_ctx *jit, const zend_op *opline, uint32_t op2_info) { uint32_t offset; offset = (opline->opcode == ZEND_ROPE_INIT) ? opline->result.var : opline->op1.var + opline->extended_value * sizeof(zend_string*); if (opline->op2_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op2); zend_string *str; ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); str = Z_STR_P(zv); ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ir_CONST_ADDR(str)); } else { zend_jit_addr op2_addr = OP2_ADDR(); ir_ref ref; ZEND_ASSERT((op2_info & (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)) == MAY_BE_STRING); ref = jit_Z_PTR(jit, op2_addr); ir_STORE(ir_ADD_OFFSET(jit_FP(jit), offset), ref); if (opline->op2_type == IS_CV) { ir_ref if_refcounted, long_path; if_refcounted = jit_if_REFCOUNTED(jit, op2_addr); ir_IF_TRUE(if_refcounted); jit_GC_ADDREF(jit, ref); long_path = ir_END(); ir_IF_FALSE(if_refcounted); ir_MERGE_WITH(long_path); } } if (opline->opcode == ZEND_ROPE_END) { zend_jit_addr res_addr = RES_ADDR(); ir_ref ref; ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_rope_end), ir_ADD_OFFSET(jit_FP(jit), opline->op1.var), ir_CONST_U32(opline->extended_value)); jit_set_Z_PTR(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX); } return 1; } static int zend_jit_zval_copy_deref_reg(zend_jit_ctx *jit, zend_jit_addr res_addr, uint32_t res_info, zend_jit_addr val_addr, ir_ref type, ir_ref *values) { ir_ref if_type, val; if (res_info == MAY_BE_LONG) { if_type = ir_IF(ir_EQ(type, ir_CONST_U32(IS_LONG))); ir_IF_TRUE(if_type); val = jit_ZVAL_ADDR(jit, val_addr); ir_END_PHI_list(*values, val); ir_IF_FALSE(if_type); val = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val)); ir_END_PHI_list(*values, val); } else if (res_info == MAY_BE_DOUBLE) { if_type = ir_IF(ir_EQ(type, ir_CONST_U32(IS_DOUBLE))); ir_IF_TRUE(if_type); val = jit_ZVAL_ADDR(jit, val_addr); ir_END_PHI_list(*values, val); ir_IF_FALSE(if_type); val = ir_ADD_OFFSET(jit_Z_PTR(jit, val_addr), offsetof(zend_reference, val)); ir_END_PHI_list(*values, val); } else { ZEND_UNREACHABLE(); } return 1; } static int zend_jit_zval_copy_deref(zend_jit_ctx *jit, zend_jit_addr res_addr, zend_jit_addr val_addr, ir_ref type) { ir_ref if_refcounted, if_reference, if_refcounted2, ptr, val2, ptr2, type2; ir_refs *merge_inputs, *types, *ptrs; #if SIZEOF_ZEND_LONG == 4 ir_ref val = jit_ZVAL_ADDR(jit, val_addr); ir_refs *values; /* we need this only for zval.w2 copy */ #endif ir_refs_init(merge_inputs, 4); ir_refs_init(types, 4); ir_refs_init(ptrs, 4); #if SIZEOF_ZEND_LONG == 4 ir_refs_init(values, 4); #endif // JIT: ptr = Z_PTR_P(val); ptr = jit_Z_PTR(jit, val_addr); // JIT: if (Z_OPT_REFCOUNTED_P(val)) { if_refcounted = ir_IF(ir_AND_U32(type, ir_CONST_U32(Z_TYPE_FLAGS_MASK))); ir_IF_FALSE_cold(if_refcounted); ir_refs_add(merge_inputs, ir_END()); ir_refs_add(types, type); ir_refs_add(ptrs, ptr); #if SIZEOF_ZEND_LONG == 4 ir_refs_add(values, val); #endif ir_IF_TRUE(if_refcounted); // JIT: if (UNEXPECTED(Z_OPT_ISREF_P(val))) { if_reference = ir_IF(ir_EQ(type, ir_CONST_U32(IS_REFERENCE_EX))); // if_reference = ir_IF(ir_EQ(ir_TRUNC_U8(type), ir_CONST_U8(IS_REFERENCE))); // TODO: fix IR to avoid need for extra register ??? ir_IF_TRUE(if_reference); // JIT: val = Z_REFVAL_P(val); val2 = ir_ADD_OFFSET(ptr, offsetof(zend_reference, val)); type2 = jit_Z_TYPE_INFO_ref(jit, val2); ptr2 = jit_Z_PTR_ref(jit, val2); // JIT: if (Z_OPT_REFCOUNTED_P(val)) { if_refcounted2 = ir_IF(ir_AND_U32(type2, ir_CONST_U32(Z_TYPE_FLAGS_MASK))); ir_IF_FALSE_cold(if_refcounted2); ir_refs_add(merge_inputs, ir_END()); ir_refs_add(types, type2); ir_refs_add(ptrs, ptr2); #if SIZEOF_ZEND_LONG == 4 ir_refs_add(values, val2); #endif ir_IF_TRUE(if_refcounted2); ir_MERGE_WITH_EMPTY_FALSE(if_reference); type = ir_PHI_2(IR_U32, type2, type); ptr = ir_PHI_2(IR_ADDR, ptr2, ptr); #if SIZEOF_ZEND_LONG == 4 val = ir_PHI_2(IR_ADDR, val2, val); #endif // JIT: Z_ADDREF_P(val); jit_GC_ADDREF(jit, ptr); ir_refs_add(merge_inputs, ir_END()); ir_refs_add(types, type); ir_refs_add(ptrs, ptr); #if SIZEOF_ZEND_LONG == 4 ir_refs_add(values, val); #endif ir_MERGE_N(merge_inputs->count, merge_inputs->refs); type = ir_PHI_N(IR_U32, types->count, types->refs); ptr = ir_PHI_N(IR_ADDR, ptrs->count, ptrs->refs); #if SIZEOF_ZEND_LONG == 4 val = ir_PHI_N(IR_ADDR, values->count, values->refs); val_addr = ZEND_ADDR_REF_ZVAL(val); #endif // JIT: Z_PTR_P(res) = ptr; jit_set_Z_PTR(jit, res_addr, ptr); #if SIZEOF_ZEND_LONG == 4 jit_set_Z_W2(jit, res_addr, jit_Z_W2(jit, val_addr)); #endif jit_set_Z_TYPE_INFO_ex(jit, res_addr, type); return 1; } static int zend_jit_fetch_dimension_address_inner(zend_jit_ctx *jit, const zend_op *opline, uint32_t type, uint32_t op1_info, uint32_t op2_info, zend_jit_addr op2_addr, zend_ssa_range *op2_range, uint8_t dim_type, const void *found_exit_addr, const void *not_found_exit_addr, const void *exit_addr, bool result_type_guard, ir_ref ht_ref, ir_refs *found_inputs, ir_refs *found_vals, ir_ref *end_inputs, ir_ref *not_found_inputs) { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); ir_ref ref = IR_UNUSED, cond, if_found; ir_ref if_type = IS_UNUSED; ir_refs *test_zval_inputs, *test_zval_values; ir_refs_init(test_zval_inputs, 4); ir_refs_init(test_zval_values, 4); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R && !exit_addr) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } if (op2_info & MAY_BE_LONG) { bool op2_loaded = 0; bool packed_loaded = 0; bool bad_packed_key = 0; ir_ref if_packed = IS_UNDEF; ir_ref h = IR_UNUSED; ir_ref idx_not_found_inputs = IR_UNUSED; if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_LONG)) { // JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_LONG)) if_type = jit_if_Z_TYPE(jit, op2_addr, IS_LONG); ir_IF_TRUE(if_type); } if (op1_info & MAY_BE_PACKED_GUARD) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } cond = ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))), ir_CONST_U32(HASH_FLAG_PACKED)); if (op1_info & MAY_BE_ARRAY_PACKED) { ir_GUARD(cond, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(cond, ir_CONST_ADDR(exit_addr)); } } if (type == BP_VAR_W) { // JIT: hval = Z_LVAL_P(dim); h = jit_Z_LVAL(jit, op2_addr); op2_loaded = 1; } if (op1_info & MAY_BE_ARRAY_PACKED) { zend_long val = -1; if (Z_MODE(op2_addr) == IS_CONST_ZVAL) { val = Z_LVAL_P(Z_ZV(op2_addr)); if (val >= 0 && val < HT_MAX_SIZE) { packed_loaded = 1; } else { bad_packed_key = 1; } h = ir_CONST_LONG(val); } else { if (!op2_loaded) { // JIT: hval = Z_LVAL_P(dim); h = jit_Z_LVAL(jit, op2_addr); op2_loaded = 1; } packed_loaded = 1; } if (dim_type == IS_UNDEF && type == BP_VAR_W && packed_loaded) { /* don't generate "fast" code for packed array */ packed_loaded = 0; } if (packed_loaded) { // JIT: ZEND_HASH_INDEX_FIND(ht, hval, retval, num_undef); if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { if_packed = ir_IF( ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))), ir_CONST_U32(HASH_FLAG_PACKED))); ir_IF_TRUE(if_packed); } // JIT: if (EXPECTED((zend_ulong)(_h) < (zend_ulong)(_ht)->nNumUsed)) ref = ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed))); #if SIZEOF_ZEND_LONG == 8 if ((Z_MODE(op2_addr) == IS_CONST_ZVAL && val >= 0 && val <= UINT32_MAX) || (op2_range && op2_range->min >= 0 && op2_range->max <= UINT32_MAX)) { /* comapre only the lower 32-bits to allow load fusion on x86_64 */ cond = ir_ULT(ir_TRUNC_U32(h), ref); } else { cond = ir_ULT(h, ir_ZEXT_L(ref)); } #else cond = ir_ULT(h, ref); #endif if (type == BP_JIT_IS) { if (not_found_exit_addr) { ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr)); } else { ir_ref if_fit = ir_IF(cond); ir_IF_FALSE(if_fit); ir_END_list(*end_inputs); ir_IF_TRUE(if_fit); } } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { ir_GUARD(cond, ir_CONST_ADDR(exit_addr)); } else if (type == BP_VAR_IS && not_found_exit_addr) { ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr)); } else if (type == BP_VAR_RW && not_found_exit_addr) { ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr)); } else if (type == BP_VAR_IS && result_type_guard) { ir_ref if_fit = ir_IF(cond); ir_IF_FALSE(if_fit); ir_END_list(*not_found_inputs); ir_IF_TRUE(if_fit); } else { ir_ref if_fit = ir_IF(cond); ir_IF_FALSE(if_fit); ir_END_list(idx_not_found_inputs); ir_IF_TRUE(if_fit); } // JIT: _ret = &_ht->arPacked[h]; ref = ir_MUL_L(h, ir_CONST_LONG(sizeof(zval))); ref = ir_BITCAST_A(ref); ref = ir_ADD_A(ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked))), ref); if (type == BP_JIT_IS) { ir_refs_add(test_zval_values, ref); ir_refs_add(test_zval_inputs, ir_END()); } } } switch (type) { case BP_JIT_IS: if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { if (if_packed) { ir_IF_FALSE(if_packed); if_packed = IR_UNUSED; } if (!op2_loaded) { // JIT: hval = Z_LVAL_P(dim); h = jit_Z_LVAL(jit, op2_addr); op2_loaded = 1; } if (packed_loaded) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h); } else { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h); } if (not_found_exit_addr) { ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr)); } else { if_found = ir_IF(ref); ir_IF_FALSE(if_found); ir_END_list(*end_inputs); ir_IF_TRUE(if_found); } ir_refs_add(test_zval_values, ref); ir_refs_add(test_zval_inputs, ir_END()); } else if (!not_found_exit_addr && !packed_loaded) { ir_END_list(*end_inputs); } break; case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: if (packed_loaded) { ir_ref type_ref = jit_Z_TYPE_ref(jit, ref); if (result_type_guard) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ } else if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { ir_GUARD(type_ref, ir_CONST_ADDR(exit_addr)); } else if (type == BP_VAR_IS && not_found_exit_addr) { ir_GUARD(type_ref, ir_CONST_ADDR(not_found_exit_addr)); } else { ir_ref if_def = ir_IF(type_ref); ir_IF_FALSE(if_def); ir_END_list(idx_not_found_inputs); ir_IF_TRUE(if_def); } ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); } if (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) { if (if_packed) { ir_IF_FALSE(if_packed); if_packed = IR_UNUSED; } if (!op2_loaded) { // JIT: hval = Z_LVAL_P(dim); h = jit_Z_LVAL(jit, op2_addr); op2_loaded = 1; } if (packed_loaded) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(_zend_hash_index_find), ht_ref, h); } else { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_find), ht_ref, h); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else if (type == BP_VAR_IS && not_found_exit_addr) { ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr)); } else if (type == BP_VAR_IS && result_type_guard) { if_found = ir_IF(ref); ir_IF_FALSE(if_found); ir_END_list(*not_found_inputs); ir_IF_TRUE(if_found); } else { if_found = ir_IF(ref); ir_IF_FALSE(if_found); ir_END_list(idx_not_found_inputs); ir_IF_TRUE(if_found); } ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); } else if (!packed_loaded) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else if (type == BP_VAR_IS && not_found_exit_addr) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(not_found_exit_addr)); } else if (type == BP_VAR_IS && result_type_guard) { ir_END_list(*not_found_inputs); } else { ir_END_list(idx_not_found_inputs); } } if (idx_not_found_inputs) { ir_MERGE_list(idx_not_found_inputs); switch (type) { case BP_VAR_R: ZEND_ASSERT(JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE); // JIT: zend_error(E_WARNING,"Undefined array key " ZEND_LONG_FMT, hval); // JIT: retval = &EG(uninitialized_zval); jit_SET_EX_OPLINE(jit, opline); if (Z_MODE(op2_addr) == IS_REG) { if (!op2_loaded) { // JIT: hval = Z_LVAL_P(dim); h = jit_Z_LVAL(jit, op2_addr); } if (GCC_GLOBAL_REGS) { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key_ex), h); } else { ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_long_key_ex), h, jit_FP(jit)); } } else { ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_offset, IR_FASTCALL_FUNC)); } ir_END_list(*end_inputs); break; case BP_VAR_IS: case BP_VAR_UNSET: if (!not_found_exit_addr) { // JIT: retval = &EG(uninitialized_zval); jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); ir_END_list(*end_inputs); } break; default: ZEND_UNREACHABLE(); } } break; case BP_VAR_RW: if (packed_loaded) { if (not_found_exit_addr) { ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); } else { ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref)); ir_IF_TRUE(if_def); ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); ir_IF_FALSE_cold(if_def); ir_END_list(idx_not_found_inputs); } } if (!packed_loaded || !not_found_exit_addr || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH)) { if (if_packed) { ir_IF_FALSE(if_packed); if_packed = IR_UNUSED; ir_END_list(idx_not_found_inputs); } else if (!packed_loaded) { ir_END_list(idx_not_found_inputs); } ir_MERGE_list(idx_not_found_inputs); if (!op2_loaded) { // JIT: hval = Z_LVAL_P(dim); h = jit_Z_LVAL(jit, op2_addr); } if (packed_loaded) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw_no_packed), ht_ref, h); } else { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_index_lookup_rw), ht_ref, h); } if (not_found_exit_addr) { ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr)); } else { if_found = ir_IF(ref); ir_IF_FALSE(if_found); ir_END_list(*end_inputs); ir_IF_TRUE(if_found); } ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); } break; case BP_VAR_W: if (packed_loaded) { ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, ref)); ir_IF_TRUE_cold(if_def); ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); ir_IF_FALSE(if_def); ir_END_list(idx_not_found_inputs); } if (!(op1_info & MAY_BE_ARRAY_KEY_LONG) || (op1_info & MAY_BE_ARRAY_NUMERIC_HASH) || packed_loaded || bad_packed_key || dim_type == IS_UNDEF) { if (if_packed) { ir_IF_FALSE(if_packed); if_packed = IR_UNUSED; ir_END_list(idx_not_found_inputs); } else if (!packed_loaded) { ir_END_list(idx_not_found_inputs); } ir_MERGE_list(idx_not_found_inputs); if (!op2_loaded) { // JIT: hval = Z_LVAL_P(dim); h = jit_Z_LVAL(jit, op2_addr); } ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_index_lookup), ht_ref, h); ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); } break; default: ZEND_UNREACHABLE(); } } if (op2_info & MAY_BE_STRING) { ir_ref key; if (if_type) { ir_IF_FALSE(if_type); if_type = IS_UNUSED; } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { // JIT: if (EXPECTED(Z_TYPE_P(dim) == IS_STRING)) if_type = jit_if_Z_TYPE(jit, op2_addr, IS_STRING); ir_IF_TRUE(if_type); } // JIT: offset_key = Z_STR_P(dim); key = jit_Z_PTR(jit, op2_addr); // JIT: retval = zend_hash_find(ht, offset_key); switch (type) { case BP_JIT_IS: if (opline->op2_type != IS_CONST) { ir_ref if_num, end1, ref2; if_num = ir_IF( ir_ULE( ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))), ir_CONST_CHAR('9'))); ir_IF_TRUE_cold(if_num); ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key); end1 = ir_END(); ir_IF_FALSE(if_num); ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key); ir_MERGE_WITH(end1); ref = ir_PHI_2(IR_ADDR, ref2, ref); } else { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key); } if (not_found_exit_addr) { ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr)); } else { if_found = ir_IF(ref); ir_IF_FALSE(if_found); ir_END_list(*end_inputs); ir_IF_TRUE(if_found); } ir_refs_add(test_zval_values, ref); ir_refs_add(test_zval_inputs, ir_END()); break; case BP_VAR_R: case BP_VAR_IS: case BP_VAR_UNSET: if (opline->op2_type != IS_CONST) { ir_ref if_num, end1, ref2; if_num = ir_IF( ir_ULE( ir_LOAD_C(ir_ADD_OFFSET(key, offsetof(zend_string, val))), ir_CONST_CHAR('9'))); ir_IF_TRUE_cold(if_num); ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_find), ht_ref, key); end1 = ir_END(); ir_IF_FALSE(if_num); ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find), ht_ref, key); ir_MERGE_WITH(end1); ref = ir_PHI_2(IR_ADDR, ref2, ref); } else { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_find_known_hash), ht_ref, key); } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && type == BP_VAR_R) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else if (type == BP_VAR_IS && not_found_exit_addr) { ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr)); } else if (type == BP_VAR_IS && result_type_guard) { if_found = ir_IF(ref); ir_IF_FALSE(if_found); ir_END_list(*not_found_inputs); ir_IF_TRUE(if_found); } else { if_found = ir_IF(ref); switch (type) { case BP_VAR_R: ir_IF_FALSE_cold(if_found); // JIT: zend_error(E_WARNING, "Undefined array key \"%s\"", ZSTR_VAL(offset_key)); jit_SET_EX_OPLINE(jit, opline); ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_undefined_key, IR_FASTCALL_FUNC)); ir_END_list(*end_inputs); break; case BP_VAR_IS: case BP_VAR_UNSET: ir_IF_FALSE(if_found); // JIT: retval = &EG(uninitialized_zval); jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); ir_END_list(*end_inputs); break; default: ZEND_UNREACHABLE(); } ir_IF_TRUE(if_found); } ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); break; case BP_VAR_RW: if (opline->op2_type != IS_CONST) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_rw), ht_ref, key); } else { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_hash_lookup_rw), ht_ref, key); } if (not_found_exit_addr) { ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr)); } else { if_found = ir_IF(ref); ir_IF_FALSE(if_found); ir_END_list(*end_inputs); ir_IF_TRUE(if_found); } ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); break; case BP_VAR_W: if (opline->op2_type != IS_CONST) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_symtable_lookup_w), ht_ref, key); } else { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_lookup), ht_ref, key); } ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); break; default: ZEND_UNREACHABLE(); } } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { if (if_type) { ir_IF_FALSE_cold(if_type); if_type = IS_UNDEF; } if (type != BP_VAR_RW) { jit_SET_EX_OPLINE(jit, opline); } ref = jit_ZVAL_ADDR(jit, op2_addr); switch (type) { case BP_VAR_R: ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_r_helper), ht_ref, ref, jit_ZVAL_ADDR(jit, res_addr)); ir_END_list(*end_inputs); break; case BP_JIT_IS: ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_fetch_dim_isset_helper), ht_ref, ref); if (not_found_exit_addr) { ir_GUARD(ref, ir_CONST_ADDR(not_found_exit_addr)); ir_refs_add(found_inputs, ir_END()); } else if (found_exit_addr) { ir_GUARD_NOT(ref, ir_CONST_ADDR(found_exit_addr)); ir_END_list(*end_inputs); } else { if_found = ir_IF(ref); ir_IF_TRUE(if_found); ir_refs_add(found_inputs, ir_END()); ir_IF_FALSE(if_found); ir_END_list(*end_inputs); } break; case BP_VAR_IS: case BP_VAR_UNSET: ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_is_helper), ht_ref, ref, jit_ZVAL_ADDR(jit, res_addr)); ir_END_list(*end_inputs); break; case BP_VAR_RW: ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_rw_helper), ht_ref, ref); if_found = ir_IF(ref); ir_IF_TRUE(if_found); ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); ir_IF_FALSE(if_found); ir_END_list(*end_inputs); break; case BP_VAR_W: ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_w_helper), ht_ref, ref); if_found = ir_IF(ref); ir_IF_TRUE(if_found); ir_refs_add(found_inputs, ir_END()); ir_refs_add(found_vals, ref); ir_IF_FALSE(if_found); ir_END_list(*end_inputs); break; default: ZEND_UNREACHABLE(); } } if (type == BP_JIT_IS && !(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { /* dead code */ ir_END_list(*end_inputs); } else if (type == BP_JIT_IS && (op1_info & MAY_BE_ARRAY) && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) && test_zval_inputs->count) { ir_MERGE_N(test_zval_inputs->count, test_zval_inputs->refs); ref = ir_PHI_N(IR_ADDR, test_zval_values->count, test_zval_values->refs); if (op1_info & MAY_BE_ARRAY_OF_REF) { ref = jit_ZVAL_DEREF_ref(jit, ref); } cond = ir_GT(jit_Z_TYPE_ref(jit, ref), ir_CONST_U8(IS_NULL)); if (not_found_exit_addr) { ir_GUARD(cond, ir_CONST_ADDR(not_found_exit_addr)); ir_refs_add(found_inputs, ir_END()); } else if (found_exit_addr) { ir_GUARD_NOT(cond, ir_CONST_ADDR(found_exit_addr)); ir_END_list(*end_inputs); } else { ir_ref if_set = ir_IF(cond); ir_IF_FALSE(if_set); ir_END_list(*end_inputs); ir_IF_TRUE(if_set); ir_refs_add(found_inputs, ir_END()); } } return 1; } static int zend_jit_fetch_dim_read(zend_jit_ctx *jit, const zend_op *opline, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_avoid_refcounting, uint32_t op2_info, zend_jit_addr op2_addr, zend_ssa_range *op2_range, uint32_t res_info, zend_jit_addr res_addr, uint8_t dim_type) { zend_jit_addr orig_op1_addr; const void *exit_addr = NULL; const void *not_found_exit_addr = NULL; bool result_type_guard = 0; bool result_avoid_refcounting = 0; uint32_t may_be_string = (opline->opcode != ZEND_FETCH_LIST_R) ? MAY_BE_STRING : 0; int may_throw = 0; ir_ref if_type = IR_UNUSED; ir_ref end_inputs = IR_UNUSED; ir_ref not_found_inputs = IR_UNUSED; orig_op1_addr = OP1_ADDR(); if (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && !has_concrete_type(op1_info)) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) { if (!(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG)))) { result_type_guard = 1; res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; } if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) && (opline->opcode == ZEND_FETCH_LIST_R || !(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || op1_avoid_refcounting) && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) && (ssa_op+1)->op1_use == ssa_op->result_def && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF) - (MAY_BE_STRING|MAY_BE_LONG))) && zend_jit_may_avoid_refcounting(opline+1, res_info)) { result_avoid_refcounting = 1; ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; } if (opline->opcode == ZEND_FETCH_DIM_IS && !(res_info & MAY_BE_NULL)) { uint32_t flags = 0; uint32_t old_op1_info = 0; uint32_t old_info; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; int32_t exit_point; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !op1_avoid_refcounting) { flags |= ZEND_JIT_EXIT_FREE_OP1; } if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { flags |= ZEND_JIT_EXIT_FREE_OP2; } if (op1_avoid_refcounting) { old_op1_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_REG(stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_NULL, 0); SET_STACK_REG_EX(stack, EX_VAR_TO_NUM(opline->result.var), ZREG_NONE, ZREG_TYPE_ONLY); exit_point = zend_jit_trace_get_exit_point(opline+1, flags); SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_info); not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!not_found_exit_addr) { return 0; } if (op1_avoid_refcounting) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_op1_info); } } } if (op1_info & MAY_BE_REF) { ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } if (op1_info & MAY_BE_ARRAY) { ir_ref ht_ref, ref; zend_jit_addr val_addr; ir_refs *found_inputs, *found_vals; ir_refs_init(found_inputs, 10); ir_refs_init(found_vals, 10); if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { if (exit_addr && !(op1_info & (MAY_BE_OBJECT|may_be_string))) { jit_guard_Z_TYPE(jit, op1_addr, IS_ARRAY, exit_addr); } else { if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY); ir_IF_TRUE(if_type); } } ht_ref = jit_Z_PTR(jit, op1_addr); if ((op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) || (opline->opcode != ZEND_FETCH_DIM_IS && JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE)) { may_throw = 1; } if (!zend_jit_fetch_dimension_address_inner(jit, opline, (opline->opcode != ZEND_FETCH_DIM_IS) ? BP_VAR_R : BP_VAR_IS, op1_info, op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, exit_addr, result_type_guard, ht_ref, found_inputs, found_vals, &end_inputs, ¬_found_inputs)) { return 0; } if (found_inputs->count) { ir_MERGE_N(found_inputs->count, found_inputs->refs); ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs); val_addr = ZEND_ADDR_REF_ZVAL(ref); if (result_type_guard) { uint8_t type = concrete_type(res_info); uint32_t flags = 0; if (opline->opcode != ZEND_FETCH_LIST_R && (opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !op1_avoid_refcounting) { flags |= ZEND_JIT_EXIT_FREE_OP1; } if ((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { flags |= ZEND_JIT_EXIT_FREE_OP2; } val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type, (op1_info & MAY_BE_ARRAY_OF_REF) != 0, flags, op1_avoid_refcounting); if (!val_addr) { return 0; } if (not_found_inputs) { ir_END_list(not_found_inputs); ir_MERGE_list(not_found_inputs); } // ZVAL_COPY jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting); if (Z_MODE(res_addr) != IS_REG) { } else if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { return 0; } } else if (op1_info & MAY_BE_ARRAY_OF_REF) { // ZVAL_COPY_DEREF ir_ref type_info = jit_Z_TYPE_INFO(jit, val_addr); if (!zend_jit_zval_copy_deref(jit, res_addr, val_addr, type_info)) { return 0; } } else { // ZVAL_COPY jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, 1); } ir_END_list(end_inputs); } else if (not_found_inputs) { ir_MERGE_list(not_found_inputs); jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); ir_END_list(end_inputs); } else if (!end_inputs && jit->ctx.control) { ir_END_list(end_inputs); /* dead code */ } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { if (if_type) { ir_IF_FALSE_cold(if_type); if_type = IS_UNDEF; } if (opline->opcode != ZEND_FETCH_LIST_R && (op1_info & MAY_BE_STRING)) { ir_ref str_ref; may_throw = 1; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_STRING))) { if (exit_addr && !(op1_info & MAY_BE_OBJECT)) { jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, exit_addr); } else { if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); ir_IF_TRUE(if_type); } } jit_SET_EX_OPLINE(jit, opline); str_ref = jit_Z_PTR(jit, op1_addr); if (opline->opcode != ZEND_FETCH_DIM_IS) { ir_ref ref; if ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_GUARD)) == MAY_BE_LONG) { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_offset_r_helper), str_ref, jit_Z_LVAL(jit, op2_addr)); } else { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_r_helper), str_ref, jit_ZVAL_ADDR(jit, op2_addr)); } jit_set_Z_PTR(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING); } else { ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_str_is_helper), str_ref, jit_ZVAL_ADDR(jit, op2_addr), jit_ZVAL_ADDR(jit, res_addr)); } ir_END_list(end_inputs); } if (op1_info & MAY_BE_OBJECT) { ir_ref arg2; if (if_type) { ir_IF_FALSE_cold(if_type); if_type = IS_UNDEF; } may_throw = 1; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) { if (exit_addr) { jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr); } else { if_type = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT); ir_IF_TRUE(if_type); } } jit_SET_EX_OPLINE(jit, opline); if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1); } else { arg2 = jit_ZVAL_ADDR(jit, op2_addr); } if (opline->opcode != ZEND_FETCH_DIM_IS) { ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_r_helper), jit_ZVAL_ADDR(jit, op1_addr), arg2, jit_ZVAL_ADDR(jit, res_addr)); } else { ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_is_helper), jit_ZVAL_ADDR(jit, op1_addr), arg2, jit_ZVAL_ADDR(jit, res_addr)); } ir_END_list(end_inputs); } if ((op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string))) && (!exit_addr || !(op1_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|may_be_string)))) { if (if_type) { ir_IF_FALSE_cold(if_type); if_type = IS_UNDEF; } if ((opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) || (op2_info & MAY_BE_UNDEF)) { jit_SET_EX_OPLINE(jit, opline); if (opline->opcode != ZEND_FETCH_DIM_IS && (op1_info & MAY_BE_UNDEF)) { may_throw = 1; zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op1_addr), opline->op1.var, NULL, 0, 1, 0); } if (op2_info & MAY_BE_UNDEF) { may_throw = 1; zend_jit_type_check_undef(jit, jit_Z_TYPE(jit, op2_addr), opline->op2.var, NULL, 0, 1, 0); } } if (opline->opcode != ZEND_FETCH_DIM_IS) { ir_ref ref; may_throw = 1; if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { ref = jit_ZVAL_ADDR(jit, orig_op1_addr); } else { jit_SET_EX_OPLINE(jit, opline); ref = jit_ZVAL_ADDR(jit, op1_addr); } if (opline->opcode == ZEND_FETCH_LIST_R) { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_array_use), ref); } else { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_array_access), ref); } } jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); ir_END_list(end_inputs); } } if (end_inputs) { ir_MERGE_list(end_inputs); #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { /* Magic offsetGet() may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif if (opline->op2_type & (IS_TMP_VAR|IS_VAR)) { if ((op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { may_throw = 1; } jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); } if (opline->opcode != ZEND_FETCH_LIST_R && !op1_avoid_refcounting) { if (opline->op1_type & (IS_TMP_VAR|IS_VAR)) { if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { may_throw = 1; } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } } if (may_throw) { zend_jit_check_exception(jit); } } else if (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) { ir_BEGIN(IR_UNUSED); /* unreachable tail */ } return 1; } static zend_jit_addr zend_jit_prepare_array_update(zend_jit_ctx *jit, const zend_op *opline, uint32_t *op1_info_ptr, zend_jit_addr op1_addr, ir_ref *if_type, ir_ref *ht_ref, int *may_throw) { ir_ref ref = IR_UNUSED; ir_ref array_reference_end = IR_UNUSED, array_reference_ref = IR_UNUSED; ir_refs *array_inputs, *array_values; uint32_t op1_info = *op1_info_ptr; ir_refs_init(array_inputs, 4); ir_refs_init(array_values, 4); ref = jit_ZVAL_ADDR(jit, op1_addr); if (op1_info & MAY_BE_REF) { ir_ref if_reference, if_array, end1, ref2; *may_throw = 1; if_reference = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE); ir_IF_FALSE(if_reference); end1 = ir_END(); ir_IF_TRUE_cold(if_reference); array_reference_ref = ir_ADD_OFFSET(jit_Z_PTR_ref(jit, ref), offsetof(zend_reference, val)); if_array = jit_if_Z_TYPE_ref(jit, array_reference_ref, ir_CONST_U8(IS_ARRAY)); ir_IF_TRUE(if_array); array_reference_end = ir_END(); ir_IF_FALSE_cold(if_array); if (opline->opcode != ZEND_FETCH_DIM_RW && opline->opcode != ZEND_ASSIGN_DIM_OP) { jit_SET_EX_OPLINE(jit, opline); } ref2 = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_prepare_assign_dim_ref), ref); ir_GUARD(ref2, jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); ir_MERGE_WITH(end1); ref = ir_PHI_2(IR_ADDR, ref2, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } if (op1_info & MAY_BE_ARRAY) { ir_ref op1_ref = ref; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { *if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY); ir_IF_TRUE(*if_type); } if (array_reference_end) { ir_MERGE_WITH(array_reference_end); op1_ref = ir_PHI_2(IR_ADDR, ref, array_reference_ref); } // JIT: SEPARATE_ARRAY() ref = jit_Z_PTR_ref(jit, op1_ref); if (RC_MAY_BE_N(op1_info)) { if (RC_MAY_BE_1(op1_info)) { ir_ref if_refcount_1 = ir_IF(ir_EQ(jit_GC_REFCOUNT(jit, ref), ir_CONST_U32(1))); ir_IF_TRUE(if_refcount_1); ir_refs_add(array_inputs, ir_END()); ir_refs_add(array_values, ref); ir_IF_FALSE(if_refcount_1); } ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_zval_array_dup), op1_ref); } if (array_inputs->count || (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL))) { ir_refs_add(array_inputs, ir_END()); ir_refs_add(array_values, ref); } } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL)) { if (*if_type) { ir_IF_FALSE_cold(*if_type); *if_type = IR_UNUSED; } if (op1_info & (MAY_BE_ANY-(MAY_BE_NULL|MAY_BE_ARRAY))) { *if_type = ir_IF(ir_LE(jit_Z_TYPE(jit, op1_addr), ir_CONST_U8(IS_NULL))); ir_IF_TRUE(*if_type); } if ((op1_info & MAY_BE_UNDEF) && (opline->opcode == ZEND_FETCH_DIM_RW || opline->opcode == ZEND_ASSIGN_DIM_OP)) { ir_ref end1 = IR_UNUSED; *may_throw = 1; if (op1_info & MAY_BE_NULL) { ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr)); ir_IF_TRUE(if_def); end1 = ir_END(); ir_IF_FALSE(if_def); } ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var)); if (end1) { ir_MERGE_WITH(end1); } } // JIT: ZVAL_ARR(container, zend_new_array(8)); ref = ir_CALL_1(IR_ADDR, jit_STUB_FUNC_ADDR(jit, jit_stub_new_array, IR_FASTCALL_FUNC), jit_ZVAL_ADDR(jit, op1_addr)); if (array_inputs->count) { ir_refs_add(array_inputs, ir_END()); ir_refs_add(array_values, ref); } op1_info &= ~(MAY_BE_UNDEF | MAY_BE_NULL); op1_info |= MAY_BE_ARRAY | MAY_BE_RC1; *op1_info_ptr = op1_info; } if (array_inputs->count) { ir_MERGE_N(array_inputs->count, array_inputs->refs); ref = ir_PHI_N(IR_ADDR, array_values->count, array_values->refs); } *ht_ref = ref; return op1_addr; } static int zend_jit_fetch_dim(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t op2_info, zend_jit_addr op2_addr, zend_ssa_range *op2_range, zend_jit_addr res_addr, uint8_t dim_type) { int may_throw = 0; ir_ref end_inputs = IR_UNUSED; ir_ref ref, if_type = IR_UNUSED, ht_ref; if (opline->opcode == ZEND_FETCH_DIM_RW) { jit_SET_EX_OPLINE(jit, opline); } op1_addr = zend_jit_prepare_array_update(jit, opline, &op1_info, op1_addr, &if_type, &ht_ref, &may_throw); if (op1_info & MAY_BE_ARRAY) { ir_refs *found_inputs, *found_vals; ir_refs_init(found_inputs, 8); ir_refs_init(found_vals, 8); if (opline->op2_type == IS_UNUSED) { ir_ref if_ok; may_throw = 1; // JIT:var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert), ht_ref, jit_EG(uninitialized_zval)); // JIT: if (UNEXPECTED(!var_ptr)) { if_ok = ir_IF(ref); ir_IF_FALSE_cold(if_ok); if (opline->opcode != ZEND_FETCH_DIM_RW) { jit_SET_EX_OPLINE(jit, opline); } ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC)); ir_END_list(end_inputs); ir_IF_TRUE(if_ok); jit_set_Z_PTR(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT); ir_END_list(end_inputs); } else { uint32_t type; switch (opline->opcode) { case ZEND_FETCH_DIM_W: case ZEND_FETCH_LIST_W: type = BP_VAR_W; break; case ZEND_FETCH_DIM_RW: may_throw = 1; type = BP_VAR_RW; break; case ZEND_FETCH_DIM_UNSET: type = BP_VAR_UNSET; break; default: ZEND_UNREACHABLE(); } if (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING))) { may_throw = 1; } if (!zend_jit_fetch_dimension_address_inner(jit, opline, type, op1_info, op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL, 0, ht_ref, found_inputs, found_vals, &end_inputs, NULL)) { return 0; } if (type == BP_VAR_RW || (op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { if (end_inputs) { ir_MERGE_list(end_inputs); jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); end_inputs = ir_END(); } } else if (!(op2_info & (MAY_BE_ANY|MAY_BE_UNDEF))) { /* impossible dead path */ end_inputs = ir_END(); } else { ZEND_ASSERT(end_inputs == IR_UNUSED); } if (found_inputs->count) { ir_MERGE_N(found_inputs->count, found_inputs->refs); ref = ir_PHI_N(IR_ADDR, found_vals->count, found_vals->refs); jit_set_Z_PTR(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT); ir_END_list(end_inputs); } } } if (op1_info & (MAY_BE_ANY-MAY_BE_ARRAY)) { ir_ref arg2; may_throw = 1; if (if_type) { ir_IF_FALSE(if_type); if_type = IR_UNUSED; } if (opline->opcode != ZEND_FETCH_DIM_RW) { jit_SET_EX_OPLINE(jit, opline); } if (opline->op2_type == IS_UNUSED) { arg2 = IR_NULL; } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1); } else { arg2 = jit_ZVAL_ADDR(jit, op2_addr); } switch (opline->opcode) { case ZEND_FETCH_DIM_W: case ZEND_FETCH_LIST_W: ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_w_helper), jit_ZVAL_ADDR(jit, op1_addr), arg2, jit_ZVAL_ADDR(jit, res_addr)); break; case ZEND_FETCH_DIM_RW: ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_dim_obj_rw_helper), jit_ZVAL_ADDR(jit, op1_addr), arg2, jit_ZVAL_ADDR(jit, res_addr)); break; // case ZEND_FETCH_DIM_UNSET: // | EXT_CALL zend_jit_fetch_dim_obj_unset_helper, r0 // break; default: ZEND_UNREACHABLE(); } if (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_ARRAY)) { ir_END_list(end_inputs); } } #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { /* ASSIGN_DIM may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op2_info & MAY_HAVE_DTOR) && (op2_info & MAY_BE_RC1)) { may_throw = 1; } if (end_inputs) { ir_MERGE_list(end_inputs); } jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); if (may_throw) { zend_jit_check_exception(jit); } return 1; } static int zend_jit_isset_isempty_dim(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_avoid_refcounting, uint32_t op2_info, zend_jit_addr op2_addr, zend_ssa_range *op2_range, uint8_t dim_type, int may_throw, uint8_t smart_branch_opcode, uint32_t target_label, uint32_t target_label2, const void *exit_addr) { zend_jit_addr res_addr; ir_ref if_type = IR_UNUSED; ir_ref false_inputs = IR_UNUSED, end_inputs = IR_UNUSED; ir_refs *true_inputs; ir_refs_init(true_inputs, 8); // TODO: support for empty() ??? ZEND_ASSERT(!(opline->extended_value & ZEND_ISEMPTY)); res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (op1_info & MAY_BE_REF) { ir_ref ref = jit_ZVAL_ADDR(jit, op1_addr); ref = jit_ZVAL_DEREF_ref(jit, ref); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } if (op1_info & MAY_BE_ARRAY) { const void *found_exit_addr = NULL; const void *not_found_exit_addr = NULL; ir_ref ht_ref; if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY)) { if_type = jit_if_Z_TYPE(jit, op1_addr, IS_ARRAY); ir_IF_TRUE(if_type); } ht_ref = jit_Z_PTR(jit, op1_addr); if (exit_addr && !(op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) && !may_throw && (!(opline->op1_type & (IS_TMP_VAR|IS_VAR)) || op1_avoid_refcounting) && (!(opline->op2_type & (IS_TMP_VAR|IS_VAR)) || !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)))) { if (smart_branch_opcode == ZEND_JMPNZ) { found_exit_addr = exit_addr; } else { not_found_exit_addr = exit_addr; } } if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_JIT_IS, op1_info, op2_info, op2_addr, op2_range, dim_type, found_exit_addr, not_found_exit_addr, NULL, 0, ht_ref, true_inputs, NULL, &false_inputs, NULL)) { return 0; } if (found_exit_addr) { ir_MERGE_list(false_inputs); return 1; } else if (not_found_exit_addr) { ir_MERGE_N(true_inputs->count, true_inputs->refs); return 1; } } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_ARRAY)) { if (if_type) { ir_IF_FALSE(if_type); if_type = IR_UNUSED; } if (op1_info & (MAY_BE_STRING|MAY_BE_OBJECT)) { ir_ref ref, arg1, arg2, if_true; jit_SET_EX_OPLINE(jit, opline); arg1 = jit_ZVAL_ADDR(jit, op1_addr); if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); arg2 = ir_CONST_ADDR(Z_ZV(op2_addr)+1); } else { arg2 = jit_ZVAL_ADDR(jit, op2_addr); } ref = ir_CALL_2(IR_I32, ir_CONST_FC_FUNC(zend_jit_isset_dim_helper), arg1, arg2); if_true = ir_IF(ref); ir_IF_TRUE(if_true); ir_refs_add(true_inputs, ir_END()); ir_IF_FALSE(if_true); ir_END_list(false_inputs); } else { if (op2_info & MAY_BE_UNDEF) { ir_ref end1 = IR_UNUSED; if (op2_info & MAY_BE_ANY) { ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op2_addr)); ir_IF_TRUE(if_def); end1 = ir_END(); ir_IF_FALSE(if_def); } jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op2.var)); if (end1) { ir_MERGE_WITH(end1); } } ir_END_list(false_inputs); } } #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & MAY_BE_OBJECT)) { /* Magic offsetExists() may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif if (true_inputs->count) { ir_MERGE_N(true_inputs->count, true_inputs->refs); jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); if (!op1_avoid_refcounting) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } if (may_throw) { zend_jit_check_exception_undef_result(jit, opline); } if (!(opline->extended_value & ZEND_ISEMPTY)) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPNZ) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else { ir_END_list(end_inputs); } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END()); } else if (smart_branch_opcode == ZEND_JMPNZ) { _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END()); } else { ZEND_UNREACHABLE(); } } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_TRUE); ir_END_list(end_inputs); } } else { ZEND_UNREACHABLE(); // TODO: support for empty() } } ir_MERGE_list(false_inputs); jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); if (!op1_avoid_refcounting) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } if (may_throw) { zend_jit_check_exception_undef_result(jit, opline); } if (!(opline->extended_value & ZEND_ISEMPTY)) { if (exit_addr) { if (smart_branch_opcode == ZEND_JMPZ) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else { ir_END_list(end_inputs); } } else if (smart_branch_opcode) { if (smart_branch_opcode == ZEND_JMPZ) { _zend_jit_add_predecessor_ref(jit, target_label, jit->b, ir_END()); } else if (smart_branch_opcode == ZEND_JMPNZ) { _zend_jit_add_predecessor_ref(jit, target_label2, jit->b, ir_END()); } else { ZEND_UNREACHABLE(); } } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_FALSE); ir_END_list(end_inputs); } } else { ZEND_UNREACHABLE(); // TODO: support for empty() } if (!exit_addr && smart_branch_opcode) { jit->b = -1; } else { ir_MERGE_list(end_inputs); } return 1; } static int zend_jit_assign_dim(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_indirect, uint32_t op2_info, zend_jit_addr op2_addr, zend_ssa_range *op2_range, uint32_t val_info, zend_jit_addr op3_addr, zend_jit_addr op3_def_addr, zend_jit_addr res_addr, uint8_t dim_type, int may_throw) { ir_ref if_type = IR_UNUSED; ir_ref end_inputs = IR_UNUSED, ht_ref; if (op3_addr != op3_def_addr && op3_def_addr) { if (!zend_jit_update_regs(jit, (opline+1)->op1.var, op3_addr, op3_def_addr, val_info)) { return 0; } if (Z_MODE(op3_def_addr) == IS_REG && Z_MODE(op3_addr) != IS_REG) { op3_addr = op3_def_addr; } } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && (val_info & MAY_BE_UNDEF)) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_not_Z_TYPE(jit, op3_addr, IS_UNDEF, exit_addr); val_info &= ~MAY_BE_UNDEF; } op1_addr = zend_jit_prepare_array_update(jit, opline, &op1_info, op1_addr, &if_type, &ht_ref, &may_throw); if (op1_info & MAY_BE_ARRAY) { if (opline->op2_type == IS_UNUSED) { uint32_t var_info = MAY_BE_NULL; ir_ref if_ok, ref; zend_jit_addr var_addr; // JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert), ht_ref, jit_EG(uninitialized_zval)); // JIT: if (UNEXPECTED(!var_ptr)) { if_ok = ir_IF(ref); ir_IF_FALSE_cold(if_ok); // JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); jit_SET_EX_OPLINE(jit, opline); ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC)); ir_END_list(end_inputs); ir_IF_TRUE(if_ok); var_addr = ZEND_ADDR_REF_ZVAL(ref); if (!zend_jit_simple_assign(jit, opline, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { return 0; } } else { uint32_t var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); zend_jit_addr var_addr; ir_ref ref; ir_refs *found_inputs, *found_values; ir_refs_init(found_inputs, 8); ir_refs_init(found_values, 8); if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_W, op1_info, op2_info, op2_addr, op2_range, dim_type, NULL, NULL, NULL, 0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) { return 0; } if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { var_info |= MAY_BE_REF; } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { var_info |= MAY_BE_RC1; } if (found_inputs->count) { ir_MERGE_N(found_inputs->count, found_inputs->refs); ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs); var_addr = ZEND_ADDR_REF_ZVAL(ref); // JIT: value = zend_assign_to_variable(variable_ptr, value, OP_DATA_TYPE); if (opline->op1_type == IS_VAR && Z_MODE(op3_addr) != IS_REG && opline->result_type == IS_UNUSED && (res_addr == 0 || Z_MODE(res_addr) != IS_REG)) { if (!zend_jit_assign_to_variable_call(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0)) { return 0; } } else { if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, var_info, -1, (opline+1)->op1_type, op3_addr, val_info, res_addr, 0, 0)) { return 0; } } } } ir_END_list(end_inputs); } if (op1_info & (MAY_BE_ANY-MAY_BE_ARRAY)) { ir_ref arg2, arg4; if (if_type) { ir_IF_FALSE_cold(if_type); if_type = IR_UNUSED; } jit_SET_EX_OPLINE(jit, opline); if (opline->op2_type == IS_UNUSED) { arg2 = IR_NULL; } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1); } else { arg2 = jit_ZVAL_ADDR(jit, op2_addr); } if (opline->result_type == IS_UNUSED) { arg4 = IR_NULL; } else { arg4 = jit_ZVAL_ADDR(jit, res_addr); } ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_helper), jit_ZVAL_ADDR(jit, op1_addr), arg2, jit_ZVAL_ADDR(jit, op3_addr), arg4); #ifdef ZEND_JIT_USE_RC_INFERENCE if (((opline+1)->op1_type & (IS_TMP_VAR|IS_VAR)) && (val_info & MAY_BE_RC1)) { /* ASSIGN_DIM may increase refcount of the value */ val_info |= MAY_BE_RCN; } #endif jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, NULL); ir_END_list(end_inputs); } #ifdef ZEND_JIT_USE_RC_INFERENCE if ((opline->op2_type & (IS_TMP_VAR|IS_VAR)) && (op1_info & (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_FALSE|MAY_BE_ARRAY|MAY_BE_OBJECT))) { /* ASSIGN_DIM may increase refcount of the key */ op2_info |= MAY_BE_RCN; } #endif ir_MERGE_list(end_inputs); jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, opline); if (!op1_indirect) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } if (may_throw) { zend_jit_check_exception(jit); } return 1; } static int zend_jit_assign_dim_op(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op1_def_info, zend_jit_addr op1_addr, bool op1_indirect, uint32_t op2_info, zend_jit_addr op2_addr, zend_ssa_range *op2_range, uint32_t op1_data_info, zend_jit_addr op3_addr, zend_ssa_range *op1_data_range, uint8_t dim_type, int may_throw) { zend_jit_addr var_addr = IS_UNUSED; const void *not_found_exit_addr = NULL; uint32_t var_info = MAY_BE_NULL; ir_ref if_type = IS_UNUSED; ir_ref end_inputs = IR_UNUSED, ht_ref; bool emit_fast_path = 1; ZEND_ASSERT(opline->result_type == IS_UNUSED); if (may_throw) { jit_SET_EX_OPLINE(jit, opline); } op1_addr = zend_jit_prepare_array_update(jit, opline, &op1_info, op1_addr, &if_type, &ht_ref, &may_throw); if (Z_MODE(op3_addr) == IS_REG && Z_LOAD(op3_addr) && jit->ra[Z_SSA_VAR(op3_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, op3_addr); } if (op1_info & MAY_BE_ARRAY) { uint32_t var_def_info = zend_array_element_type(op1_def_info, opline->op1_type, 1, 0); if (opline->op2_type == IS_UNUSED) { var_info = MAY_BE_NULL; ir_ref if_ok, ref; // JIT: var_ptr = zend_hash_next_index_insert(Z_ARRVAL_P(container), &EG(uninitialized_zval)); ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_hash_next_index_insert), ht_ref, jit_EG(uninitialized_zval)); // JIT: if (UNEXPECTED(!var_ptr)) { if_ok = ir_IF(ref); ir_IF_FALSE_cold(if_ok); // JIT: zend_throw_error(NULL, "Cannot add element to the array as the next element is already occupied"); ir_CALL(IR_VOID, jit_STUB_FUNC_ADDR(jit, jit_stub_cannot_add_element, IR_FASTCALL_FUNC)); ir_END_list(end_inputs); ir_IF_TRUE(if_ok); var_addr = ZEND_ADDR_REF_ZVAL(ref); } else { ir_ref ref; ir_refs *found_inputs, *found_values; ir_refs_init(found_inputs, 8); ir_refs_init(found_values, 8); var_info = zend_array_element_type(op1_info, opline->op1_type, 0, 0); if (op1_info & (MAY_BE_ARRAY_OF_REF|MAY_BE_OBJECT)) { var_info |= MAY_BE_REF; } if (var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { var_info |= MAY_BE_RC1; } if (dim_type != IS_UNKNOWN && dim_type != IS_UNDEF && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY && (op2_info & (MAY_BE_LONG|MAY_BE_STRING)) && !(op2_info & ((MAY_BE_ANY|MAY_BE_UNDEF) - (MAY_BE_LONG|MAY_BE_STRING)))) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); not_found_exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!not_found_exit_addr) { return 0; } } if (!zend_jit_fetch_dimension_address_inner(jit, opline, BP_VAR_RW, op1_info, op2_info, op2_addr, op2_range, dim_type, NULL, not_found_exit_addr, NULL, 0, ht_ref, found_inputs, found_values, &end_inputs, NULL)) { return 0; } if (found_inputs->count) { ir_MERGE_N(found_inputs->count, found_inputs->refs); ref = ir_PHI_N(IR_ADDR, found_values->count, found_values->refs); var_addr = ZEND_ADDR_REF_ZVAL(ref); if (not_found_exit_addr && dim_type != IS_REFERENCE) { jit_guard_Z_TYPE(jit, var_addr, dim_type, not_found_exit_addr); var_info = (1 << dim_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); } if (var_info & MAY_BE_REF) { binary_op_type binary_op = get_binary_op(opline->extended_value); ir_ref if_ref, if_typed, noref_path, ref_path, ref, reference, ref2, arg2; ref = jit_ZVAL_ADDR(jit, var_addr); if_ref = jit_if_Z_TYPE(jit, var_addr, IS_REFERENCE); ir_IF_FALSE(if_ref); noref_path = ir_END(); ir_IF_TRUE(if_ref); reference = jit_Z_PTR_ref(jit, ref); ref2 = ir_ADD_OFFSET(reference, offsetof(zend_reference, val)); if_typed = jit_if_TYPED_REF(jit, reference); ir_IF_FALSE(if_typed); ref_path = ir_END(); ir_IF_TRUE_cold(if_typed); if (Z_MODE(op3_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var); if (!zend_jit_spill_store_inv(jit, op3_addr, real_addr, op1_data_info)) { return 0; } op3_addr = real_addr; } arg2 = jit_ZVAL_ADDR(jit, op3_addr); ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref), reference, arg2, ir_CONST_FC_FUNC(binary_op)); ir_END_list(end_inputs); ir_MERGE_2(noref_path, ref_path); ref = ir_PHI_2(IR_ADDR, ref, ref2); var_addr = ZEND_ADDR_REF_ZVAL(ref); } } else { emit_fast_path = 0; } } if (emit_fast_path) { uint8_t val_op_type = (opline+1)->op1_type; if (val_op_type & (IS_TMP_VAR|IS_VAR)) { /* prevent FREE_OP in the helpers */ val_op_type = IS_CV; } switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: case ZEND_DIV: if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, 0, var_addr, var_def_info, var_info, 1 /* may overflow */, may_throw)) { return 0; } break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: if (!zend_jit_long_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, NULL, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, op1_data_range, 0, var_addr, var_def_info, var_info, may_throw)) { return 0; } break; case ZEND_CONCAT: if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, op3_addr, op1_data_info, var_addr, may_throw)) { return 0; } break; default: ZEND_UNREACHABLE(); } ir_END_list(end_inputs); } } if (op1_info & (MAY_BE_ANY-MAY_BE_ARRAY)) { binary_op_type binary_op; ir_ref arg2; if (if_type) { ir_IF_FALSE_cold(if_type); if_type = IS_UNUSED; } if (opline->op2_type == IS_UNUSED) { arg2 = IR_NULL; } else if (opline->op2_type == IS_CONST && Z_EXTRA_P(RT_CONSTANT(opline, opline->op2)) == ZEND_EXTRA_VALUE) { ZEND_ASSERT(Z_MODE(op2_addr) == IS_CONST_ZVAL); arg2 = ir_CONST_ADDR(Z_ZV(op2_addr) + 1); } else { arg2 = jit_ZVAL_ADDR(jit, op2_addr); } binary_op = get_binary_op(opline->extended_value); ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_dim_op_helper), jit_ZVAL_ADDR(jit, op1_addr), arg2, jit_ZVAL_ADDR(jit, op3_addr), ir_CONST_FC_FUNC(binary_op)); ir_END_list(end_inputs); } if (end_inputs) { ir_MERGE_list(end_inputs); } jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL); jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL); if (!op1_indirect) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); } if (may_throw) { zend_jit_check_exception(jit); } return 1; } static int zend_jit_fe_reset(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) { zend_jit_addr res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); // JIT: ZVAL_COPY(res, value); if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); jit_ZVAL_COPY_CONST(jit, res_addr, MAY_BE_ANY, MAY_BE_ANY, zv, 1); } else { zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); jit_ZVAL_COPY(jit, res_addr, -1, op1_addr, op1_info, opline->op1_type == IS_CV); } // JIT: Z_FE_POS_P(res) = 0; ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->result.var + offsetof(zval, u2.fe_pos)), ir_CONST_U32(0)); return 1; } static int zend_jit_packed_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint32_t op_info) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_PACKED_GUARD); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); ir_ref ref; if (!exit_addr) { return 0; } ref = ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(jit_Z_PTR(jit, addr), offsetof(zend_array, u.flags))), ir_CONST_U32(HASH_FLAG_PACKED)); if (op_info & MAY_BE_ARRAY_PACKED) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_GUARD_NOT(ref, ir_CONST_ADDR(exit_addr)); } return 1; } static int zend_jit_fe_fetch(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, unsigned int target_label, uint8_t exit_opcode, const void *exit_addr) { zend_jit_addr op1_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op1.var); ir_ref ref, ht_ref, hash_pos_ref, packed_pos_ref, hash_p_ref = IR_UNUSED, packed_p_ref = IR_UNUSED, if_packed = IR_UNUSED; ir_ref if_def_hash = IR_UNUSED, if_def_packed = IR_UNUSED; ir_ref exit_inputs = IR_UNUSED; if (!MAY_BE_HASH(op1_info) && !MAY_BE_PACKED(op1_info)) { /* empty array */ if (exit_addr) { if (exit_opcode == ZEND_JMP) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } } else { zend_basic_block *bb; ZEND_ASSERT(jit->b >= 0); bb = &jit->ssa->cfg.blocks[jit->b]; _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ir_END()); jit->b = -1; } return 1; } // JIT: array = EX_VAR(opline->op1.var); // JIT: fe_ht = Z_ARRVAL_P(array); ht_ref = jit_Z_PTR(jit, op1_addr); if (op1_info & MAY_BE_PACKED_GUARD) { if (!zend_jit_packed_guard(jit, opline, opline->op1.var, op1_info)) { return 0; } } // JIT: pos = Z_FE_POS_P(array); hash_pos_ref = packed_pos_ref = ir_LOAD_U32(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos))); if (MAY_BE_HASH(op1_info)) { ir_ref loop_ref, pos2_ref, p2_ref; if (MAY_BE_PACKED(op1_info)) { ref = ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, u.flags))), ir_CONST_U32(HASH_FLAG_PACKED)); if_packed = ir_IF(ref); ir_IF_FALSE(if_packed); } // JIT: p = fe_ht->arData + pos; if (sizeof(void*) == 8) { ref = ir_ZEXT_A(hash_pos_ref); } else { ref = ir_BITCAST_A(hash_pos_ref); } hash_p_ref = ir_ADD_A( ir_MUL_A(ref, ir_CONST_ADDR(sizeof(Bucket))), ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arData)))); loop_ref = ir_LOOP_BEGIN(ir_END()); hash_pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, IR_UNUSED); hash_p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, IR_UNUSED); // JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { ref = ir_ULT(hash_pos_ref, ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed)))); // JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); // JIT: ZEND_VM_CONTINUE(); if (exit_addr) { if (exit_opcode == ZEND_JMP) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_ref if_fit = ir_IF(ref); ir_IF_FALSE(if_fit); ir_END_list(exit_inputs); ir_IF_TRUE(if_fit); } } else { ir_ref if_fit = ir_IF(ref); ir_IF_FALSE(if_fit); ir_END_list(exit_inputs); ir_IF_TRUE(if_fit); } // JIT: pos++; pos2_ref = ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1)); // JIT: value_type = Z_TYPE_INFO_P(value); // JIT: if (EXPECTED(value_type != IS_UNDEF)) { if (!exit_addr || exit_opcode == ZEND_JMP) { if_def_hash = ir_IF(jit_Z_TYPE_ref(jit, hash_p_ref)); ir_IF_FALSE(if_def_hash); } else { ir_GUARD_NOT(jit_Z_TYPE_ref(jit, hash_p_ref), ir_CONST_ADDR(exit_addr)); } // JIT: p++; p2_ref = ir_ADD_OFFSET(hash_p_ref, sizeof(Bucket)); ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END()); ir_PHI_SET_OP(hash_pos_ref, 2, pos2_ref); ir_PHI_SET_OP(hash_p_ref, 2, p2_ref); if (MAY_BE_PACKED(op1_info)) { ir_IF_TRUE(if_packed); } } if (MAY_BE_PACKED(op1_info)) { ir_ref loop_ref, pos2_ref, p2_ref; // JIT: p = fe_ht->arPacked + pos; if (sizeof(void*) == 8) { ref = ir_ZEXT_A(packed_pos_ref); } else { ref = ir_BITCAST_A(packed_pos_ref); } packed_p_ref = ir_ADD_A( ir_MUL_A(ref, ir_CONST_ADDR(sizeof(zval))), ir_LOAD_A(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, arPacked)))); loop_ref = ir_LOOP_BEGIN(ir_END()); packed_pos_ref = ir_PHI_2(IR_U32, packed_pos_ref, IR_UNUSED); packed_p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, IR_UNUSED); // JIT: if (UNEXPECTED(pos >= fe_ht->nNumUsed)) { ref = ir_ULT(packed_pos_ref, ir_LOAD_U32(ir_ADD_OFFSET(ht_ref, offsetof(zend_array, nNumUsed)))); // JIT: ZEND_VM_SET_RELATIVE_OPCODE(opline, opline->extended_value); // JIT: ZEND_VM_CONTINUE(); if (exit_addr) { if (exit_opcode == ZEND_JMP) { ir_GUARD(ref, ir_CONST_ADDR(exit_addr)); } else { ir_ref if_fit = ir_IF(ref); ir_IF_FALSE(if_fit); ir_END_list(exit_inputs); ir_IF_TRUE(if_fit); } } else { ir_ref if_fit = ir_IF(ref); ir_IF_FALSE(if_fit); ir_END_list(exit_inputs); ir_IF_TRUE(if_fit); } // JIT: pos++; pos2_ref = ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1)); // JIT: value_type = Z_TYPE_INFO_P(value); // JIT: if (EXPECTED(value_type != IS_UNDEF)) { if (!exit_addr || exit_opcode == ZEND_JMP) { if_def_packed = ir_IF(jit_Z_TYPE_ref(jit, packed_p_ref)); ir_IF_FALSE(if_def_packed); } else { ir_GUARD_NOT(jit_Z_TYPE_ref(jit, packed_p_ref), ir_CONST_ADDR(exit_addr)); } // JIT: p++; p2_ref = ir_ADD_OFFSET(packed_p_ref, sizeof(zval)); ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END()); ir_PHI_SET_OP(packed_pos_ref, 2, pos2_ref); ir_PHI_SET_OP(packed_p_ref, 2, p2_ref); } if (!exit_addr || exit_opcode == ZEND_JMP) { zend_jit_addr val_addr; zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->op2.var); uint32_t val_info; ir_ref p_ref = IR_UNUSED, hash_path = IR_UNUSED; if (RETURN_VALUE_USED(opline)) { zend_jit_addr res_addr = RES_ADDR(); if (MAY_BE_HASH(op1_info)) { ir_ref key_ref = IR_UNUSED, if_key = IR_UNUSED, key_path = IR_UNUSED; ZEND_ASSERT(if_def_hash); ir_IF_TRUE(if_def_hash); // JIT: Z_FE_POS_P(array) = pos + 1; ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)), ir_ADD_U32(hash_pos_ref, ir_CONST_U32(1))); if (op1_info & MAY_BE_ARRAY_KEY_STRING) { key_ref = ir_LOAD_A(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, key))); } if ((op1_info & MAY_BE_ARRAY_KEY_LONG) && (op1_info & MAY_BE_ARRAY_KEY_STRING)) { // JIT: if (!p->key) { if_key = ir_IF(key_ref); ir_IF_TRUE(if_key); } if (op1_info & MAY_BE_ARRAY_KEY_STRING) { ir_ref if_interned, interned_path; // JIT: ZVAL_STR_COPY(EX_VAR(opline->result.var), p->key); jit_set_Z_PTR(jit, res_addr, key_ref); ref = ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(key_ref, offsetof(zend_refcounted, gc.u.type_info))), ir_CONST_U32(IS_STR_INTERNED)); if_interned = ir_IF(ref); ir_IF_TRUE(if_interned); jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING); interned_path = ir_END(); ir_IF_FALSE(if_interned); jit_GC_ADDREF(jit, key_ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_STRING_EX); ir_MERGE_WITH(interned_path); if (op1_info & MAY_BE_ARRAY_KEY_LONG) { key_path = ir_END(); } } if (op1_info & MAY_BE_ARRAY_KEY_LONG) { if (op1_info & MAY_BE_ARRAY_KEY_STRING) { ir_IF_FALSE(if_key); } // JIT: ZVAL_LONG(EX_VAR(opline->result.var), p->h); ref = ir_LOAD_L(ir_ADD_OFFSET(hash_p_ref, offsetof(Bucket, h))); jit_set_Z_LVAL(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); if (op1_info & MAY_BE_ARRAY_KEY_STRING) { ir_MERGE_WITH(key_path); } } if (MAY_BE_PACKED(op1_info)) { hash_path = ir_END(); } else { p_ref = hash_p_ref; } } if (MAY_BE_PACKED(op1_info)) { ZEND_ASSERT(if_def_packed); ir_IF_TRUE(if_def_packed); // JIT: Z_FE_POS_P(array) = pos + 1; ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)), ir_ADD_U32(packed_pos_ref, ir_CONST_U32(1))); // JIT: ZVAL_LONG(EX_VAR(opline->result.var), pos); if (sizeof(zend_long) == 8) { packed_pos_ref = ir_ZEXT_L(packed_pos_ref); } else { packed_pos_ref = ir_BITCAST_L(packed_pos_ref); } jit_set_Z_LVAL(jit, res_addr, packed_pos_ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); if (MAY_BE_HASH(op1_info)) { ir_MERGE_WITH(hash_path); p_ref = ir_PHI_2(IR_ADDR, packed_p_ref, hash_p_ref); } else { p_ref = packed_p_ref; } } } else { ir_ref pos_ref = IR_UNUSED; if (if_def_hash && if_def_packed) { ir_IF_TRUE(if_def_hash); ir_MERGE_WITH_EMPTY_TRUE(if_def_packed); pos_ref = ir_PHI_2(IR_U32, hash_pos_ref, packed_pos_ref); p_ref = ir_PHI_2(IR_ADDR, hash_p_ref, packed_p_ref); } else if (if_def_hash) { ir_IF_TRUE(if_def_hash); pos_ref = hash_pos_ref; p_ref = hash_p_ref; } else if (if_def_packed) { ir_IF_TRUE(if_def_packed); pos_ref = packed_pos_ref; p_ref = packed_p_ref; } else { ZEND_UNREACHABLE(); } // JIT: Z_FE_POS_P(array) = pos + 1; ir_STORE(ir_ADD_OFFSET(jit_FP(jit), opline->op1.var + offsetof(zval, u2.fe_pos)), ir_ADD_U32(pos_ref, ir_CONST_U32(1))); } val_info = ((op1_info & MAY_BE_ARRAY_OF_ANY) >> MAY_BE_ARRAY_SHIFT); if (val_info & MAY_BE_ARRAY) { val_info |= MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } if (op1_info & MAY_BE_ARRAY_OF_REF) { val_info |= MAY_BE_REF | MAY_BE_RC1 | MAY_BE_RCN | MAY_BE_ANY | MAY_BE_ARRAY_KEY_ANY | MAY_BE_ARRAY_OF_ANY | MAY_BE_ARRAY_OF_REF; } else if (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { val_info |= MAY_BE_RC1 | MAY_BE_RCN; } val_addr = ZEND_ADDR_REF_ZVAL(p_ref); if (opline->op2_type == IS_CV) { // JIT: zend_assign_to_variable(variable_ptr, value, IS_CV, EX_USES_STRICT_TYPES()); if (!zend_jit_assign_to_variable(jit, opline, var_addr, var_addr, op2_info, -1, IS_CV, val_addr, val_info, 0, 0, 1)) { return 0; } } else { // JIT: ZVAL_COPY(res, value); jit_ZVAL_COPY(jit, var_addr, -1, val_addr, val_info, 1); } if (!exit_addr) { zend_basic_block *bb; ZEND_ASSERT(jit->b >= 0); bb = &jit->ssa->cfg.blocks[jit->b]; _zend_jit_add_predecessor_ref(jit, bb->successors[1], jit->b, ir_END()); ZEND_ASSERT(exit_inputs); if (!jit->ctx.ir_base[exit_inputs].op2) { ref = exit_inputs; } else { ir_MERGE_list(exit_inputs); ref = ir_END(); } _zend_jit_add_predecessor_ref(jit, bb->successors[0], jit->b, ref); jit->b = -1; } } else { ZEND_ASSERT(exit_inputs); ir_MERGE_list(exit_inputs); } return 1; } static int zend_jit_load_this(zend_jit_ctx *jit, uint32_t var) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); zend_jit_addr var_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); ir_ref ref = jit_Z_PTR(jit, this_addr); jit_set_Z_PTR(jit, var_addr, ref); jit_set_Z_TYPE_INFO(jit, var_addr, IS_OBJECT_EX); jit_GC_ADDREF(jit, ref); return 1; } static int zend_jit_fetch_this(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, bool check_only) { if (!op_array->scope || (op_array->fn_flags & ZEND_ACC_STATIC) || ((op_array->fn_flags & (ZEND_ACC_CLOSURE|ZEND_ACC_IMMUTABLE)) == ZEND_ACC_CLOSURE)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (!JIT_G(current_frame) || !TRACE_FRAME_IS_THIS_CHECKED(JIT_G(current_frame))) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_Z_TYPE(jit, this_addr, IS_OBJECT, exit_addr); if (JIT_G(current_frame)) { TRACE_FRAME_SET_THIS_CHECKED(JIT_G(current_frame)); } } } else { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); ir_ref if_object = jit_if_Z_TYPE(jit, this_addr, IS_OBJECT); ir_IF_FALSE_cold(if_object); jit_SET_EX_OPLINE(jit, opline); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_invalid_this)); ir_IF_TRUE(if_object); } } if (!check_only) { if (!zend_jit_load_this(jit, opline->result.var)) { return 0; } } return 1; } static int zend_jit_class_guard(zend_jit_ctx *jit, const zend_op *opline, ir_ref obj_ref, zend_class_entry *ce) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(ir_EQ(ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))), ir_CONST_ADDR(ce)), ir_CONST_ADDR(exit_addr)); return 1; } static int zend_jit_fetch_obj(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, bool op1_avoid_refcounting, zend_class_entry *trace_ce, zend_jit_addr res_addr, uint8_t prop_type, int may_throw) { zval *member; zend_property_info *prop_info; bool may_be_dynamic = 1; zend_jit_addr prop_addr; uint32_t res_info = RES_INFO(); ir_ref prop_type_ref = IR_UNUSED; ir_ref obj_ref = IR_UNUSED; ir_ref prop_ref = IR_UNUSED; ir_ref end_inputs = IR_UNUSED; ir_ref slow_inputs = IR_UNUSED; ir_ref end_values = IR_UNUSED; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); prop_info = zend_get_known_property_info(op_array, ce, Z_STR_P(member), on_this, op_array->filename); if (on_this) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); obj_ref = jit_Z_PTR(jit, this_addr); } else { if (opline->op1_type == IS_VAR && opline->opcode == ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr); } if (op1_info & MAY_BE_REF) { op1_addr = jit_ZVAL_DEREF(jit, op1_addr); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr); } else { ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT); ir_IF_FALSE_cold(if_obj); if (opline->opcode != ZEND_FETCH_OBJ_IS) { ir_ref op1_ref = IR_UNUSED; jit_SET_EX_OPLINE(jit, opline); if (opline->opcode != ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_UNDEF)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); ir_ref fast_path = IR_UNUSED; if (op1_info & MAY_BE_ANY) { ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr)); ir_IF_TRUE(if_def); fast_path = ir_END(); ir_IF_FALSE_cold(if_def); } ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var)); if (fast_path) { ir_MERGE_WITH(fast_path); } op1_ref = jit_ZVAL_ADDR(jit, orig_op1_addr); } else { op1_ref = jit_ZVAL_ADDR(jit, op1_addr); } if (opline->opcode == ZEND_FETCH_OBJ_W) { ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_write), op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member))); jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR); } else { ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_read), op1_ref, ir_CONST_ADDR(Z_STRVAL_P(member))); jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); } } else { jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); } ir_END_list(end_inputs); ir_IF_TRUE(if_obj); } } obj_ref = jit_Z_PTR(jit, op1_addr); } ZEND_ASSERT(obj_ref); if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, Z_STR_P(member), on_this, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (on_this && JIT_G(current_frame) && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { ZEND_ASSERT(JIT_G(current_frame)->ce == ce); } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) { if (on_this && JIT_G(current_frame)) { JIT_G(current_frame)->ce = ce; TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); } } else { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } } } } if (!prop_info) { ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)); ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))))); ir_IF_FALSE_cold(if_same); ir_END_list(slow_inputs); ir_IF_TRUE(if_same); ir_ref offset_ref = ir_LOAD_A( ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*))); may_be_dynamic = zend_may_be_dynamic_property(ce, Z_STR_P(member), opline->op1_type == IS_UNUSED, op_array); if (may_be_dynamic) { ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET))); if (opline->opcode == ZEND_FETCH_OBJ_W) { ir_IF_TRUE_cold(if_dynamic); ir_END_list(slow_inputs); } else { ir_IF_TRUE_cold(if_dynamic); jit_SET_EX_OPLINE(jit, opline); if (opline->opcode != ZEND_FETCH_OBJ_IS) { if (Z_MODE(res_addr) == IS_REG) { ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic_ex), obj_ref, offset_ref); ir_END_PHI_list(end_values, val_addr); } else { ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_dynamic), obj_ref, offset_ref); ir_END_list(end_inputs); } } else { if (Z_MODE(res_addr) == IS_REG) { ir_ref val_addr = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic_ex), obj_ref, offset_ref); ir_END_PHI_list(end_values, val_addr); } else { ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_dynamic), obj_ref, offset_ref); ir_END_list(end_inputs); } } } ir_IF_FALSE(if_dynamic); } prop_ref = ir_ADD_A(obj_ref, offset_ref); prop_type_ref = jit_Z_TYPE_ref(jit, prop_ref); ir_ref if_def = ir_IF(prop_type_ref); ir_IF_FALSE_cold(if_def); ir_END_list(slow_inputs); ir_IF_TRUE(if_def); prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref); if (opline->opcode == ZEND_FETCH_OBJ_W && (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT)))) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; ir_ref allowed_inputs = IR_UNUSED; ir_ref forbidden_inputs = IR_UNUSED; ir_ref prop_info_ref = ir_LOAD_A( ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2)); ir_ref if_has_prop_info = ir_IF(prop_info_ref); ir_IF_TRUE_cold(if_has_prop_info); ir_ref prop_flags = ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, flags))); ir_ref if_readonly_or_avis = ir_IF(ir_AND_U32(prop_flags, ir_CONST_U32(ZEND_ACC_READONLY|ZEND_ACC_PPP_SET_MASK))); ir_IF_FALSE(if_readonly_or_avis); ir_END_list(allowed_inputs); ir_IF_TRUE_cold(if_readonly_or_avis); ir_ref if_readonly = ir_IF(ir_AND_U32(prop_flags, ir_CONST_U32(ZEND_ACC_READONLY))); ir_IF_TRUE(if_readonly); ir_END_list(forbidden_inputs); ir_IF_FALSE(if_readonly); ir_ref has_avis_access = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_asymmetric_property_has_set_access), prop_info_ref); ir_ref if_avis_access = ir_IF(has_avis_access); ir_IF_TRUE(if_avis_access); ir_END_list(allowed_inputs); ir_IF_FALSE(if_avis_access); ir_END_list(forbidden_inputs); ir_MERGE_list(forbidden_inputs); ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT); ir_IF_TRUE(if_prop_obj); ref = jit_Z_PTR(jit, prop_addr); jit_GC_ADDREF(jit, ref); jit_set_Z_PTR(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX); ir_END_list(end_inputs); ir_IF_FALSE_cold(if_prop_obj); jit_SET_EX_OPLINE(jit, opline); if_readonly = ir_IF(ir_AND_U32(prop_flags, ir_CONST_U32(ZEND_ACC_READONLY))); ir_IF_TRUE(if_readonly); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_indirect_modification_error), prop_info_ref); jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR); ir_END_list(end_inputs); ir_IF_FALSE(if_readonly); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_asymmetric_visibility_property_modification_error), prop_info_ref, ir_CONST_ADDR("indirectly modify")); jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR); ir_END_list(end_inputs); ir_MERGE_list(allowed_inputs); if (flags == ZEND_FETCH_DIM_WRITE) { jit_SET_EX_OPLINE(jit, opline); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion), prop_ref, prop_info_ref); ir_END_list(end_inputs); ir_IF_FALSE(if_has_prop_info); } else if (flags == ZEND_FETCH_REF) { ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref), prop_ref, prop_info_ref, jit_ZVAL_ADDR(jit, res_addr)); ir_END_list(end_inputs); ir_IF_FALSE(if_has_prop_info); } else { ZEND_ASSERT(flags == 0); ir_MERGE_WITH_EMPTY_FALSE(if_has_prop_info); } } } else { prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset); prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { if (opline->opcode == ZEND_FETCH_OBJ_W || !(res_info & MAY_BE_GUARD) || !JIT_G(current_frame)) { /* perform IS_UNDEF check only after result type guard (during deoptimization) */ int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr); ir_GUARD(prop_type_ref, ir_CONST_ADDR(exit_addr)); } } else { prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr); ir_ref if_def = ir_IF(prop_type_ref); ir_IF_FALSE_cold(if_def); ir_END_list(slow_inputs); ir_IF_TRUE(if_def); } if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_READONLY)) { ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT); ir_IF_TRUE(if_prop_obj); ir_ref ref = jit_Z_PTR(jit, prop_addr); jit_GC_ADDREF(jit, ref); jit_set_Z_PTR(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX); ir_END_list(end_inputs); ir_IF_FALSE_cold(if_prop_obj); jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_readonly_property_indirect_modification_error), ir_CONST_ADDR(prop_info)); jit_set_Z_TYPE_INFO(jit, res_addr, _IS_ERROR); ir_END_list(end_inputs); goto result_fetched; } else if (opline->opcode == ZEND_FETCH_OBJ_W && (prop_info->flags & ZEND_ACC_PPP_SET_MASK)) { /* Readonly properties which are also asymmetric are never mutable indirectly, which is * handled by the previous branch. */ ir_ref has_access = ir_CALL_1(IR_BOOL, ir_CONST_FC_FUNC(zend_asymmetric_property_has_set_access), ir_CONST_ADDR(prop_info)); ir_ref if_access = ir_IF(has_access); ir_IF_FALSE_cold(if_access); ir_ref if_prop_obj = jit_if_Z_TYPE(jit, prop_addr, IS_OBJECT); ir_IF_TRUE(if_prop_obj); ir_ref ref = jit_Z_PTR(jit, prop_addr); jit_GC_ADDREF(jit, ref); jit_set_Z_PTR(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_OBJECT_EX); ir_END_list(end_inputs); ir_IF_FALSE_cold(if_prop_obj); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_asymmetric_visibility_property_modification_error), ir_CONST_ADDR(prop_info), ir_CONST_ADDR("indirectly modify")); ir_END_list(end_inputs); ir_IF_TRUE(if_access); } if (opline->opcode == ZEND_FETCH_OBJ_W && (opline->extended_value & ZEND_FETCH_OBJ_FLAGS) && ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; if (flags == ZEND_FETCH_DIM_WRITE) { if ((ZEND_TYPE_FULL_MASK(prop_info->type) & MAY_BE_ARRAY) == 0) { if (!prop_type_ref) { prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr); } ir_ref if_null_or_flase = ir_IF(ir_LE(prop_type_ref, ir_CONST_U32(IR_FALSE))); ir_IF_TRUE_cold(if_null_or_flase); jit_SET_EX_OPLINE(jit, opline); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_check_array_promotion), prop_ref, ir_CONST_ADDR(prop_info)); ir_END_list(end_inputs); ir_IF_FALSE(if_null_or_flase); } } else if (flags == ZEND_FETCH_REF) { ir_ref ref; if (!prop_type_ref) { prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr); } ir_ref if_reference = ir_IF(ir_EQ(prop_type_ref, ir_CONST_U32(IS_REFERENCE_EX))); ir_IF_FALSE(if_reference); if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { ref = ir_CONST_ADDR(prop_info); } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))); ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table))); ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset)); } ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_create_typed_ref), prop_ref, ref, jit_ZVAL_ADDR(jit, res_addr)); ir_END_list(end_inputs); ir_IF_TRUE(if_reference); } else { ZEND_UNREACHABLE(); } } } if (opline->opcode == ZEND_FETCH_OBJ_W) { ZEND_ASSERT(prop_ref); jit_set_Z_PTR(jit, res_addr, prop_ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT); if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info) { ssa->var_info[ssa_op->result_def].indirect_reference = 1; } ir_END_list(end_inputs); } else { if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame) && prop_info) { ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr)); } else if ((res_info & MAY_BE_GUARD) && Z_MODE(res_addr) == IS_REG) { ir_END_PHI_list(end_values, jit_ZVAL_ADDR(jit, prop_addr)); } else if (Z_MODE(res_addr) == IS_REG) { prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr); if (!zend_jit_zval_copy_deref_reg(jit, res_addr, res_info & ~MAY_BE_GUARD, prop_addr, prop_type_ref, &end_values)) { return 0; } } else { prop_type_ref = jit_Z_TYPE_INFO(jit, prop_addr); if (!zend_jit_zval_copy_deref(jit, res_addr, prop_addr, prop_type_ref)) { return 0; } ir_END_list(end_inputs); } } result_fetched: if (op1_avoid_refcounting) { SET_STACK_REG(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), ZREG_NONE); } if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE || !prop_info) { ir_MERGE_list(slow_inputs); jit_SET_EX_OPLINE(jit, opline); op1_info |= MAY_BE_RC1 | MAY_BE_RCN; /* object may be captured/released in magic handler */ if (opline->opcode == ZEND_FETCH_OBJ_W) { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_w_slow), obj_ref); ir_END_list(end_inputs); } else if (opline->opcode != ZEND_FETCH_OBJ_IS) { if (Z_MODE(res_addr) == IS_REG) { ir_ref val_ref = ir_CALL_1(IR_ADDR, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow_ex), obj_ref); ir_END_PHI_list(end_values, val_ref); } else { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_r_slow), obj_ref); ir_END_list(end_inputs); } } else { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_fetch_obj_is_slow), obj_ref); ir_END_list(end_inputs); } } if (end_values) { ir_ref val_ref = ir_PHI_list(end_values); zend_jit_addr val_addr = ZEND_ADDR_REF_ZVAL(val_ref); bool result_avoid_refcounting = 0; ZEND_ASSERT(opline->opcode == ZEND_FETCH_OBJ_R || opline->opcode == ZEND_FETCH_OBJ_FUNC_ARG || opline->opcode == ZEND_FETCH_OBJ_IS); ZEND_ASSERT(end_inputs == IR_UNUSED); if ((res_info & MAY_BE_GUARD) && JIT_G(current_frame)) { uint8_t type = concrete_type(res_info); uint32_t flags = ZEND_JIT_EXIT_CHECK_EXCEPTION; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this && !op1_avoid_refcounting) { flags |= ZEND_JIT_EXIT_FREE_OP1; } if ((opline->result_type & (IS_VAR|IS_TMP_VAR)) && !(flags & ZEND_JIT_EXIT_FREE_OP1) && (res_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) && (ssa_op+1)->op1_use == ssa_op->result_def && zend_jit_may_avoid_refcounting(opline+1, res_info)) { result_avoid_refcounting = 1; ssa->var_info[ssa_op->result_def].avoid_refcounting = 1; } val_addr = zend_jit_guard_fetch_result_type(jit, opline, val_addr, type, 1, flags, op1_avoid_refcounting); if (!val_addr) { return 0; } res_info &= ~MAY_BE_GUARD; ssa->var_info[ssa_op->result_def].type &= ~MAY_BE_GUARD; } // ZVAL_COPY jit_ZVAL_COPY(jit, res_addr, -1, val_addr, res_info, !result_avoid_refcounting); if (!zend_jit_store_var_if_necessary(jit, opline->result.var, res_addr, res_info)) { return 0; } } else { ir_MERGE_list(end_inputs); } if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { if (opline->op1_type == IS_VAR && opline->opcode == ZEND_FETCH_OBJ_W && (op1_info & MAY_BE_RC1)) { zend_jit_addr orig_op1_addr = OP1_ADDR(); ir_ref if_refcounted, ptr, refcount, if_non_zero; ir_ref merge_inputs = IR_UNUSED; if_refcounted = jit_if_REFCOUNTED(jit, orig_op1_addr); ir_IF_FALSE( if_refcounted); ir_END_list(merge_inputs); ir_IF_TRUE( if_refcounted); ptr = jit_Z_PTR(jit, orig_op1_addr); refcount = jit_GC_DELREF(jit, ptr); if_non_zero = ir_IF(refcount); ir_IF_TRUE( if_non_zero); ir_END_list(merge_inputs); ir_IF_FALSE( if_non_zero); jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_extract_helper), ptr); ir_END_list(merge_inputs); ir_MERGE_list(merge_inputs); } else if (!op1_avoid_refcounting) { if (on_this) { op1_info &= ~MAY_BE_RC1; } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } } if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_info && (opline->opcode != ZEND_FETCH_OBJ_W || !(opline->extended_value & ZEND_FETCH_OBJ_FLAGS) || !ZEND_TYPE_IS_SET(prop_info->type)) && (!(opline->op1_type & (IS_VAR|IS_TMP_VAR)) || on_this || op1_indirect)) { may_throw = 0; } if (may_throw) { if (Z_MODE(res_addr) == IS_REG) { zend_jit_check_exception_undef_result(jit, opline); } else { zend_jit_check_exception(jit); } } return 1; } static int zend_jit_assign_obj(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t val_info, zend_jit_addr val_addr, zend_jit_addr val_def_addr, zend_jit_addr res_addr, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, zend_class_entry *trace_ce, uint8_t prop_type, int may_throw) { zval *member; zend_string *name; zend_property_info *prop_info; zend_jit_addr prop_addr; ir_ref obj_ref = IR_UNUSED; ir_ref prop_ref = IR_UNUSED; ir_ref delayed_end_input = IR_UNUSED; ir_ref end_inputs = IR_UNUSED; ir_ref slow_inputs = IR_UNUSED; uint32_t res_info = RES_INFO(); if (Z_MODE(val_addr) == IS_REG && Z_LOAD(val_addr) && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, val_addr); } if (val_addr != val_def_addr && val_def_addr) { if (!zend_jit_update_regs(jit, (opline+1)->op1.var, val_addr, val_def_addr, val_info)) { return 0; } if (Z_MODE(val_def_addr) == IS_REG && Z_MODE(val_addr) != IS_REG) { val_addr = val_def_addr; } } ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); name = Z_STR_P(member); prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); obj_ref = jit_Z_PTR(jit, this_addr); } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr); } if (op1_info & MAY_BE_REF) { op1_addr = jit_ZVAL_DEREF(jit, op1_addr); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr); } else { ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT); ir_IF_FALSE_cold(if_obj); jit_SET_EX_OPLINE(jit, opline); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_assign), jit_ZVAL_ADDR(jit, op1_addr), ir_CONST_ADDR(ZSTR_VAL(name))); if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) != IS_REG) { jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); } ir_END_list(end_inputs); ir_IF_TRUE(if_obj); } } obj_ref = jit_Z_PTR(jit, op1_addr); } ZEND_ASSERT(obj_ref); if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (on_this && JIT_G(current_frame) && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { ZEND_ASSERT(JIT_G(current_frame)->ce == ce); } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) { if (on_this && JIT_G(current_frame)) { JIT_G(current_frame)->ce = ce; TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); } } else { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } if (ssa->var_info && ssa_op->op1_def >= 0) { ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_def].ce = ce; ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; } } } } if (!prop_info) { ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)); ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))))); ir_IF_FALSE_cold(if_same); ir_END_list(slow_inputs); ir_IF_TRUE(if_same); ir_ref offset_ref = ir_LOAD_A( ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*))); ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET))); ir_IF_TRUE_cold(if_dynamic); ir_END_list(slow_inputs); ir_IF_FALSE(if_dynamic); prop_ref = ir_ADD_A(obj_ref, offset_ref); ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref)); ir_IF_FALSE_cold(if_def); ir_END_list(slow_inputs); ir_IF_TRUE(if_def); prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref); if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { ir_ref arg3, arg4; ir_ref prop_info_ref = ir_LOAD_A( ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2)); ir_ref if_has_prop_info = ir_IF(prop_info_ref); ir_IF_TRUE_cold(if_has_prop_info); if (Z_MODE(val_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var); if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) { return 0; } arg3 = jit_ZVAL_ADDR(jit, real_addr); } else { arg3 = jit_ZVAL_ADDR(jit, val_addr); } if (!RETURN_VALUE_USED(opline)) { arg4 = IR_NULL; } else if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); arg4 = jit_ZVAL_ADDR(jit, real_addr); } else { arg4 = jit_ZVAL_ADDR(jit, res_addr); } // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); jit_SET_EX_OPLINE(jit, opline); ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop), prop_ref, prop_info_ref, arg3, arg4); if ((opline+1)->op1_type == IS_CONST) { // TODO: ??? // if (Z_TYPE_P(value) == orig_type) { // CACHE_PTR_EX(cache_slot + 2, NULL); } ir_END_list(end_inputs); ir_IF_FALSE(if_has_prop_info); } } else { prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset); prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref); /* With the exception of __clone(), readonly assignment always happens on IS_UNDEF, doding * the fast path. Thus, the fast path is not useful. */ if (prop_info->flags & ZEND_ACC_READONLY) { ZEND_ASSERT(slow_inputs == IR_UNUSED); goto slow_path; } // Undefined property with potential magic __get()/__set() or lazy object if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE && prop_type != IS_UNDEF) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr)); } else { ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr)); ir_IF_FALSE_cold(if_def); ir_END_list(slow_inputs); ir_IF_TRUE(if_def); } if (ZEND_TYPE_IS_SET(prop_info->type)) { ir_ref ref, arg3, arg4; // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); jit_SET_EX_OPLINE(jit, opline); if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { ref = ir_CONST_ADDR(prop_info); } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))); ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table))); ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset)); } if (Z_MODE(val_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var); if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) { return 0; } arg3 = jit_ZVAL_ADDR(jit, real_addr); } else { arg3 = jit_ZVAL_ADDR(jit, val_addr); } if (!RETURN_VALUE_USED(opline)) { arg4 = IR_NULL; } else if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); arg4 = jit_ZVAL_ADDR(jit, real_addr); } else { arg4 = jit_ZVAL_ADDR(jit, res_addr); } ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_to_typed_prop), prop_ref, ref, arg3, arg4); ir_END_list(end_inputs); } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { if (Z_MODE(val_addr) != IS_REG && (res_addr == 0 || Z_MODE(res_addr) != IS_REG) && opline->result_type == IS_UNUSED) { if (!zend_jit_assign_to_variable_call(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, res_addr, 0)) { return 0; } } else { zend_jit_addr real_res_addr; if (res_addr && Z_MODE(res_addr) == IS_REG) { real_res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } else { real_res_addr = res_addr; } if (!zend_jit_assign_to_variable(jit, opline, prop_addr, prop_addr, -1, -1, (opline+1)->op1_type, val_addr, val_info, real_res_addr, 0, 0)) { return 0; } } if (end_inputs || slow_inputs) { if (((opline+1)->op1_type & (IS_VAR|IS_TMP_VAR)) && (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { /* skip FREE_OP_DATA() */ delayed_end_input = ir_END(); } else { ir_END_list(end_inputs); } } } if (slow_inputs) { ir_ref arg3, arg5; ir_MERGE_list(slow_inputs); slow_path: jit_SET_EX_OPLINE(jit, opline); if (Z_MODE(val_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var); if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) { return 0; } arg3 = jit_ZVAL_ADDR(jit, real_addr); } else { arg3 = jit_ZVAL_ADDR(jit, val_addr); } if (!RETURN_VALUE_USED(opline)) { arg5 = IR_NULL; } else if (Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); arg5 = jit_ZVAL_ADDR(jit, real_addr); } else { arg5 = jit_ZVAL_ADDR(jit, res_addr); } // JIT: value = zobj->handlers->write_property(zobj, name, value, CACHE_ADDR(opline->extended_value)); ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_helper), obj_ref, ir_CONST_ADDR(name), arg3, ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), arg5); ir_END_list(end_inputs); } if (end_inputs) { ir_MERGE_list(end_inputs); if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { val_info |= MAY_BE_RC1|MAY_BE_RCN; } jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline); if (delayed_end_input) { ir_MERGE_WITH(delayed_end_input); } } if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } if (RETURN_VALUE_USED(opline) && Z_MODE(res_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); if (!zend_jit_load_reg(jit, real_addr, res_addr, res_info)) { return 0; } } if (may_throw) { zend_jit_check_exception(jit); } return 1; } static int zend_jit_assign_obj_op(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, uint32_t val_info, zend_jit_addr val_addr, zend_ssa_range *val_range, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, zend_class_entry *trace_ce, uint8_t prop_type) { zval *member; zend_string *name; zend_property_info *prop_info; zend_jit_addr prop_addr; bool use_prop_guard = 0; bool may_throw = 0; binary_op_type binary_op = get_binary_op(opline->extended_value); ir_ref obj_ref = IR_UNUSED; ir_ref prop_ref = IR_UNUSED; ir_ref end_inputs = IR_UNUSED; ir_ref slow_inputs = IR_UNUSED; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); ZEND_ASSERT(opline->result_type == IS_UNUSED); member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); name = Z_STR_P(member); prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); obj_ref = jit_Z_PTR(jit, this_addr); } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr); } if (op1_info & MAY_BE_REF) { op1_addr = jit_ZVAL_DEREF(jit, op1_addr); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr); } else { ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT); ir_IF_FALSE_cold(if_obj); jit_SET_EX_OPLINE(jit, opline); ir_CALL_2(IR_VOID, (op1_info & MAY_BE_UNDEF) ? ir_CONST_FC_FUNC(zend_jit_invalid_property_assign_op) : ir_CONST_FC_FUNC(zend_jit_invalid_property_assign), jit_ZVAL_ADDR(jit, op1_addr), ir_CONST_ADDR(ZSTR_VAL(name))); may_throw = 1; ir_END_list(end_inputs); ir_IF_TRUE(if_obj); } } obj_ref = jit_Z_PTR(jit, op1_addr); } ZEND_ASSERT(obj_ref); if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (on_this && JIT_G(current_frame) && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { ZEND_ASSERT(JIT_G(current_frame)->ce == ce); } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) { if (on_this && JIT_G(current_frame)) { JIT_G(current_frame)->ce = ce; TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); } } else { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } if (ssa->var_info && ssa_op->op1_def >= 0) { ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_def].ce = ce; ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; } } } } use_prop_guard = (prop_type != IS_UNKNOWN && prop_type != IS_UNDEF && prop_type != IS_REFERENCE && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); if (Z_MODE(val_addr) == IS_REG && Z_LOAD(val_addr) && jit->ra[Z_SSA_VAR(val_addr)].ref == IR_NULL) { /* Force load */ zend_jit_use_reg(jit, val_addr); } if (!prop_info) { ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS)); ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))))); ir_IF_FALSE_cold(if_same); ir_END_list(slow_inputs); ir_IF_TRUE(if_same); if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { ir_ref prop_info_ref = ir_LOAD_A( ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2)); ir_ref if_has_prop_info = ir_IF(prop_info_ref); ir_IF_TRUE_cold(if_has_prop_info); ir_END_list(slow_inputs); ir_IF_FALSE(if_has_prop_info); } ir_ref offset_ref = ir_LOAD_A( ir_ADD_OFFSET(run_time_cache, ((opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*))); ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET))); ir_IF_TRUE_cold(if_dynamic); ir_END_list(slow_inputs); ir_IF_FALSE(if_dynamic); prop_ref = ir_ADD_A(obj_ref, offset_ref); if (!use_prop_guard) { ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref)); ir_IF_FALSE_cold(if_def); ir_END_list(slow_inputs); ir_IF_TRUE(if_def); } prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref); } else { prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset); prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref); if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr)); } else { ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr)); ir_IF_FALSE_cold(if_def); ir_END_list(slow_inputs); ir_IF_TRUE(if_def); } } if (ZEND_TYPE_IS_SET(prop_info->type)) { ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2; may_throw = 1; jit_SET_EX_OPLINE(jit, opline); if (Z_MODE(val_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var); if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) { return 0; } arg2 = jit_ZVAL_ADDR(jit, real_addr); } else { arg2 = jit_ZVAL_ADDR(jit, val_addr); } if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE); ir_IF_FALSE(if_ref); noref_path = ir_END(); ir_IF_TRUE(if_ref); reference = jit_Z_PTR(jit, prop_addr); ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val)); if_typed = jit_if_TYPED_REF(jit, reference); ir_IF_FALSE(if_typed); ref_path = ir_END(); ir_IF_TRUE_cold(if_typed); ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref), reference, arg2, ir_CONST_FC_FUNC(binary_op)); ir_END_list(end_inputs); ir_MERGE_2(noref_path, ref_path); prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref); prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref); // JIT: value = zend_assign_to_typed_prop(prop_info, property_val, value EXECUTE_DATA_CC); if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { ref = ir_CONST_ADDR(prop_info); } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))); ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table))); ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset)); } ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_prop), prop_ref, ref, arg2, ir_CONST_FC_FUNC(binary_op)); ir_END_list(end_inputs); } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { zend_jit_addr var_addr = prop_addr; uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; uint32_t var_def_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; if (use_prop_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr); var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); } if (var_info & MAY_BE_REF) { ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref, arg2; may_throw = 1; if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE); ir_IF_FALSE(if_ref); noref_path = ir_END(); ir_IF_TRUE(if_ref); reference = jit_Z_PTR(jit, var_addr); ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val)); if_typed = jit_if_TYPED_REF(jit, reference); ir_IF_FALSE(if_typed); ref_path = ir_END(); ir_IF_TRUE_cold(if_typed); jit_SET_EX_OPLINE(jit, opline); if (Z_MODE(val_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var); if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) { return 0; } arg2 = jit_ZVAL_ADDR(jit, real_addr); } else { arg2 = jit_ZVAL_ADDR(jit, val_addr); } ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_op_to_typed_ref), reference, arg2, ir_CONST_FC_FUNC(binary_op)); ir_END_list(end_inputs); ir_MERGE_2(noref_path, ref_path); prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref); var_addr = ZEND_ADDR_REF_ZVAL(prop_ref); var_info &= ~MAY_BE_REF; } uint8_t val_op_type = (opline+1)->op1_type; if (val_op_type & (IS_TMP_VAR|IS_VAR)) { /* prevent FREE_OP in the helpers */ val_op_type = IS_CV; } switch (opline->extended_value) { case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if (opline->extended_value != ZEND_ADD || (var_info & MAY_BE_ANY) != MAY_BE_ARRAY || (val_info & MAY_BE_ANY) == MAY_BE_ARRAY) { may_throw = 1; } } if (!zend_jit_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, 0, var_addr, var_def_info, var_info, 1 /* may overflow */, 0)) { return 0; } break; case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { if ((var_info & MAY_BE_ANY) != MAY_BE_STRING || (val_info & MAY_BE_ANY) != MAY_BE_STRING) { may_throw = 1; } } goto long_math; case ZEND_SL: case ZEND_SR: if ((var_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || (val_info & (MAY_BE_STRING|MAY_BE_DOUBLE|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { may_throw = 1; } if (val_op_type != IS_CONST || Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) < 0) { may_throw = 1; } goto long_math; case ZEND_MOD: if ((var_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) || (val_info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE))) { may_throw = 1; } if (val_op_type != IS_CONST || Z_TYPE_P(RT_CONSTANT((opline+1), (opline+1)->op1)) != IS_LONG || Z_LVAL_P(RT_CONSTANT((opline+1), (opline+1)->op1)) == 0) { may_throw = 1; } long_math: if (!zend_jit_long_math_helper(jit, opline, opline->extended_value, IS_CV, opline->op1, var_addr, var_info, NULL, val_op_type, (opline+1)->op1, val_addr, val_info, val_range, 0, var_addr, var_def_info, var_info, /* may throw */ 1)) { return 0; } break; case ZEND_CONCAT: may_throw = 1; if (!zend_jit_concat_helper(jit, opline, IS_CV, opline->op1, var_addr, var_info, val_op_type, (opline+1)->op1, val_addr, val_info, var_addr, 0)) { return 0; } break; default: ZEND_UNREACHABLE(); } if (end_inputs || slow_inputs) { ir_END_list(end_inputs); } } if (slow_inputs) { ir_ref arg3; ir_MERGE_list(slow_inputs); may_throw = 1; if (Z_MODE(val_addr) == IS_REG) { zend_jit_addr real_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, (opline+1)->op1.var); if (!zend_jit_spill_store_inv(jit, val_addr, real_addr, val_info)) { return 0; } arg3 = jit_ZVAL_ADDR(jit, real_addr); } else { arg3 = jit_ZVAL_ADDR(jit, val_addr); } jit_SET_EX_OPLINE(jit, opline); ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); ir_CALL_5(IR_VOID, ir_CONST_FC_FUNC(zend_jit_assign_obj_op_helper), obj_ref, ir_CONST_ADDR(name), arg3, ir_ADD_OFFSET(run_time_cache, (opline+1)->extended_value & ~ZEND_FETCH_OBJ_FLAGS), ir_CONST_FC_FUNC(binary_op)); ir_END_list(end_inputs); } if (end_inputs) { ir_MERGE_list(end_inputs); } if (val_info & (MAY_BE_REF|MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { val_info |= MAY_BE_RC1|MAY_BE_RCN; } // JIT: FREE_OP_DATA(); jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, val_info, opline); if (opline->op1_type != IS_UNUSED && !delayed_fetch_this && !op1_indirect) { if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { may_throw = 1; } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } if (may_throw) { zend_jit_check_exception(jit); } return 1; } static int zend_jit_incdec_obj(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, const zend_ssa_op *ssa_op, uint32_t op1_info, zend_jit_addr op1_addr, bool op1_indirect, zend_class_entry *ce, bool ce_is_instanceof, bool on_this, bool delayed_fetch_this, zend_class_entry *trace_ce, uint8_t prop_type) { zval *member; zend_string *name; zend_property_info *prop_info; zend_jit_addr res_addr = 0; zend_jit_addr prop_addr; bool use_prop_guard = 0; bool may_throw = 0; uint32_t res_info = (opline->result_type != IS_UNDEF) ? RES_INFO() : 0; ir_ref obj_ref = IR_UNUSED; ir_ref prop_ref = IR_UNUSED; ir_ref end_inputs = IR_UNUSED; ir_ref slow_inputs = IR_UNUSED; ZEND_ASSERT(opline->op2_type == IS_CONST); ZEND_ASSERT(op1_info & MAY_BE_OBJECT); if (opline->result_type != IS_UNUSED) { res_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, opline->result.var); } member = RT_CONSTANT(opline, opline->op2); ZEND_ASSERT(Z_TYPE_P(member) == IS_STRING && Z_STRVAL_P(member)[0] != '\0'); name = Z_STR_P(member); prop_info = zend_get_known_property_info(op_array, ce, name, on_this, op_array->filename); if (on_this) { zend_jit_addr this_addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, offsetof(zend_execute_data, This)); obj_ref = jit_Z_PTR(jit, this_addr); } else { if (opline->op1_type == IS_VAR && (op1_info & MAY_BE_INDIRECT) && Z_REG(op1_addr) == ZREG_FP) { op1_addr = jit_ZVAL_INDIRECT_DEREF(jit, op1_addr); } if (op1_info & MAY_BE_REF) { op1_addr = jit_ZVAL_DEREF(jit, op1_addr); } if (op1_info & ((MAY_BE_UNDEF|MAY_BE_ANY)- MAY_BE_OBJECT)) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_Z_TYPE(jit, op1_addr, IS_OBJECT, exit_addr); } else { ir_ref if_obj = jit_if_Z_TYPE(jit, op1_addr, IS_OBJECT); ir_IF_FALSE_cold(if_obj); jit_SET_EX_OPLINE(jit, opline); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_invalid_property_incdec), jit_ZVAL_ADDR(jit, op1_addr), ir_CONST_ADDR(ZSTR_VAL(name))); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler)); ir_IF_TRUE(if_obj); } } obj_ref = jit_Z_PTR(jit, op1_addr); } ZEND_ASSERT(obj_ref); if (!prop_info && trace_ce && (trace_ce->ce_flags & ZEND_ACC_IMMUTABLE)) { prop_info = zend_get_known_property_info(op_array, trace_ce, name, on_this, op_array->filename); if (prop_info) { ce = trace_ce; ce_is_instanceof = 0; if (!(op1_info & MAY_BE_CLASS_GUARD)) { if (on_this && JIT_G(current_frame) && TRACE_FRAME_IS_THIS_CLASS_CHECKED(JIT_G(current_frame))) { ZEND_ASSERT(JIT_G(current_frame)->ce == ce); } else if (zend_jit_class_guard(jit, opline, obj_ref, ce)) { if (on_this && JIT_G(current_frame)) { JIT_G(current_frame)->ce = ce; TRACE_FRAME_SET_THIS_CLASS_CHECKED(JIT_G(current_frame)); } } else { return 0; } if (ssa->var_info && ssa_op->op1_use >= 0) { ssa->var_info[ssa_op->op1_use].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_use].ce = ce; ssa->var_info[ssa_op->op1_use].is_instanceof = ce_is_instanceof; } if (ssa->var_info && ssa_op->op1_def >= 0) { ssa->var_info[ssa_op->op1_def].type |= MAY_BE_CLASS_GUARD; ssa->var_info[ssa_op->op1_def].ce = ce; ssa->var_info[ssa_op->op1_def].is_instanceof = ce_is_instanceof; } } } } use_prop_guard = (prop_type != IS_UNKNOWN && prop_type != IS_UNDEF && prop_type != IS_REFERENCE && (op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_OBJECT); if (!prop_info) { ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); ir_ref ref = ir_LOAD_A(ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS)); ir_ref if_same = ir_IF(ir_EQ(ref, ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))))); ir_IF_FALSE_cold(if_same); ir_END_list(slow_inputs); ir_IF_TRUE(if_same); if (!ce || ce_is_instanceof || (ce->ce_flags & (ZEND_ACC_HAS_TYPE_HINTS|ZEND_ACC_TRAIT))) { ir_ref prop_info_ref = ir_LOAD_A( ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*) * 2)); ir_ref if_has_prop_info = ir_IF(prop_info_ref); ir_IF_TRUE_cold(if_has_prop_info); ir_END_list(slow_inputs); ir_IF_FALSE(if_has_prop_info); } ir_ref offset_ref = ir_LOAD_A( ir_ADD_OFFSET(run_time_cache, (opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS) + sizeof(void*))); ir_ref if_dynamic = ir_IF(ir_LT(offset_ref, ir_CONST_ADDR(ZEND_FIRST_PROPERTY_OFFSET))); ir_IF_TRUE_cold(if_dynamic); ir_END_list(slow_inputs); ir_IF_FALSE(if_dynamic); prop_ref = ir_ADD_A(obj_ref, offset_ref); if (!use_prop_guard) { ir_ref if_def = ir_IF(jit_Z_TYPE_ref(jit, prop_ref)); ir_IF_FALSE_cold(if_def); ir_END_list(slow_inputs); ir_IF_TRUE(if_def); } prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref); } else { prop_ref = ir_ADD_OFFSET(obj_ref, prop_info->offset); prop_addr = ZEND_ADDR_REF_ZVAL(prop_ref); if (ZEND_TYPE_IS_SET(prop_info->type) || !use_prop_guard) { if (JIT_G(trigger) == ZEND_JIT_ON_HOT_TRACE) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, ZEND_JIT_EXIT_TO_VM); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(jit_Z_TYPE_INFO(jit, prop_addr), ir_CONST_ADDR(exit_addr)); } else { ir_ref if_def = ir_IF(jit_Z_TYPE_INFO(jit, prop_addr)); ir_IF_FALSE_cold(if_def); ir_END_list(slow_inputs); ir_IF_TRUE(if_def); } } if (ZEND_TYPE_IS_SET(prop_info->type)) { const void *func; ir_ref ref; may_throw = 1; jit_SET_EX_OPLINE(jit, opline); if (ce && ce->ce_flags & ZEND_ACC_IMMUTABLE) { ref = ir_CONST_ADDR(prop_info); } else { int prop_info_offset = (((prop_info->offset - (sizeof(zend_object) - sizeof(zval))) / sizeof(zval)) * sizeof(void*)); ref = ir_LOAD_A(ir_ADD_OFFSET(obj_ref, offsetof(zend_object, ce))); ref = ir_LOAD_A(ir_ADD_OFFSET(ref, offsetof(zend_class_entry, properties_info_table))); ref = ir_LOAD_A(ir_ADD_OFFSET(ref, prop_info_offset)); } if (opline->result_type == IS_UNUSED) { switch (opline->opcode) { case ZEND_PRE_INC_OBJ: case ZEND_POST_INC_OBJ: func = zend_jit_inc_typed_prop; break; case ZEND_PRE_DEC_OBJ: case ZEND_POST_DEC_OBJ: func = zend_jit_dec_typed_prop; break; default: ZEND_UNREACHABLE(); } ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func), prop_ref, ref); } else { switch (opline->opcode) { case ZEND_PRE_INC_OBJ: func = zend_jit_pre_inc_typed_prop; break; case ZEND_PRE_DEC_OBJ: func = zend_jit_pre_dec_typed_prop; break; case ZEND_POST_INC_OBJ: func = zend_jit_post_inc_typed_prop; break; case ZEND_POST_DEC_OBJ: func = zend_jit_post_dec_typed_prop; break; default: ZEND_UNREACHABLE(); } ir_CALL_3(IR_VOID, ir_CONST_FC_FUNC(func), prop_ref, ref, jit_ZVAL_ADDR(jit, res_addr)); } ir_END_list(end_inputs); } } if (!prop_info || !ZEND_TYPE_IS_SET(prop_info->type)) { uint32_t var_info = MAY_BE_ANY|MAY_BE_REF|MAY_BE_RC1|MAY_BE_RCN; zend_jit_addr var_addr = prop_addr; ir_ref if_long = IR_UNUSED; ir_ref if_overflow = IR_UNUSED; if (use_prop_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_Z_TYPE(jit, prop_addr, prop_type, exit_addr); var_info = (1 << prop_type) | (var_info & ~(MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)); } if (var_info & MAY_BE_REF) { const void *func; ir_ref if_ref, if_typed, noref_path, ref_path, reference, ref; if_ref = jit_if_Z_TYPE(jit, prop_addr, IS_REFERENCE); ir_IF_FALSE(if_ref); noref_path = ir_END(); ir_IF_TRUE(if_ref); reference = jit_Z_PTR(jit, var_addr); ref = ir_ADD_OFFSET(reference, offsetof(zend_reference, val)); if_typed = jit_if_TYPED_REF(jit, reference); ir_IF_FALSE(if_typed); ref_path = ir_END(); ir_IF_TRUE_cold(if_typed); switch (opline->opcode) { case ZEND_PRE_INC_OBJ: func = zend_jit_pre_inc_typed_ref; break; case ZEND_PRE_DEC_OBJ: func = zend_jit_pre_dec_typed_ref; break; case ZEND_POST_INC_OBJ: func = zend_jit_post_inc_typed_ref; break; case ZEND_POST_DEC_OBJ: func = zend_jit_post_dec_typed_ref; break; default: ZEND_UNREACHABLE(); } may_throw = 1; jit_SET_EX_OPLINE(jit, opline); ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(func), reference, (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr)); ir_END_list(end_inputs); ir_MERGE_2(noref_path, ref_path); prop_ref = ir_PHI_2(IR_ADDR, prop_ref, ref); var_addr = ZEND_ADDR_REF_ZVAL(prop_ref); var_info &= ~MAY_BE_REF; } if (var_info & MAY_BE_LONG) { ir_ref addr, ref; if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { if_long = jit_if_Z_TYPE(jit, var_addr, IS_LONG); ir_IF_TRUE(if_long); } addr = jit_ZVAL_ADDR(jit, var_addr); ref = ir_LOAD_L(addr); if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { if (opline->result_type != IS_UNUSED) { jit_set_Z_LVAL(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } } if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { ref = ir_ADD_OV_L(ref, ir_CONST_LONG(1)); } else { ref = ir_SUB_OV_L(ref, ir_CONST_LONG(1)); } ir_STORE(addr, ref); if_overflow = ir_IF(ir_OVERFLOW(ref)); ir_IF_FALSE(if_overflow); if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) { if (opline->result_type != IS_UNUSED) { jit_set_Z_LVAL(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_LONG); } } ir_END_list(end_inputs); } if (var_info & (MAY_BE_ANY - MAY_BE_LONG)) { if (var_info & (MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE)) { may_throw = 1; } if (if_long) { ir_IF_FALSE_cold(if_long); } if (opline->opcode == ZEND_POST_INC_OBJ || opline->opcode == ZEND_POST_DEC_OBJ) { jit_ZVAL_COPY(jit, res_addr, -1, var_addr, var_info, 1); } if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_inc), jit_ZVAL_ADDR(jit, var_addr), jit_ZVAL_ADDR(jit, res_addr)); } else { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(increment_function), jit_ZVAL_ADDR(jit, var_addr)); } } else { if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { ir_CALL_2(IR_VOID, ir_CONST_FC_FUNC(zend_jit_pre_dec), jit_ZVAL_ADDR(jit, var_addr), jit_ZVAL_ADDR(jit, res_addr)); } else { ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(decrement_function), jit_ZVAL_ADDR(jit, var_addr)); } } ir_END_list(end_inputs); } if (var_info & MAY_BE_LONG) { ir_IF_TRUE_cold(if_overflow); if (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_POST_INC_OBJ) { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0)); jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0x41e00000)); #else jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x43e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE); if (opline->opcode == ZEND_PRE_INC_OBJ && opline->result_type != IS_UNUSED) { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0)); jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0x41e00000)); #else jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x43e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } } else { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0x00200000)); jit_set_Z_W2(jit, var_addr, ir_CONST_U32(0xc1e00000)); #else jit_set_Z_LVAL(jit, var_addr, ir_CONST_LONG(0xc3e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, var_addr, IS_DOUBLE); if (opline->opcode == ZEND_PRE_DEC_OBJ && opline->result_type != IS_UNUSED) { #if SIZEOF_ZEND_LONG == 4 jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0x00200000)); jit_set_Z_W2(jit, res_addr, ir_CONST_U32(0xc1e00000)); #else jit_set_Z_LVAL(jit, res_addr, ir_CONST_LONG(0xc3e0000000000000)); #endif jit_set_Z_TYPE_INFO(jit, res_addr, IS_DOUBLE); } } if (opline->result_type != IS_UNUSED && (opline->opcode == ZEND_PRE_INC_OBJ || opline->opcode == ZEND_PRE_DEC_OBJ) && prop_info && !ZEND_TYPE_IS_SET(prop_info->type) && (res_info & MAY_BE_GUARD) && (res_info & MAY_BE_LONG)) { zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; uint32_t old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); int32_t exit_point; const void *exit_addr; SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_DOUBLE, 0); exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); ssa->var_info[ssa_op->result_def].type = res_info & ~MAY_BE_GUARD; jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } else { ir_END_list(end_inputs); } } } if (slow_inputs) { const void *func; ir_MERGE_list(slow_inputs); // JIT: zend_jit_pre_inc_obj_helper(zobj, name, CACHE_ADDR(opline->extended_value), result); switch (opline->opcode) { case ZEND_PRE_INC_OBJ: func = zend_jit_pre_inc_obj_helper; break; case ZEND_PRE_DEC_OBJ: func = zend_jit_pre_dec_obj_helper; break; case ZEND_POST_INC_OBJ: func = zend_jit_post_inc_obj_helper; break; case ZEND_POST_DEC_OBJ: func = zend_jit_post_dec_obj_helper; break; default: ZEND_UNREACHABLE(); } may_throw = 1; jit_SET_EX_OPLINE(jit, opline); ir_ref run_time_cache = ir_LOAD_A(jit_EX(run_time_cache)); ir_CALL_4(IR_VOID, ir_CONST_FC_FUNC(func), obj_ref, ir_CONST_ADDR(name), ir_ADD_OFFSET(run_time_cache, opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS), (opline->result_type == IS_UNUSED) ? IR_NULL : jit_ZVAL_ADDR(jit, res_addr)); ir_END_list(end_inputs); } if (end_inputs) { ir_MERGE_list(end_inputs); } if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && !delayed_fetch_this && !op1_indirect) { if ((op1_info & MAY_HAVE_DTOR) && (op1_info & MAY_BE_RC1)) { may_throw = 1; } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, opline); } if (may_throw) { zend_jit_check_exception(jit); } return 1; } static int zend_jit_fetch_static_prop(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array) { zend_jit_addr res_addr = RES_ADDR(); uint32_t cache_slot = opline->extended_value & ~ZEND_FETCH_OBJ_FLAGS; uint32_t flags; ir_ref ref, ref2, if_cached, fast_path, cold_path, prop_info_ref, if_typed, if_def; int fetch_type; zend_property_info *known_prop_info = NULL; zend_class_entry *ce; ce = zend_get_known_class(op_array, opline, opline->op2_type, opline->op2); if (ce && (opline->op2_type == IS_CONST || !(ce->ce_flags & ZEND_ACC_TRAIT))) { zval *zv = RT_CONSTANT(opline, opline->op1); zend_string *prop_name; ZEND_ASSERT(Z_TYPE_P(zv) == IS_STRING); prop_name = Z_STR_P(zv); zv = zend_hash_find(&ce->properties_info, prop_name); if (zv) { zend_property_info *prop_info = Z_PTR_P(zv); if (prop_info->flags & ZEND_ACC_STATIC) { if (prop_info->ce == op_array->scope || (prop_info->flags & ZEND_ACC_PUBLIC) || ((prop_info->flags & ZEND_ACC_PROTECTED) && op_array->scope && instanceof_function_slow(op_array->scope, prop_info->ce))) { known_prop_info = prop_info; } } } } switch (opline->opcode) { case ZEND_FETCH_STATIC_PROP_R: case ZEND_FETCH_STATIC_PROP_FUNC_ARG: fetch_type = BP_VAR_R; break; case ZEND_FETCH_STATIC_PROP_IS: fetch_type = BP_VAR_IS; break; case ZEND_FETCH_STATIC_PROP_W: fetch_type = BP_VAR_W; break; case ZEND_FETCH_STATIC_PROP_RW: fetch_type = BP_VAR_RW; break; case ZEND_FETCH_STATIC_PROP_UNSET: fetch_type = BP_VAR_UNSET; break; EMPTY_SWITCH_DEFAULT_CASE(); } // JIT: result = CACHED_PTR(cache_slot + sizeof(void *)); ref = ir_LOAD_A( ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), cache_slot + sizeof(void*))); // JIT: if (result) if_cached = ir_IF(ref); ir_IF_TRUE(if_cached); if (fetch_type == BP_VAR_R || fetch_type == BP_VAR_RW) { if (!known_prop_info || ZEND_TYPE_IS_SET(known_prop_info->type)) { ir_ref merge = IR_UNUSED; // JIT: if (UNEXPECTED(Z_TYPE_P(result) == IS_UNDEF) if_typed = IR_UNUSED; if_def = ir_IF(jit_Z_TYPE_ref(jit, ref)); ir_IF_FALSE_cold(if_def); if (!known_prop_info) { // JIT: if (ZEND_TYPE_IS_SET(property_info->type)) prop_info_ref = ir_LOAD_L( ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), cache_slot + sizeof(void*) * 2)); if_typed = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, type.type_mask))), ir_CONST_U32(_ZEND_TYPE_MASK))); ir_IF_FALSE(if_typed); ir_END_list(merge); ir_IF_TRUE(if_typed); } // JIT: zend_throw_error(NULL, "Typed static property %s::$%s must not be accessed before initialization", // ZSTR_VAL(property_info->ce->name), // zend_get_unmangled_property_name(property_info->name)); jit_SET_EX_OPLINE(jit, opline); ir_CALL(IR_VOID, ir_CONST_FC_FUNC(zend_jit_uninit_static_prop)); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); ir_IF_TRUE(if_def); if (!known_prop_info) { ir_END_list(merge); ir_MERGE_list(merge); } } } else if (fetch_type == BP_VAR_W) { flags = opline->extended_value & ZEND_FETCH_OBJ_FLAGS; if (flags && (!known_prop_info || ZEND_TYPE_IS_SET(known_prop_info->type))) { ir_ref merge = IR_UNUSED; if (!known_prop_info) { // JIT: if (ZEND_TYPE_IS_SET(property_info->type)) prop_info_ref = ir_LOAD_L( ir_ADD_OFFSET(ir_LOAD_A(jit_EX(run_time_cache)), cache_slot + sizeof(void*) * 2)); if_typed = ir_IF(ir_AND_U32( ir_LOAD_U32(ir_ADD_OFFSET(prop_info_ref, offsetof(zend_property_info, type.type_mask))), ir_CONST_U32(_ZEND_TYPE_MASK))); ir_IF_FALSE(if_typed); ir_END_list(merge); ir_IF_TRUE(if_typed); } else { prop_info_ref = ir_CONST_ADDR(known_prop_info); } // JIT: zend_handle_fetch_obj_flags(NULL, *retval, NULL, property_info, flags); ir_ref if_ok = ir_IF(ir_CALL_5(IR_BOOL, ir_CONST_FUNC(zend_handle_fetch_obj_flags), IR_NULL, ref, IR_NULL, prop_info_ref, ir_CONST_U32(flags))); ir_IF_FALSE_cold(if_ok); ir_IJMP(jit_STUB_ADDR(jit, jit_stub_exception_handler_undef)); ir_IF_TRUE(if_ok); if (!known_prop_info) { ir_END_list(merge); ir_MERGE_list(merge); } } } fast_path = ir_END(); ir_IF_FALSE_cold(if_cached); jit_SET_EX_OPLINE(jit, opline); ref2 = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(zend_fetch_static_property), jit_FP(jit), ir_CONST_I32(fetch_type)); zend_jit_check_exception_undef_result(jit, opline); cold_path = ir_END(); ir_MERGE_2(fast_path, cold_path); ref = ir_PHI_2(IR_ADDR, ref, ref2); if (fetch_type == BP_VAR_R || fetch_type == BP_VAR_IS) { // JIT: ZVAL_COPY_DEREF(EX_VAR(opline->result.var), result); if (!zend_jit_zval_copy_deref(jit, res_addr, ZEND_ADDR_REF_ZVAL(ref), jit_Z_TYPE_INFO_ref(jit, ref))) { return 0; } } else { // JIT: ZVAL_INDIRECT(EX_VAR(opline->result.var), result); jit_set_Z_PTR(jit, res_addr, ref); jit_set_Z_TYPE_INFO(jit, res_addr, IS_INDIRECT); } return 1; } static int zend_jit_switch(zend_jit_ctx *jit, const zend_op *opline, const zend_op_array *op_array, zend_ssa *ssa, zend_jit_trace_rec *trace, zend_jit_trace_info *trace_info) { HashTable *jumptable = Z_ARRVAL_P(RT_CONSTANT(opline, opline->op2)); const zend_op *next_opline = NULL; ir_refs *slow_inputs; ir_refs_init(slow_inputs, 8); if (trace) { ZEND_ASSERT(trace->op == ZEND_JIT_TRACE_VM || trace->op == ZEND_JIT_TRACE_END); ZEND_ASSERT(trace->opline != NULL); next_opline = trace->opline; } if (opline->op1_type == IS_CONST) { zval *zv = RT_CONSTANT(opline, opline->op1); zval *jump_zv = NULL; int b; if (opline->opcode == ZEND_SWITCH_LONG) { if (Z_TYPE_P(zv) == IS_LONG) { jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); } } else if (opline->opcode == ZEND_SWITCH_STRING) { if (Z_TYPE_P(zv) == IS_STRING) { jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); } } else if (opline->opcode == ZEND_MATCH) { if (Z_TYPE_P(zv) == IS_LONG) { jump_zv = zend_hash_index_find(jumptable, Z_LVAL_P(zv)); } else if (Z_TYPE_P(zv) == IS_STRING) { jump_zv = zend_hash_find_known_hash(jumptable, Z_STR_P(zv)); } } else { ZEND_UNREACHABLE(); } if (next_opline) { const zend_op *target; if (jump_zv != NULL) { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)); } else { target = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); } ZEND_ASSERT(target == next_opline); } else { if (jump_zv != NULL) { b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(jump_zv)) - op_array->opcodes]; } else { b = ssa->cfg.map[ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) - op_array->opcodes]; } _zend_jit_add_predecessor_ref(jit, b, jit->b, ir_END()); jit->b = -1; } } else { zend_ssa_op *ssa_op = ssa->ops ? &ssa->ops[opline - op_array->opcodes] : NULL; uint32_t op1_info = OP1_INFO(); zend_jit_addr op1_addr = OP1_ADDR(); const zend_op *default_opline = ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value); const zend_op *target; int default_b = next_opline ? -1 : ssa->cfg.map[default_opline - op_array->opcodes]; int b; int32_t exit_point; const void *exit_addr; const void *default_label = NULL; zval *zv; if (next_opline) { if (next_opline != default_opline) { exit_point = zend_jit_trace_get_exit_point(default_opline, 0); default_label = zend_jit_trace_get_exit_addr(exit_point); if (!default_label) { return 0; } } } if (opline->opcode == ZEND_SWITCH_LONG) { if (op1_info & MAY_BE_LONG) { const void *fallback_label = NULL; if (next_opline) { if (next_opline != opline + 1) { exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; } } } if (op1_info & MAY_BE_REF) { ir_ref ref, if_long, fast_path, ref2; ref = jit_ZVAL_ADDR(jit, op1_addr); if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_TRUE(if_long); fast_path = ir_END(); ir_IF_FALSE_cold(if_long); // JIT: ZVAL_DEREF(op) if (fallback_label) { jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label); } else { ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_REFERENCE); ir_IF_FALSE_cold(if_ref); ir_refs_add(slow_inputs, ir_END()); ir_IF_TRUE(if_ref); } ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val)); op1_addr = ZEND_ADDR_REF_ZVAL(ref2); if (fallback_label) { jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label); } else { if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_FALSE_cold(if_long); ir_refs_add(slow_inputs, ir_END()); ir_IF_TRUE(if_long); } ir_MERGE_2(fast_path, ir_END()); ref = ir_PHI_2(IR_ADDR, ref, ref2); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (fallback_label) { jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label); } else { ir_ref if_long = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_FALSE_cold(if_long); ir_refs_add(slow_inputs, ir_END()); ir_IF_TRUE(if_long); } } ir_ref ref = jit_Z_LVAL(jit, op1_addr); if (!HT_IS_PACKED(jumptable)) { ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find), ir_CONST_ADDR(jumptable), ref); ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData)); /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */ if (sizeof(Bucket) == 32) { ref = ir_SHR_L(ref, ir_CONST_LONG(5)); } else { ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket))); } } ref = ir_SWITCH(ref); if (next_opline) { ir_ref continue_list = IR_UNUSED; ZEND_HASH_FOREACH_VAL(jumptable, zv) { ir_ref idx; target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); if (HT_IS_PACKED(jumptable)) { idx = ir_CONST_LONG(zv - jumptable->arPacked); } else { idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData); } ir_CASE_VAL(ref, idx); if (target == next_opline) { ir_END_list(continue_list); } else { exit_point = zend_jit_trace_get_exit_point(target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } } ZEND_HASH_FOREACH_END(); ir_CASE_DEFAULT(ref); if (next_opline == default_opline) { ir_END_list(continue_list); } else { jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label)); } if (continue_list) { ir_MERGE_list(continue_list); } else { ZEND_ASSERT(slow_inputs->count); ir_MERGE_N(slow_inputs->count, slow_inputs->refs); } } else { ZEND_HASH_FOREACH_VAL(jumptable, zv) { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); b = ssa->cfg.map[target - op_array->opcodes]; _zend_jit_add_predecessor_ref(jit, b, jit->b, ref); } ZEND_HASH_FOREACH_END(); _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref); if (slow_inputs->count) { ir_MERGE_N(slow_inputs->count, slow_inputs->refs); _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END()); } jit->b = -1; } } else if (!next_opline) { _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END()); jit->b = -1; } } else if (opline->opcode == ZEND_SWITCH_STRING) { if (op1_info & MAY_BE_STRING) { const void *fallback_label = NULL; if (next_opline) { if (next_opline != opline + 1) { exit_point = zend_jit_trace_get_exit_point(opline + 1, 0); fallback_label = zend_jit_trace_get_exit_addr(exit_point); if (!fallback_label) { return 0; } } } if (op1_info & MAY_BE_REF) { ir_ref ref, if_string, fast_path, ref2; ref = jit_ZVAL_ADDR(jit, op1_addr); if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); ir_IF_TRUE(if_string); fast_path = ir_END(); ir_IF_FALSE_cold(if_string); // JIT: ZVAL_DEREF(op) if (fallback_label) { jit_guard_Z_TYPE(jit, op1_addr, IS_REFERENCE, fallback_label); } else { ir_ref if_ref = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); ir_IF_FALSE_cold(if_ref); ir_refs_add(slow_inputs, ir_END()); ir_IF_TRUE(if_ref); } ref2 = ir_ADD_OFFSET(jit_Z_PTR(jit, op1_addr), offsetof(zend_reference, val)); op1_addr = ZEND_ADDR_REF_ZVAL(ref2); if (fallback_label) { jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, fallback_label); } else { if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); ir_IF_FALSE_cold(if_string); ir_refs_add(slow_inputs, ir_END()); ir_IF_TRUE(if_string); } ir_MERGE_2(fast_path, ir_END()); ref = ir_PHI_2(IR_ADDR, ref, ref2); op1_addr = ZEND_ADDR_REF_ZVAL(ref); } else if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_STRING)) { if (fallback_label) { jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, fallback_label); } else { ir_ref if_string = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); ir_IF_FALSE_cold(if_string); ir_refs_add(slow_inputs, ir_END()); ir_IF_TRUE(if_string); } } ir_ref ref = jit_Z_PTR(jit, op1_addr); ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find), ir_CONST_ADDR(jumptable), ref); ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData)); /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */ if (sizeof(Bucket) == 32) { ref = ir_SHR_L(ref, ir_CONST_LONG(5)); } else { ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket))); } ref = ir_SWITCH(ref); if (next_opline) { ir_ref continue_list = IR_UNUSED; ZEND_HASH_FOREACH_VAL(jumptable, zv) { ir_ref idx; target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); if (HT_IS_PACKED(jumptable)) { idx = ir_CONST_LONG(zv - jumptable->arPacked); } else { idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData); } ir_CASE_VAL(ref, idx); if (target == next_opline) { ir_END_list(continue_list); } else { exit_point = zend_jit_trace_get_exit_point(target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } } ZEND_HASH_FOREACH_END(); ir_CASE_DEFAULT(ref); if (next_opline == default_opline) { ir_END_list(continue_list); } else { jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label)); } if (continue_list) { ir_MERGE_list(continue_list); } else { ZEND_ASSERT(slow_inputs->count); ir_MERGE_N(slow_inputs->count, slow_inputs->refs); } } else { ZEND_HASH_FOREACH_VAL(jumptable, zv) { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); b = ssa->cfg.map[target - op_array->opcodes]; _zend_jit_add_predecessor_ref(jit, b, jit->b, ref); } ZEND_HASH_FOREACH_END(); _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref); if (slow_inputs->count) { ir_MERGE_N(slow_inputs->count, slow_inputs->refs); _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END()); } jit->b = -1; } } else if (!next_opline) { _zend_jit_add_predecessor_ref(jit, jit->b + 1, jit->b, ir_END()); jit->b = -1; } } else if (opline->opcode == ZEND_MATCH) { ir_ref if_type = IR_UNUSED, default_input_list = IR_UNUSED, ref = IR_UNUSED; ir_ref continue_list = IR_UNUSED; if (op1_info & (MAY_BE_LONG|MAY_BE_STRING)) { ir_ref long_path = IR_UNUSED; if (op1_info & MAY_BE_REF) { op1_addr = jit_ZVAL_DEREF(jit, op1_addr); } if (op1_info & MAY_BE_LONG) { if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-MAY_BE_LONG)) { if (op1_info & (MAY_BE_STRING|MAY_BE_UNDEF)) { if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_TRUE(if_type); } else if (default_label) { jit_guard_Z_TYPE(jit, op1_addr, IS_LONG, default_label); } else if (next_opline) { ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_FALSE(if_type); ir_END_list(continue_list); ir_IF_TRUE(if_type); } else { ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_LONG); ir_IF_FALSE(if_type); ir_END_list(default_input_list); ir_IF_TRUE(if_type); } } ref = jit_Z_LVAL(jit, op1_addr); ref = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_index_find), ir_CONST_ADDR(jumptable), ref); if (op1_info & MAY_BE_STRING) { long_path = ir_END(); } } if (op1_info & MAY_BE_STRING) { if (if_type) { ir_IF_FALSE(if_type); if_type = IS_UNUSED; } if (op1_info & ((MAY_BE_ANY|MAY_BE_UNDEF)-(MAY_BE_LONG|MAY_BE_STRING))) { if (op1_info & MAY_BE_UNDEF) { if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); ir_IF_TRUE(if_type); } else if (default_label) { jit_guard_Z_TYPE(jit, op1_addr, IS_STRING, default_label); } else if (next_opline) { ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); ir_IF_FALSE(if_type); ir_END_list(continue_list); ir_IF_TRUE(if_type); } else { ir_ref if_type = jit_if_Z_TYPE(jit, op1_addr, IS_STRING); ir_IF_FALSE(if_type); ir_END_list(default_input_list); ir_IF_TRUE(if_type); } } ir_ref ref2 = jit_Z_PTR(jit, op1_addr); ref2 = ir_CALL_2(IR_LONG, ir_CONST_FC_FUNC(zend_hash_find), ir_CONST_ADDR(jumptable), ref2); if (op1_info & MAY_BE_LONG) { ir_MERGE_WITH(long_path); ref = ir_PHI_2(IR_LONG, ref2, ref); } else { ref = ref2; } } ref = ir_SUB_L(ref, ir_CONST_LONG((uintptr_t)jumptable->arData)); /* Signed DIV by power of 2 may be optimized into SHR only for positive operands */ if (HT_IS_PACKED(jumptable)) { ZEND_ASSERT(sizeof(zval) == 16); ref = ir_SHR_L(ref, ir_CONST_LONG(4)); } else { if (sizeof(Bucket) == 32) { ref = ir_SHR_L(ref, ir_CONST_LONG(5)); } else { ref = ir_DIV_L(ref, ir_CONST_LONG(sizeof(Bucket))); } } ref = ir_SWITCH(ref); if (next_opline) { ZEND_HASH_FOREACH_VAL(jumptable, zv) { ir_ref idx; target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); if (HT_IS_PACKED(jumptable)) { idx = ir_CONST_LONG(zv - jumptable->arPacked); } else { idx = ir_CONST_LONG((Bucket*)zv - jumptable->arData); } ir_CASE_VAL(ref, idx); if (target == next_opline) { ir_END_list(continue_list); } else { exit_point = zend_jit_trace_get_exit_point(target, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_SIDE_EXIT(jit, ir_CONST_ADDR(exit_addr)); } } ZEND_HASH_FOREACH_END(); ir_CASE_DEFAULT(ref); if (next_opline == default_opline) { ir_END_list(continue_list); } else { jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label)); } } else { ZEND_HASH_FOREACH_VAL(jumptable, zv) { target = ZEND_OFFSET_TO_OPLINE(opline, Z_LVAL_P(zv)); b = ssa->cfg.map[target - op_array->opcodes]; _zend_jit_add_predecessor_ref(jit, b, jit->b, ref); } ZEND_HASH_FOREACH_END(); _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ref); } } else if (!(op1_info & MAY_BE_UNDEF)) { if (next_opline) { if (next_opline == default_opline) { ir_END_list(continue_list); } else { jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label)); } } else { _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END()); } } if (op1_info & MAY_BE_UNDEF) { if (if_type) { ir_IF_FALSE(if_type); if_type = IS_UNUSED; } if (op1_info & (MAY_BE_ANY-(MAY_BE_LONG|MAY_BE_STRING))) { if (default_label) { jit_guard_Z_TYPE(jit, op1_addr, IS_UNDEF, default_label); } else if (next_opline) { ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr)); ir_IF_TRUE(if_def); ir_END_list(continue_list); ir_IF_FALSE_cold(if_def); } else { ir_ref if_def = ir_IF(jit_Z_TYPE(jit, op1_addr)); ir_IF_TRUE(if_def); ir_END_list(default_input_list); ir_IF_FALSE_cold(if_def); } } jit_SET_EX_OPLINE(jit, opline); ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_undefined_op_helper), ir_CONST_U32(opline->op1.var)); zend_jit_check_exception_undef_result(jit, opline); if (default_label) { jit_SIDE_EXIT(jit, ir_CONST_ADDR(default_label)); } else if (next_opline) { ir_END_list(continue_list); } else { ir_END_list(default_input_list); } } if (next_opline) { ZEND_ASSERT(continue_list); ir_MERGE_list(continue_list); } else { if (default_input_list) { if (jit->ctx.ir_base[ref].op == IR_SWITCH) { ZEND_ASSERT(jit->ctx.ir_base[ref].op3 == IR_UNUSED); jit->ctx.ir_base[ref].op3 = default_input_list; } else { ir_MERGE_list(default_input_list); _zend_jit_add_predecessor_ref(jit, default_b, jit->b, ir_END()); } } jit->b = -1; } } else { ZEND_UNREACHABLE(); } } return 1; } static int zend_jit_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend_ssa *ssa) { int i, count; zend_basic_block *bb; zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) ? 0 : (IR_START_BR_TARGET|IR_ENTRY_BR_TARGET)); jit->ctx.spill_base = ZREG_FP; jit->op_array = jit->current_op_array = op_array; jit->ssa = ssa; jit->bb_start_ref = zend_arena_calloc(&CG(arena), ssa->cfg.blocks_count * 2, sizeof(ir_ref)); jit->bb_predecessors = jit->bb_start_ref + ssa->cfg.blocks_count; count = 0; for (i = 0, bb = ssa->cfg.blocks; i < ssa->cfg.blocks_count; i++, bb++) { jit->bb_predecessors[i] = count; count += bb->predecessors_count; } jit->bb_edges = zend_arena_calloc(&CG(arena), count, sizeof(ir_ref)); if (!GCC_GLOBAL_REGS) { if (ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) { ir_ref execute_data_ref = ir_PARAM(IR_ADDR, "execute_data", 1); ir_ref opline_ref = ir_PARAM(IR_ADDR, "opline", 2); jit_STORE_FP(jit, execute_data_ref); jit_STORE_IP(jit, opline_ref); } jit->ctx.flags |= IR_FASTCALL_FUNC; } return 1; } static zend_vm_opcode_handler_t zend_jit_finish(zend_jit_ctx *jit) { void *entry; size_t size; zend_string *str = NULL; if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP| ZEND_JIT_DEBUG_IR_SRC|ZEND_JIT_DEBUG_IR_AFTER_SCCP|ZEND_JIT_DEBUG_IR_AFTER_SCCP| ZEND_JIT_DEBUG_IR_AFTER_SCHEDULE|ZEND_JIT_DEBUG_IR_AFTER_REGS|ZEND_JIT_DEBUG_IR_FINAL|ZEND_JIT_DEBUG_IR_CODEGEN)) { if (jit->name) { str = zend_string_copy(jit->name); } else { str = zend_jit_func_name(jit->op_array); } } if (jit->op_array) { /* Only for function JIT */ _zend_jit_fix_merges(jit); #if defined(IR_TARGET_AARCH64) } else if (jit->trace) { jit->ctx.deoptimization_exits = jit->trace->exit_count; jit->ctx.get_exit_addr = zend_jit_trace_get_exit_addr; #endif } else { #if defined(IR_TARGET_X86) || defined(IR_TARGET_X64) jit->ctx.flags |= IR_GEN_CACHE_DEMOTE; #endif } entry = zend_jit_ir_compile(&jit->ctx, &size, str ? ZSTR_VAL(str) : NULL); if (entry) { if (JIT_G(debug) & (ZEND_JIT_DEBUG_ASM|ZEND_JIT_DEBUG_GDB|ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) { #ifdef HAVE_CAPSTONE if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { if (str) { ir_disasm_add_symbol(ZSTR_VAL(str), (uintptr_t)entry, size); } ir_disasm(str ? ZSTR_VAL(str) : "unknown", entry, size, (JIT_G(debug) & ZEND_JIT_DEBUG_ASM_ADDR) != 0, &jit->ctx, stderr); } #endif #ifndef _WIN32 if (str) { if (JIT_G(debug) & ZEND_JIT_DEBUG_GDB) { uintptr_t sp_offset = 0; // ir_mem_unprotect(entry, size); if (!(jit->ctx.flags & IR_FUNCTION) && ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) { #if !defined(ZEND_WIN32) && !defined(IR_TARGET_AARCH64) sp_offset = zend_jit_hybrid_vm_sp_adj; #else sp_offset = sizeof(void*); #endif } else { sp_offset = sizeof(void*); } ir_gdb_register(ZSTR_VAL(str), entry, size, sp_offset, 0); // ir_mem_protect(entry, size); } if (JIT_G(debug) & (ZEND_JIT_DEBUG_PERF|ZEND_JIT_DEBUG_PERF_DUMP)) { ir_perf_map_register(ZSTR_VAL(str), entry, size); if (JIT_G(debug) & ZEND_JIT_DEBUG_PERF_DUMP) { ir_perf_jitdump_register(ZSTR_VAL(str), entry, size); } } } #endif } if (jit->op_array) { /* Only for function JIT */ const zend_op_array *op_array = jit->op_array; zend_op *opline = (zend_op*)op_array->opcodes; if (!(op_array->fn_flags & ZEND_ACC_HAS_TYPE_HINTS)) { while (opline->opcode == ZEND_RECV) { opline++; } } opline->handler = (zend_vm_opcode_handler_t)entry; if (jit->ctx.entries_count) { /* For all entries */ int i = jit->ctx.entries_count; do { ir_insn *insn = &jit->ctx.ir_base[jit->ctx.entries[--i]]; op_array->opcodes[insn->op2].handler = (zend_vm_opcode_handler_t)((char*)entry + insn->op3); } while (i != 0); } } else { /* Only for tracing JIT */ zend_jit_trace_info *t = jit->trace; zend_jit_trace_stack *stack; uint32_t i; if (t) { for (i = 0; i < t->stack_map_size; i++) { stack = t->stack_map + i; if (stack->flags & ZREG_SPILL_SLOT) { stack->reg = (jit->ctx.flags & IR_USE_FRAME_POINTER) ? IR_REG_FP : IR_REG_SP; stack->ref = ir_get_spill_slot_offset(&jit->ctx, stack->ref); } } } zend_jit_trace_add_code(entry, size); } } if (str) { zend_string_release(str); } return (zend_vm_opcode_handler_t)entry; } static const void *zend_jit_trace_allocate_exit_group(uint32_t n) { const void *entry; size_t size; ir_code_buffer code_buffer; code_buffer.start = dasm_buf; code_buffer.end = dasm_end; code_buffer.pos = *dasm_ptr; entry = ir_emit_exitgroup(n, ZEND_JIT_EXIT_POINTS_PER_GROUP, zend_jit_stub_handlers[jit_stub_trace_exit], &code_buffer, &size); *dasm_ptr = code_buffer.pos; if (entry) { #ifdef HAVE_CAPSTONE if (JIT_G(debug) & ZEND_JIT_DEBUG_ASM) { uint32_t i; char name[32]; for (i = 0; i < ZEND_JIT_EXIT_POINTS_PER_GROUP; i++) { snprintf(name, sizeof(name), "jit$$trace_exit_%d", n + i); ir_disasm_add_symbol(name, (uintptr_t)entry + (i * ZEND_JIT_EXIT_POINTS_SPACING), ZEND_JIT_EXIT_POINTS_SPACING); } } #endif } return entry; } static int zend_jit_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var, uint8_t type) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!exit_addr) { return 0; } ir_GUARD(ir_EQ(jit_Z_TYPE(jit, addr), ir_CONST_U8(type)), ir_CONST_ADDR(exit_addr)); return 1; } static int zend_jit_scalar_type_guard(zend_jit_ctx *jit, const zend_op *opline, uint32_t var) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); zend_jit_addr addr = ZEND_ADDR_MEM_ZVAL(ZREG_FP, var); if (!exit_addr) { return 0; } ir_GUARD(ir_LT(jit_Z_TYPE(jit, addr), ir_CONST_U8(IS_STRING)), ir_CONST_ADDR(exit_addr)); return 1; } static bool zend_jit_noref_guard(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr var_addr) { uint32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(ir_NE(jit_Z_TYPE(jit, var_addr), ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr)); return 1; } static int zend_jit_trace_opline_guard(zend_jit_ctx *jit, const zend_op *opline) { uint32_t exit_point = zend_jit_trace_get_exit_point(NULL, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ir_GUARD(jit_CMP_IP(jit, IR_EQ, opline), ir_CONST_ADDR(exit_addr)); zend_jit_set_last_valid_opline(jit, opline); return 1; } static bool zend_jit_guard_reference(zend_jit_ctx *jit, const zend_op *opline, zend_jit_addr *var_addr_ptr, zend_jit_addr *ref_addr_ptr, bool add_ref_guard) { zend_jit_addr var_addr = *var_addr_ptr; const void *exit_addr = NULL; ir_ref ref; if (add_ref_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } ref = jit_Z_TYPE(jit, var_addr); ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr)); } ref = jit_Z_PTR(jit, var_addr); *ref_addr_ptr = ZEND_ADDR_REF_ZVAL(ref); ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val)); var_addr = ZEND_ADDR_REF_ZVAL(ref); *var_addr_ptr = var_addr; return 1; } static bool zend_jit_fetch_reference(zend_jit_ctx *jit, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_ref_guard, bool add_type_guard) { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; const void *exit_addr = NULL; ir_ref ref; if (add_ref_guard || add_type_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } } if (add_ref_guard) { ref = jit_Z_TYPE(jit, var_addr); ir_GUARD(ir_EQ(ref, ir_CONST_U8(IS_REFERENCE)), ir_CONST_ADDR(exit_addr)); } if (opline->opcode == ZEND_INIT_METHOD_CALL && opline->op1_type == IS_VAR) { /* Hack: Convert reference to regular value to simplify JIT code for INIT_METHOD_CALL */ ir_CALL_1(IR_VOID, ir_CONST_FC_FUNC(zend_jit_unref_helper), jit_ZVAL_ADDR(jit, var_addr)); *var_addr_ptr = var_addr; } else { ref = jit_Z_PTR(jit, var_addr); ref = ir_ADD_OFFSET(ref, offsetof(zend_reference, val)); var_addr = ZEND_ADDR_REF_ZVAL(ref); *var_addr_ptr = var_addr; } if (var_type != IS_UNKNOWN) { var_type &= ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED); } if (add_type_guard && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { ref = jit_Z_TYPE(jit, var_addr); ir_GUARD(ir_EQ(ref, ir_CONST_U8(var_type)), ir_CONST_ADDR(exit_addr)); ZEND_ASSERT(var_info & (1 << var_type)); if (var_type < IS_STRING) { var_info = (1 << var_type); } else if (var_type != IS_ARRAY) { var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); } else { var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); } *var_info_ptr = var_info; } else { var_info &= ~MAY_BE_REF; *var_info_ptr = var_info; } *var_info_ptr |= MAY_BE_GUARD; /* prevent generation of specialized zval dtor */ return 1; } static bool zend_jit_fetch_indirect_var(zend_jit_ctx *jit, const zend_op *opline, uint8_t var_type, uint32_t *var_info_ptr, zend_jit_addr *var_addr_ptr, bool add_indirect_guard) { zend_jit_addr var_addr = *var_addr_ptr; uint32_t var_info = *var_info_ptr; int32_t exit_point; const void *exit_addr; ir_ref ref = IR_UNUSED; if (add_indirect_guard) { int32_t exit_point = zend_jit_trace_get_exit_point(opline, 0); const void *exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_Z_TYPE(jit, var_addr, IS_INDIRECT, exit_addr); ref = jit_Z_PTR(jit, var_addr); } else { /* This LOAD of INDIRECT VAR, stored by the previous FETCH_(DIM/OBJ)_W, * is eliminated by store forwarding (S2L) */ ref = jit_Z_PTR(jit, var_addr); } *var_info_ptr &= ~MAY_BE_INDIRECT; var_addr = ZEND_ADDR_REF_ZVAL(ref); *var_addr_ptr = var_addr; if (var_type != IS_UNKNOWN) { var_type &= ~(IS_TRACE_INDIRECT|IS_TRACE_PACKED); } if (!(var_type & IS_TRACE_REFERENCE) && var_type != IS_UNKNOWN && (var_info & (MAY_BE_ANY|MAY_BE_UNDEF)) != (1 << var_type)) { exit_point = zend_jit_trace_get_exit_point(opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (!exit_addr) { return 0; } jit_guard_Z_TYPE(jit, var_addr, var_type, exit_addr); //var_info = zend_jit_trace_type_to_info_ex(var_type, var_info); ZEND_ASSERT(var_info & (1 << var_type)); if (var_type < IS_STRING) { var_info = (1 << var_type); } else if (var_type != IS_ARRAY) { var_info = (1 << var_type) | (var_info & (MAY_BE_RC1|MAY_BE_RCN)); } else { var_info = MAY_BE_ARRAY | (var_info & (MAY_BE_ARRAY_OF_ANY|MAY_BE_ARRAY_OF_REF|MAY_BE_ARRAY_KEY_ANY|MAY_BE_RC1|MAY_BE_RCN)); } *var_info_ptr = var_info; } return 1; } static int zend_jit_trace_handler(zend_jit_ctx *jit, const zend_op_array *op_array, const zend_op *opline, int may_throw, zend_jit_trace_rec *trace) { zend_jit_op_array_trace_extension *jit_extension = (zend_jit_op_array_trace_extension*)ZEND_FUNC_INFO(op_array); size_t offset = jit_extension->offset; zend_vm_opcode_handler_func_t handler = ZEND_OP_TRACE_INFO(opline, offset)->call_handler; ir_ref ref; zend_jit_set_ip(jit, opline); if (GCC_GLOBAL_REGS) { ir_CALL(IR_VOID, ir_CONST_FUNC(handler)); } else { ref = ir_CALL_2(IR_ADDR, ir_CONST_FC_FUNC(handler), jit_FP(jit), jit_IP(jit)); if (opline->opcode == ZEND_RETURN || opline->opcode == ZEND_RETURN_BY_REF || opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME || opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_GENERATOR_CREATE || opline->opcode == ZEND_INCLUDE_OR_EVAL) { jit_STORE_IP(jit, ir_AND_A(ref, ir_CONST_ADDR(~ZEND_VM_ENTER_BIT))); } else { jit_STORE_IP(jit, ref); } } if (may_throw && opline->opcode != ZEND_RETURN && opline->opcode != ZEND_RETURN_BY_REF) { zend_jit_check_exception(jit); } while (trace->op != ZEND_JIT_TRACE_VM && trace->op != ZEND_JIT_TRACE_END) { trace++; } if ((!GCC_GLOBAL_REGS && (trace->op != ZEND_JIT_TRACE_END || trace->stop != ZEND_JIT_TRACE_STOP_RETURN)) || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { if (opline->opcode == ZEND_RETURN || opline->opcode == ZEND_RETURN_BY_REF || opline->opcode == ZEND_DO_UCALL || opline->opcode == ZEND_DO_FCALL_BY_NAME || opline->opcode == ZEND_DO_FCALL || opline->opcode == ZEND_GENERATOR_CREATE || opline->opcode == ZEND_INCLUDE_OR_EVAL) { ir_ref addr = jit_EG(current_execute_data); jit_STORE_FP(jit, ir_LOAD_A(addr)); } } if (zend_jit_trace_may_exit(op_array, opline)) { if (opline->opcode == ZEND_RETURN || opline->opcode == ZEND_RETURN_BY_REF || opline->opcode == ZEND_GENERATOR_CREATE) { if (ZEND_VM_KIND == ZEND_VM_KIND_HYBRID) { if (trace->op != ZEND_JIT_TRACE_END || (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { /* this check may be handled by the following OPLINE guard or jmp [IP] */ ir_GUARD(ir_NE(jit_IP(jit), ir_CONST_ADDR(zend_jit_halt_op)), jit_STUB_ADDR(jit, jit_stub_trace_halt)); } } else { /* IP has been cleared of ZEND_VM_ENTER_BIT already */ ir_GUARD(jit_IP(jit), jit_STUB_ADDR(jit, jit_stub_trace_halt)); } } else if (opline->opcode == ZEND_GENERATOR_RETURN || opline->opcode == ZEND_YIELD || opline->opcode == ZEND_YIELD_FROM) { ir_IJMP(jit_STUB_ADDR(jit, jit_stub_trace_halt)); ir_BEGIN(IR_UNUSED); /* unreachable block */ } if (trace->op != ZEND_JIT_TRACE_END || (trace->stop != ZEND_JIT_TRACE_STOP_RETURN && trace->stop < ZEND_JIT_TRACE_STOP_INTERPRETER)) { const zend_op *next_opline = trace->opline; const zend_op *exit_opline = NULL; uint32_t exit_point; const void *exit_addr; uint32_t old_info = 0; uint32_t old_res_info = 0; zend_jit_trace_stack *stack = JIT_G(current_frame)->stack; if (zend_is_smart_branch(opline)) { bool exit_if_true = 0; exit_opline = zend_jit_trace_get_exit_opline(trace, opline + 1, &exit_if_true); } else { switch (opline->opcode) { case ZEND_JMPZ: case ZEND_JMPNZ: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: case ZEND_JMP_SET: case ZEND_COALESCE: case ZEND_JMP_NULL: case ZEND_FE_RESET_R: case ZEND_FE_RESET_RW: exit_opline = (trace->opline == opline + 1) ? OP_JMP_ADDR(opline, opline->op2) : opline + 1; break; case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: exit_opline = (trace->opline == opline + 1) ? ZEND_OFFSET_TO_OPLINE(opline, opline->extended_value) : opline + 1; break; } } switch (opline->opcode) { case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: if (opline->op2_type != IS_UNUSED) { old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); } break; case ZEND_BIND_INIT_STATIC_OR_JMP: if (opline->op1_type == IS_CV) { old_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1); } break; } if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { old_res_info = STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var)); SET_STACK_TYPE(stack, EX_VAR_TO_NUM(opline->result.var), IS_UNKNOWN, 1); } exit_point = zend_jit_trace_get_exit_point(exit_opline, 0); exit_addr = zend_jit_trace_get_exit_addr(exit_point); if (opline->result_type == IS_VAR || opline->result_type == IS_TMP_VAR) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->result.var), old_res_info); } switch (opline->opcode) { case ZEND_FE_FETCH_R: case ZEND_FE_FETCH_RW: if (opline->op2_type != IS_UNUSED) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op2.var), old_info); } break; case ZEND_BIND_INIT_STATIC_OR_JMP: if (opline->op1_type == IS_CV) { SET_STACK_INFO(stack, EX_VAR_TO_NUM(opline->op1.var), old_info); } break; } if (!exit_addr) { return 0; } ir_GUARD(jit_CMP_IP(jit, IR_EQ, next_opline), ir_CONST_ADDR(exit_addr)); } } zend_jit_set_last_valid_opline(jit, trace->opline); return 1; } static int zend_jit_deoptimizer_start(zend_jit_ctx *jit, zend_string *name, uint32_t trace_num, uint32_t exit_num) { zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) ? 0 : IR_START_BR_TARGET); jit->ctx.spill_base = ZREG_FP; jit->op_array = NULL; jit->ssa = NULL; jit->name = zend_string_copy(name); jit->ctx.flags |= IR_SKIP_PROLOGUE; return 1; } static int zend_jit_trace_start(zend_jit_ctx *jit, const zend_op_array *op_array, zend_ssa *ssa, zend_string *name, uint32_t trace_num, zend_jit_trace_info *parent, uint32_t exit_num) { zend_jit_init_ctx(jit, (ZEND_VM_KIND == ZEND_VM_KIND_CALL || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) ? 0 : IR_START_BR_TARGET); jit->ctx.spill_base = ZREG_FP; jit->op_array = NULL; jit->current_op_array = op_array; jit->ssa = ssa; jit->name = zend_string_copy(name); if (!GCC_GLOBAL_REGS) { if (!parent) { if (ZEND_VM_KIND != ZEND_VM_KIND_TAILCALL) { ir_ref execute_data_ref = ir_PARAM(IR_ADDR, "execute_data", 1); ir_ref opline_ref = ir_PARAM(IR_ADDR, "opline", 2); jit_STORE_FP(jit, execute_data_ref); jit_STORE_IP(jit, opline_ref); } jit->ctx.flags |= IR_FASTCALL_FUNC; } } if (parent) { jit->ctx.flags |= IR_SKIP_PROLOGUE; } if (parent) { int i; int parent_vars_count = parent->exit_info[exit_num].stack_size; zend_jit_trace_stack *parent_stack = parent_vars_count == 0 ? NULL : parent->stack_map + parent->exit_info[exit_num].stack_offset; /* prevent clobbering of registers used for deoptimization */ for (i = 0; i < parent_vars_count; i++) { if (STACK_FLAGS(parent_stack, i) != ZREG_CONST && STACK_REG(parent_stack, i) != ZREG_NONE) { int32_t reg = STACK_REG(parent_stack, i); ir_type type; if (STACK_FLAGS(parent_stack, i) == ZREG_ZVAL_COPY) { type = IR_ADDR; } else if (STACK_TYPE(parent_stack, i) == IS_LONG) { type = IR_LONG; } else if (STACK_TYPE(parent_stack, i) == IS_DOUBLE) { type = IR_DOUBLE; } else { ZEND_UNREACHABLE(); } if (ssa && ssa->vars[i].no_val) { /* pass */ } else { ir_ref ref = ir_RLOAD(type, reg); if (STACK_FLAGS(parent_stack, i) & (ZREG_LOAD|ZREG_STORE)) { /* op3 is used as a flag that the value is already stored in memory. * In case the IR framework decides to spill the result of IR_LOAD, * it doesn't have to store the value once again. * * See: insn->op3 check in ir_emit_rload() */ ir_set_op(&jit->ctx, ref, 3, EX_NUM_TO_VAR(i)); } } } } } if (parent && parent->exit_info[exit_num].flags & ZEND_JIT_EXIT_METHOD_CALL) { ZEND_ASSERT(parent->exit_info[exit_num].poly_func.reg >= 0 && parent->exit_info[exit_num].poly_this.reg >= 0); if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_func.reg)) { ir_RLOAD_A(parent->exit_info[exit_num].poly_func.reg); } if (!IR_REG_SPILLED(parent->exit_info[exit_num].poly_this.reg)) { ir_RLOAD_A(parent->exit_info[exit_num].poly_this.reg); } } ir_STORE(jit_EG(jit_trace_num), ir_CONST_U32(trace_num)); return 1; } static int zend_jit_trace_begin_loop(zend_jit_ctx *jit) { return ir_LOOP_BEGIN(ir_END()); } static void zend_jit_trace_gen_phi(zend_jit_ctx *jit, zend_ssa_phi *phi) { int dst_var = phi->ssa_var; int src_var = phi->sources[0]; ir_ref ref; ZEND_ASSERT(!(jit->ra[dst_var].flags & ZREG_LOAD)); ZEND_ASSERT(jit->ra[src_var].ref != IR_UNUSED && jit->ra[src_var].ref != IR_NULL); ref = ir_PHI_2( (jit->ssa->var_info[src_var].type & MAY_BE_LONG) ? IR_LONG : IR_DOUBLE, zend_jit_use_reg(jit, ZEND_ADDR_REG(src_var)), IR_UNUSED); src_var = phi->sources[1]; ZEND_ASSERT(jit->ra[src_var].ref == IR_NULL); jit->ra[src_var].flags |= ZREG_FORWARD; zend_jit_def_reg(jit, ZEND_ADDR_REG(dst_var), ref); } static int zend_jit_trace_end_loop(zend_jit_ctx *jit, int loop_ref, const void *timeout_exit_addr) { if (timeout_exit_addr) { zend_jit_check_timeout(jit, NULL, timeout_exit_addr); } ZEND_ASSERT(jit->ctx.ir_base[loop_ref].op2 == IR_UNUSED); ir_MERGE_SET_OP(loop_ref, 2, ir_LOOP_END()); return 1; } static int zend_jit_trace_return(zend_jit_ctx *jit, bool original_handler, const zend_op *opline) { if (GCC_GLOBAL_REGS || ZEND_VM_KIND == ZEND_VM_KIND_TAILCALL) { if (!original_handler) { zend_jit_tailcall_handler(jit, ir_LOAD_A(jit_IP(jit))); } else { zend_jit_tailcall_handler(jit, zend_jit_orig_opline_handler(jit)); } } else { if (original_handler) { ir_ref ref; ir_ref addr = zend_jit_orig_opline_handler(jit); #if defined(IR_TARGET_X86) addr = ir_CAST_FC_FUNC(addr); #endif ref = ir_CALL_2(IR_ADDR, addr, jit_FP(jit), jit_IP(jit)); zend_jit_vm_enter(jit, ref); return 1; } zend_jit_vm_enter(jit, jit_IP(jit)); } return 1; } static int zend_jit_link_side_trace(const void *code, size_t size, uint32_t jmp_table_size, uint32_t exit_num, const void *addr) { return ir_patch(code, size, jmp_table_size, zend_jit_trace_get_exit_addr(exit_num), addr); } static int zend_jit_trace_link_to_root(zend_jit_ctx *jit, zend_jit_trace_info *t, const void *timeout_exit_addr) { const void *link_addr; /* Skip prologue. */ ZEND_ASSERT(zend_jit_trace_prologue_size != (size_t)-1); link_addr = (const void*)((const char*)t->code_start + zend_jit_trace_prologue_size); if (timeout_exit_addr) { zend_jit_check_timeout(jit, NULL, timeout_exit_addr); } ir_IJMP(ir_CONST_ADDR(link_addr)); return 1; } static bool zend_jit_opline_supports_reg(const zend_op_array *op_array, zend_ssa *ssa, const zend_op *opline, const zend_ssa_op *ssa_op, zend_jit_trace_rec *trace) { uint32_t op1_info, op2_info; switch (opline->opcode) { case ZEND_SEND_VAR: case ZEND_SEND_VAL: case ZEND_SEND_VAL_EX: return (opline->op2_type != IS_CONST) && (opline->opcode != ZEND_SEND_VAL_EX || opline->op2.num <= MAX_ARG_FLAG_NUM); case ZEND_QM_ASSIGN: case ZEND_IS_SMALLER: case ZEND_IS_SMALLER_OR_EQUAL: case ZEND_IS_EQUAL: case ZEND_IS_NOT_EQUAL: case ZEND_IS_IDENTICAL: case ZEND_IS_NOT_IDENTICAL: case ZEND_CASE: return 1; case ZEND_RETURN: return (op_array->type != ZEND_EVAL_CODE && op_array->function_name); case ZEND_ASSIGN: return (opline->op1_type == IS_CV); case ZEND_ASSIGN_OP: if (opline->op1_type != IS_CV || opline->result_type != IS_UNUSED) { return 0; } op1_info = OP1_INFO(); op2_info = OP2_INFO(); return zend_jit_supported_binary_op(opline->extended_value, op1_info, op2_info); case ZEND_ADD: case ZEND_SUB: case ZEND_MUL: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if ((op1_info & MAY_BE_UNDEF) || (op2_info & MAY_BE_UNDEF)) { return 0; } if (trace && trace->op1_type != IS_UNKNOWN) { op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)); } if (trace && trace->op2_type != IS_UNKNOWN) { op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)); } return !(op1_info & MAY_BE_UNDEF) && !(op2_info & MAY_BE_UNDEF) && (op1_info & (MAY_BE_LONG|MAY_BE_DOUBLE)) && (op2_info & (MAY_BE_LONG|MAY_BE_DOUBLE)); case ZEND_BW_OR: case ZEND_BW_AND: case ZEND_BW_XOR: case ZEND_SL: case ZEND_SR: case ZEND_MOD: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (trace && trace->op1_type != IS_UNKNOWN) { op1_info &= 1U << (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)); } if (trace && trace->op2_type != IS_UNKNOWN) { op2_info &= 1U << (trace->op2_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)); } return (op1_info & MAY_BE_LONG) && (op2_info & MAY_BE_LONG); case ZEND_PRE_INC: case ZEND_PRE_DEC: case ZEND_POST_INC: case ZEND_POST_DEC: op1_info = OP1_INFO(); return opline->op1_type == IS_CV && (op1_info & MAY_BE_LONG) && !(op1_info & MAY_BE_REF); case ZEND_STRLEN: op1_info = OP1_INFO(); return (opline->op1_type & (IS_CV|IS_CONST)) && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_STRING; case ZEND_COUNT: op1_info = OP1_INFO(); return (opline->op1_type & (IS_CV|IS_CONST)) && (op1_info & (MAY_BE_ANY|MAY_BE_REF|MAY_BE_UNDEF)) == MAY_BE_ARRAY; case ZEND_JMPZ: case ZEND_JMPNZ: if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { if (!ssa->cfg.map) { return 0; } if (opline > op_array->opcodes + ssa->cfg.blocks[ssa->cfg.map[opline-op_array->opcodes]].start && ((opline-1)->result_type & (IS_SMART_BRANCH_JMPZ|IS_SMART_BRANCH_JMPNZ)) != 0) { return 0; } } ZEND_FALLTHROUGH; case ZEND_BOOL: case ZEND_BOOL_NOT: case ZEND_JMPZ_EX: case ZEND_JMPNZ_EX: return 1; case ZEND_FETCH_CONSTANT: return 1; case ZEND_ISSET_ISEMPTY_DIM_OBJ: if ((opline->extended_value & ZEND_ISEMPTY)) { return 0; } ZEND_FALLTHROUGH; case ZEND_FETCH_DIM_R: case ZEND_FETCH_DIM_IS: case ZEND_FETCH_LIST_R: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (trace && trace->op1_type != IS_UNKNOWN && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); } return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING)); case ZEND_ASSIGN_DIM_OP: if (opline->result_type != IS_UNUSED) { return 0; } if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) { return 0; } ZEND_FALLTHROUGH; case ZEND_ASSIGN_DIM: case ZEND_FETCH_DIM_W: case ZEND_FETCH_DIM_RW: case ZEND_FETCH_LIST_W: op1_info = OP1_INFO(); op2_info = OP2_INFO(); if (trace) { if (opline->op1_type == IS_CV) { if ((opline->opcode == ZEND_ASSIGN_DIM || opline->opcode == ZEND_ASSIGN_DIM_OP) && (opline+1)->op1_type == IS_CV && (opline+1)->op1.var == opline->op1.var) { /* skip $a[x] = $a; */ return 0; } } else if (opline->op1_type == IS_VAR) { if (trace->op1_type == IS_UNKNOWN || !(trace->op1_type & IS_TRACE_INDIRECT) || opline->result_type != IS_UNUSED) { return 0; } } if (trace->op1_type != IS_UNKNOWN && (trace->op1_type & ~(IS_TRACE_REFERENCE|IS_TRACE_INDIRECT|IS_TRACE_PACKED)) == IS_ARRAY) { op1_info &= ~((MAY_BE_ANY|MAY_BE_UNDEF) - MAY_BE_ARRAY); } } else { if (opline->op1_type != IS_CV) { return 0; } } return ((op1_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_ARRAY) && (((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_LONG) || ((op2_info & (MAY_BE_ANY|MAY_BE_UNDEF)) == MAY_BE_STRING)); case ZEND_ASSIGN_OBJ_OP: if (opline->result_type != IS_UNUSED) { return 0; } if (!zend_jit_supported_binary_op(opline->extended_value, MAY_BE_ANY, OP1_DATA_INFO())) { return 0; } ZEND_FALLTHROUGH; case ZEND_FETCH_OBJ_R: case ZEND_ASSIGN_OBJ: if (opline->op2_type != IS_CONST || Z_TYPE_P(RT_CONSTANT(opline, opline->op2)) != IS_STRING || Z_STRVAL_P(RT_CONSTANT(opline, opline->op2))[0] == '\0') { return 0; } op1_info = OP1_INFO(); return opline->op1_type == IS_UNUSED || (op1_info & MAY_BE_OBJECT); } return 0; } static bool zend_jit_var_supports_reg(zend_ssa *ssa, int var) { if (ssa->vars[var].no_val) { /* we don't need the value */ return 0; } if (!(JIT_G(opt_flags) & ZEND_JIT_REG_ALLOC_GLOBAL)) { /* Disable global register allocation, * register allocation for SSA variables connected through Phi functions */ if (ssa->vars[var].definition_phi) { return 0; } if (ssa->vars[var].phi_use_chain) { zend_ssa_phi *phi = ssa->vars[var].phi_use_chain; do { if (!ssa->vars[phi->ssa_var].no_val) { return 0; } phi = zend_ssa_next_use_phi(ssa, var, phi); } while (phi); } } if (((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_DOUBLE) && ((ssa->var_info[var].type & (MAY_BE_ANY|MAY_BE_UNDEF|MAY_BE_REF)) != MAY_BE_LONG)) { /* bad type */ return 0; } return 1; } static bool zend_jit_may_be_in_reg(const zend_op_array *op_array, zend_ssa *ssa, int var) { if (!zend_jit_var_supports_reg(ssa, var)) { return 0; } if (ssa->vars[var].definition >= 0) { uint32_t def = ssa->vars[var].definition; if (!zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + def, ssa->ops + def, NULL)) { return 0; } } if (ssa->vars[var].use_chain >= 0) { int use = ssa->vars[var].use_chain; do { if (!zend_ssa_is_no_val_use(op_array->opcodes + use, ssa->ops + use, var) && !zend_jit_opline_supports_reg(op_array, ssa, op_array->opcodes + use, ssa->ops + use, NULL)) { return 0; } use = zend_ssa_next_use(ssa->ops, var, use); } while (use >= 0); } if (JIT_G(trigger) != ZEND_JIT_ON_HOT_TRACE) { int def_block, use_block, b, use, j; zend_basic_block *bb; zend_ssa_phi *p; bool ret = 1; zend_worklist worklist; ALLOCA_FLAG(use_heap) /* Check if live range is split by ENTRY block */ if (ssa->vars[var].definition >= 0) { def_block =ssa->cfg.map[ssa->vars[var].definition]; } else { ZEND_ASSERT(ssa->vars[var].definition_phi); def_block = ssa->vars[var].definition_phi->block; } ZEND_WORKLIST_ALLOCA(&worklist, ssa->cfg.blocks_count, use_heap); if (ssa->vars[var].use_chain >= 0) { use = ssa->vars[var].use_chain; do { use_block = ssa->cfg.map[use]; if (use_block != def_block) { zend_worklist_push(&worklist, use_block); } use = zend_ssa_next_use(ssa->ops, var, use); } while (use >= 0); } p = ssa->vars[var].phi_use_chain; while (p) { use_block = p->block; if (use_block != def_block) { bb = &ssa->cfg.blocks[use_block]; for (j = 0; j < bb->predecessors_count; j++) { if (p->sources[j] == var) { use_block = ssa->cfg.predecessors[bb->predecessor_offset + j]; if (use_block != def_block) { zend_worklist_push(&worklist, use_block); } } } } p = zend_ssa_next_use_phi(ssa, var, p); } while (zend_worklist_len(&worklist) != 0) { b = zend_worklist_pop(&worklist); bb = &ssa->cfg.blocks[b]; if (bb->flags & (ZEND_BB_ENTRY|ZEND_BB_RECV_ENTRY)) { ret = 0; break; } for (j = 0; j < bb->predecessors_count; j++) { b = ssa->cfg.predecessors[bb->predecessor_offset + j]; if (b != def_block) { zend_worklist_push(&worklist, b); } } } ZEND_WORKLIST_FREE_ALLOCA(&worklist, use_heap); return ret; } return 1; } static ir_ref jit_frameless_observer(zend_jit_ctx *jit, const zend_op *opline) { // JIT: zend_observer_handler_is_unobserved(ZEND_OBSERVER_DATA(fbc)) ir_ref observer_handler; zend_function *fbc = ZEND_FLF_FUNC(opline); // Not need for runtime cache or generator checks here, we just need if_unobserved ir_ref if_unobserved = jit_observer_fcall_is_unobserved_start(jit, fbc, &observer_handler, IR_UNUSED, IR_UNUSED).if_unobserved; // Call zend_frameless_observed_call for the main logic. ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)zend_frameless_observed_call), jit_FP(jit)); ir_ref skip = ir_END(); ir_IF_TRUE(if_unobserved); return skip; } static void jit_frameless_icall0(zend_jit_ctx *jit, const zend_op *opline) { jit_SET_EX_OPLINE(jit, opline); void *function = ZEND_FLF_HANDLER(opline); zend_jit_addr res_addr = RES_ADDR(); ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr); jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); ir_ref skip_observer = IR_UNUSED; if (ZEND_OBSERVER_ENABLED) { skip_observer = jit_frameless_observer(jit, opline); } ir_CALL_1(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref); if (skip_observer != IR_UNUSED) { ir_MERGE_WITH(skip_observer); } zend_jit_check_exception(jit); } static void jit_frameless_icall1(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info) { jit_SET_EX_OPLINE(jit, opline); /* Avoid dropping RC check in case op escapes. */ if (op1_info & MAY_BE_RC1) { op1_info |= MAY_BE_RCN; } void *function = ZEND_FLF_HANDLER(opline); zend_jit_addr res_addr = RES_ADDR(); zend_jit_addr op1_addr = OP1_ADDR(); ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr); ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr); jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1); op1_info &= ~MAY_BE_UNDEF; op1_info |= MAY_BE_NULL; op1_addr = ZEND_ADDR_REF_ZVAL(op1_ref); } if (op1_info & MAY_BE_REF) { op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref); } ir_ref skip_observer = IR_UNUSED; if (ZEND_OBSERVER_ENABLED) { skip_observer = jit_frameless_observer(jit, opline); } ir_CALL_2(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref); if (skip_observer != IR_UNUSED) { ir_MERGE_WITH(skip_observer); } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); zend_jit_check_exception(jit); } static void jit_frameless_icall2(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info) { jit_SET_EX_OPLINE(jit, opline); /* Avoid dropping RC check in case op escapes. */ if (op1_info & MAY_BE_RC1) { op1_info |= MAY_BE_RCN; } if (op2_info & MAY_BE_RC1) { op2_info |= MAY_BE_RCN; } void *function = ZEND_FLF_HANDLER(opline); zend_jit_addr res_addr = RES_ADDR(); zend_jit_addr op1_addr = OP1_ADDR(); zend_jit_addr op2_addr = OP2_ADDR(); ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr); ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr); ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr); jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1); op1_info &= ~MAY_BE_UNDEF; op1_info |= MAY_BE_NULL; op1_addr = ZEND_ADDR_REF_ZVAL(op1_ref); } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1); op2_info &= ~MAY_BE_UNDEF; op2_info |= MAY_BE_NULL; op2_addr = ZEND_ADDR_REF_ZVAL(op2_ref); } if (op1_info & MAY_BE_REF) { op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref); } if (op2_info & MAY_BE_REF) { op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref); } ir_ref skip_observer = IR_UNUSED; if (ZEND_OBSERVER_ENABLED) { skip_observer = jit_frameless_observer(jit, opline); } ir_CALL_3(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref); if (skip_observer != IR_UNUSED) { ir_MERGE_WITH(skip_observer); } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); /* Set OP1 to UNDEF in case FREE_OP2() throws. */ if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) != 0 && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0 && (op2_info & MAY_BE_RC1) && (op2_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF); if (JIT_G(current_frame)) { SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1); } } jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL); zend_jit_check_exception(jit); } static void jit_frameless_icall3(zend_jit_ctx *jit, const zend_op *opline, uint32_t op1_info, uint32_t op2_info, uint32_t op1_data_info) { jit_SET_EX_OPLINE(jit, opline); /* Avoid dropping RC check in case op escapes. */ if (op1_info & MAY_BE_RC1) { op1_info |= MAY_BE_RCN; } if (op2_info & MAY_BE_RC1) { op2_info |= MAY_BE_RCN; } if (op1_data_info & MAY_BE_RC1) { op1_data_info |= MAY_BE_RCN; } void *function = ZEND_FLF_HANDLER(opline); uint8_t op_data_type = (opline + 1)->op1_type; zend_jit_addr res_addr = RES_ADDR(); zend_jit_addr op1_addr = OP1_ADDR(); zend_jit_addr op2_addr = OP2_ADDR(); zend_jit_addr op3_addr = OP1_DATA_ADDR(); ir_ref res_ref = jit_ZVAL_ADDR(jit, res_addr); ir_ref op1_ref = jit_ZVAL_ADDR(jit, op1_addr); ir_ref op2_ref = jit_ZVAL_ADDR(jit, op2_addr); ir_ref op3_ref = jit_ZVAL_ADDR(jit, op3_addr); jit_set_Z_TYPE_INFO(jit, res_addr, IS_NULL); if (opline->op1_type == IS_CV && (op1_info & MAY_BE_UNDEF)) { op1_ref = zend_jit_zval_check_undef(jit, op1_ref, opline->op1.var, opline, 1); op1_info &= ~MAY_BE_UNDEF; op1_info |= MAY_BE_NULL; op1_addr = ZEND_ADDR_REF_ZVAL(op1_ref); } if (opline->op2_type == IS_CV && (op2_info & MAY_BE_UNDEF)) { op2_ref = zend_jit_zval_check_undef(jit, op2_ref, opline->op2.var, opline, 1); op2_info &= ~MAY_BE_UNDEF; op2_info |= MAY_BE_NULL; op2_addr = ZEND_ADDR_REF_ZVAL(op2_ref); } if ((opline+1)->op1_type == IS_CV && (op1_data_info & MAY_BE_UNDEF)) { op3_ref = zend_jit_zval_check_undef(jit, op3_ref, (opline+1)->op1.var, opline, 1); op1_data_info &= ~MAY_BE_UNDEF; op1_data_info |= MAY_BE_NULL; op3_addr = ZEND_ADDR_REF_ZVAL(op3_ref); } if (op1_info & MAY_BE_REF) { op1_ref = jit_ZVAL_DEREF_ref(jit, op1_ref); } if (op2_info & MAY_BE_REF) { op2_ref = jit_ZVAL_DEREF_ref(jit, op2_ref); } if (op1_data_info & MAY_BE_REF) { op3_ref = jit_ZVAL_DEREF_ref(jit, op3_ref); } ir_ref skip_observer = IR_UNUSED; if (ZEND_OBSERVER_ENABLED) { skip_observer = jit_frameless_observer(jit, opline); } ir_CALL_4(IR_VOID, ir_CONST_ADDR((size_t)function), res_ref, op1_ref, op2_ref, op3_ref); if (skip_observer != IR_UNUSED) { ir_MERGE_WITH(skip_observer); } jit_FREE_OP(jit, opline->op1_type, opline->op1, op1_info, NULL); /* Set OP1 to UNDEF in case FREE_OP2() throws. */ bool op1_undef = false; if ((opline->op1_type & (IS_VAR|IS_TMP_VAR)) && (((opline->op2_type & (IS_VAR|IS_TMP_VAR)) && (op2_info & MAY_BE_RC1) && (op2_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) || ((op_data_type & (IS_VAR|IS_TMP_VAR)) && (op1_data_info & MAY_BE_RC1) && (op1_data_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))))) { op1_undef = true; jit_set_Z_TYPE_INFO(jit, op1_addr, IS_UNDEF); if (JIT_G(current_frame)) { SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op1.var), IS_UNKNOWN, 1); } } jit_FREE_OP(jit, opline->op2_type, opline->op2, op2_info, NULL); /* If OP1 is set to UNDEF, we don't need to set OP2 to UNDEF on free because * zend_fetch_debug_backtrace aborts when it encounters the first UNDEF TMP|VAR. */ if (!op1_undef && (opline->op2_type & (IS_VAR|IS_TMP_VAR)) != 0 && (op_data_type & (IS_VAR|IS_TMP_VAR)) != 0 && (op1_data_info & MAY_BE_RC1) && (op1_data_info & (MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_ARRAY_OF_OBJECT|MAY_BE_ARRAY_OF_RESOURCE|MAY_BE_ARRAY_OF_ARRAY))) { jit_set_Z_TYPE_INFO(jit, op2_addr, IS_UNDEF); if (JIT_G(current_frame)) { SET_STACK_TYPE(JIT_G(current_frame)->stack, EX_VAR_TO_NUM(opline->op2.var), IS_UNKNOWN, 1); } } jit_FREE_OP(jit, (opline+1)->op1_type, (opline+1)->op1, op1_data_info, NULL); zend_jit_check_exception(jit); } /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * indent-tabs-mode: t * End: */