Speed up call frame leaving by grouping refcounted CVs

Leaving the call frame must destroy all the CVs.
It loops over all the vars and then uses i_zval_ptr_dtor on them.
We can skip the non-refcounted CVs in bulk to speed up that loop.

Author: ndossche

Zend/Optimizer/compact_vars.c

@@ -18,17 +18,69 @@
 #include "zend_bitset.h"
 #include "zend_observer.h"
 
+/* Types a CV may have, excluding array kinds. */
+#define CV_TYPE_KINDS (MAY_BE_UNDEF|MAY_BE_ANY|MAY_BE_REF)
+/* Type kinds that are not refcounted. */
+#define CV_TYPE_KINDS_NO_RC (MAY_BE_UNDEF|MAY_BE_NULL|MAY_BE_BOOL|MAY_BE_LONG|MAY_BE_DOUBLE)
+
+/* Fills in `rc_cvs` with the CV numbers that may be refcounted, based on SSA info (if available). */
+static void compute_rc_cvs(
+		const zend_op_array *op_array, const zend_ssa *ssa, zend_bitset rc_cvs)
+{
+	if (!ssa || !ssa->var_info) {
+		/* No type information: every CV has to be assumed refcounted. */
+		zend_bitset_fill(rc_cvs, zend_bitset_len(op_array->last_var));
+		return;
+	}
+
+	zend_bitset_clear(rc_cvs, zend_bitset_len(op_array->last_var));
+
+	/* TODO: Argument slots right now have to be always destroyed because
+	 * when ZEND_RECV fails, they can be refcounted (e.g. passing an array to an int parameter).
+	 * This can be improved by making the ZEND_RECV op destroy the argument iself. */
+	for (uint32_t i = 0; i < MIN((uint32_t) op_array->last_var, op_array->num_args); i++) {
+		zend_bitset_incl(rc_cvs, i);
+	}
+	for (int i = 0; i < ssa->vars_count; i++) {
+		int cv = ssa->vars[i].var;
+
+		if (cv < 0 || cv >= op_array->last_var || zend_bitset_in(rc_cvs, cv)) {
+			continue;
+		}
+		if (ssa->var_info[i].type & CV_TYPE_KINDS & ~CV_TYPE_KINDS_NO_RC) {
+			zend_bitset_incl(rc_cvs, cv);
+		}
+	}
+
+	/* ZEND_RETURN_BY_REF turns its op1 into a reference without being recorded as
+	 * such in the SSA. */
+	/* TODO: should this be recorded by type inference and SSA construction via a new def? */
+	if (op_array->fn_flags & ZEND_ACC_RETURN_REFERENCE) {
+		for (uint32_t i = 0; i < op_array->last; i++) {
+			const zend_op *opline = &op_array->opcodes[i];
+
+			if (opline->opcode == ZEND_RETURN_BY_REF && opline->op1_type == IS_CV) {
+				zend_bitset_incl(rc_cvs, EX_VAR_TO_NUM(opline->op1.var));
+			}
+		}
+	}
+}
+
 /* This pass removes all CVs and temporaries that are completely unused. It does *not* merge any CVs or TMPs.
- * This pass does not operate on SSA form anymore. */
-void zend_optimizer_compact_vars(zend_op_array *op_array) {
+ *
+ * It also sorts the refcounted CVs so they appear at the front, to fill in `op_array->last_var_to_free`.
+ * This improves call frame cleanup performance by skipping (in bulk) the CVs that are not refcounted. */
+void zend_optimizer_compact_vars(zend_op_array *op_array, const zend_ssa *ssa) {
 	int i;
 
 	ALLOCA_FLAG(use_heap1);
 	ALLOCA_FLAG(use_heap2);
+	ALLOCA_FLAG(use_heap3);
 	uint32_t used_vars_len = zend_bitset_len(op_array->last_var + op_array->T);
 	zend_bitset used_vars = ZEND_BITSET_ALLOCA(used_vars_len, use_heap1);
 	uint32_t *vars_map = do_alloca((op_array->last_var + op_array->T) * sizeof(uint32_t), use_heap2);
-	uint32_t num_cvs, num_tmps;
+	uint32_t rc_cvs_len = zend_bitset_len(op_array->last_var);
+	zend_bitset rc_cvs = ZEND_BITSET_ALLOCA(rc_cvs_len, use_heap3);
 
 	/* Determine which CVs are used */
 	zend_bitset_clear(used_vars, used_vars_len);
@@ -52,16 +104,36 @@ void zend_optimizer_compact_vars(zend_op_array *op_array) {
 		}
 	}
 
-	num_cvs = 0;
+	compute_rc_cvs(op_array, ssa, rc_cvs);
+
+	uint32_t num_cvs = 0;
+	uint32_t last_var_to_free = 0;
 	for (i = 0; i < op_array->last_var; i++) {
-		if (zend_bitset_in(used_vars, i)) {
-			vars_map[i] = num_cvs++;
-		} else {
+		/* Parameters have a fixed position because other components depend on that ordering. */
+		if (i < op_array->num_args) {
+			ZEND_ASSERT(zend_bitset_in(used_vars, i)
+				&& "A parameter CV is written by its RECV and cannot be unused");
+		} else if (!zend_bitset_in(used_vars, i) || !zend_bitset_in(rc_cvs, i)) {
 			vars_map[i] = (uint32_t) -1;
+			continue;
+		}
+		vars_map[i] = num_cvs++;
+		if (zend_bitset_in(rc_cvs, i)) {
+			/* Parameters keep their slot even when they are not refcounted,
+			 * so take the highest refcounted slot rather than the group size. */
+			last_var_to_free = vars_map[i] + 1;
+		}
+	}
+
+	/* The CVs that can never hold a refcounted value go last, after `last_var_to_free`. */
+	for (i = op_array->num_args; i < op_array->last_var; i++) {
+		if (vars_map[i] == (uint32_t) -1 && zend_bitset_in(used_vars, i)) {
+			vars_map[i] = num_cvs++;
 		}
 	}
+	ZEND_ASSERT(last_var_to_free <= num_cvs);
 
-	num_tmps = 0;
+	uint32_t num_tmps = 0;
 	for (i = op_array->last_var; i < op_array->last_var + op_array->T; i++) {
 		if (zend_bitset_in(used_vars, i)) {
 			vars_map[i] = num_cvs + num_tmps++;
@@ -71,9 +143,24 @@ void zend_optimizer_compact_vars(zend_op_array *op_array) {
 	}
 
 	free_alloca(used_vars, use_heap1);
+	free_alloca(rc_cvs, use_heap3);
+
+	op_array->last_var_to_free = last_var_to_free;
+
+	/* Nothing was removed and no CV moved: the rewrite below would be a no-op.
+	 * Check if anything got moved, as the number of vars could still be the same in that case. */
 	if (num_cvs == op_array->last_var && num_tmps == op_array->T) {
-		free_alloca(vars_map, use_heap2);
-		return;
+		bool identity = true;
+		for (i = 0; i < op_array->last_var; i++) {
+			if (vars_map[i] != (uint32_t) i) {
+				identity = false;
+				break;
+			}
+		}
+		if (identity) {
+			free_alloca(vars_map, use_heap2);
+			return;
+		}
 	}
 
 	ZEND_ASSERT(num_cvs <= op_array->last_var);
@@ -93,8 +180,8 @@ void zend_optimizer_compact_vars(zend_op_array *op_array) {
 		}
 	}
 
-	/* Update CV name table */
-	if (num_cvs != op_array->last_var) {
+	/* Update CV name table, either because of CVs being removed or being moved. */
+	{
 		if (num_cvs) {
 			zend_string **names = safe_emalloc(sizeof(zend_string *), num_cvs, 0);
 			for (i = 0; i < op_array->last_var; i++) {

Zend/Optimizer/zend_dump.c

@@ -968,6 +968,9 @@ ZEND_API void zend_dump_op_array(const zend_op_array *op_array, uint32_t dump_fl
 		op_array->last,
 		op_array->num_args);
 	fprintf(stderr, ", vars=%d, tmps=%d", op_array->last_var, op_array->T);
+	if (op_array->last_var_to_free != (uint32_t) op_array->last_var) {
+		fprintf(stderr, ", vars_to_free=%d", op_array->last_var_to_free);
+	}
 	if (ssa) {
 		fprintf(stderr, ", ssa_vars=%d", ssa->vars_count);
 	}

Zend/Optimizer/zend_optimizer.c

@@ -1159,7 +1159,9 @@ static void zend_optimize(zend_op_array      *op_array,
 	if ((ZEND_OPTIMIZER_PASS_13 & ctx->optimization_level) &&
 	    (!(ZEND_OPTIMIZER_PASS_6 & ctx->optimization_level) ||
 	     !(ZEND_OPTIMIZER_PASS_7 & ctx->optimization_level))) {
-		zend_optimizer_compact_vars(op_array);
+		/* Reached only when the DFA pass did not run, so there is no type
+		 * information to decide which CVs need to be destroyed. */
+		zend_optimizer_compact_vars(op_array, NULL);
 		if (ctx->debug_level & ZEND_DUMP_AFTER_PASS_13) {
 			zend_dump_op_array(op_array, 0, "after pass 13", NULL);
 		}
@@ -1685,7 +1687,9 @@ ZEND_API void zend_optimize_script(zend_script *script, zend_long optimization_l
 
 		if (ZEND_OPTIMIZER_PASS_13 & optimization_level) {
 			for (i = 0; i < call_graph.op_arrays_count; i++) {
-				zend_optimizer_compact_vars(call_graph.op_arrays[i]);
+				func_info = ZEND_FUNC_INFO(call_graph.op_arrays[i]);
+				zend_optimizer_compact_vars(call_graph.op_arrays[i],
+					func_info ? &func_info->ssa : NULL);
 				if (debug_level & ZEND_DUMP_AFTER_PASS_13) {
 					zend_dump_op_array(call_graph.op_arrays[i], 0, "after pass 13", NULL);
 				}

Zend/Optimizer/zend_optimizer_internal.h

@@ -116,7 +116,7 @@ void zend_dfa_optimize_op_array(zend_op_array *op_array, zend_optimizer_ctx *ctx
 void zend_optimize_temporary_variables(zend_op_array *op_array, zend_optimizer_ctx *ctx);
 void zend_optimizer_nop_removal(zend_op_array *op_array, zend_optimizer_ctx *ctx);
 void zend_optimizer_compact_literals(zend_op_array *op_array, zend_optimizer_ctx *ctx);
-void zend_optimizer_compact_vars(zend_op_array *op_array);
+void zend_optimizer_compact_vars(zend_op_array *op_array, const zend_ssa *ssa);
 zend_function *zend_optimizer_get_called_func(
 		const zend_script *script, const zend_op_array *op_array, zend_op *opline, bool *is_prototype);
 uint32_t zend_optimizer_classify_function(const zend_string *name, uint32_t num_args);

Zend/zend_compile.h

@@ -552,6 +552,8 @@ struct _zend_op_array {
 	uint32_t cache_size; /* number of run_time_cache_slots * sizeof(void*) */
 	int last_var;        /* number of CV variables */
 	uint32_t last;       /* number of opcodes */
+	/* Number of leading CV slots that have to be destroyed when the call frame is left. */
+	uint32_t last_var_to_free;
 
 	zend_op *opcodes;
 	ZEND_MAP_PTR_DEF(HashTable *, static_variables_ptr);

Zend/zend_execute.c

@@ -4298,7 +4298,7 @@ ZEND_API void zend_clean_and_cache_symbol_table(zend_array *symbol_table) /* {{{
 static zend_always_inline void i_free_compiled_variables(zend_execute_data *execute_data) /* {{{ */
 {
 	zval *cv = EX_VAR_NUM(0);
-	int count = EX(func)->op_array.last_var;
+	uint32_t count = EX(func)->op_array.last_var_to_free;
 	while (EXPECTED(count != 0)) {
 		i_zval_ptr_dtor(cv);
 		cv++;

Zend/zend_object_handlers.c

@@ -1827,6 +1827,7 @@ ZEND_API ZEND_ATTRIBUTE_NONNULL zend_function *zend_get_call_trampoline_func(
 	 * value so that it doesn't contain garbage when the engine allocates space for the next stack
 	 * frame. This didn't cause any issues until now due to "lucky" structure layout. */
 	func->last_var = 0;
+	func->last_var_to_free = 0;
 	uint32_t min_T = 2 + ZEND_OBSERVER_ENABLED;
 	func->T = (fbc->type == ZEND_USER_FUNCTION)? MAX(fbc->op_array.last_var + fbc->op_array.T, min_T) : min_T;
 	func->filename = (fbc->type == ZEND_USER_FUNCTION)? fbc->op_array.filename : ZSTR_EMPTY_ALLOC();

Zend/zend_opcode.c

@@ -57,6 +57,7 @@ void init_op_array(zend_op_array *op_array, zend_function_type type, int initial
 	op_array->opcodes = emalloc(initial_ops_size * sizeof(zend_op));
 
 	op_array->last_var = 0;
+	op_array->last_var_to_free = 0;
 	op_array->vars = NULL;
 
 	op_array->T = 0;
@@ -1120,6 +1121,9 @@ ZEND_API void pass_two(zend_op_array *op_array)
 		CG(context).vars_size = op_array->last_var;
 	}
 
+	/* Without type inference every CV has to be assumed refcounted; this can be lowered via the optimizer. */
+	op_array->last_var_to_free = op_array->last_var;
+
 #if ZEND_USE_ABS_CONST_ADDR
 	if (CG(context).opcodes_size != op_array->last) {
 		op_array->opcodes = (zend_op *) erealloc(op_array->opcodes, sizeof(zend_op)*op_array->last);

ext/opcache/jit/zend_jit.c

@@ -2930,17 +2930,15 @@ static int zend_jit(const zend_op_array *op_array, zend_ssa *ssa, const zend_op
 		zend_jit_common_return(jit);
 
 		bool left_frame = false;
-		if (op_array->last_var > 100) {
+		if (op_array->last_var_to_free > 100) {
 			/* To many CVs to unroll */
 			if (!zend_jit_free_cvs(&ctx)) {
 				goto jit_failure;
 			}
 			left_frame = true;
 		}
 		if (!left_frame) {
-			int j;
-
-			for (j = 0 ; j < op_array->last_var; j++) {
+			for (uint32_t j = 0; j < op_array->last_var_to_free; j++) {
 				uint32_t info = zend_ssa_cv_info(op_array, ssa, j);
 
 				if (info & (MAY_BE_STRING|MAY_BE_ARRAY|MAY_BE_OBJECT|MAY_BE_RESOURCE|MAY_BE_REF)) {

ext/opcache/jit/zend_jit_trace.c

@@ -5597,23 +5597,22 @@ static zend_vm_opcode_handler_t zend_jit_trace(zend_jit_trace_rec *trace_buffer,
 								goto jit_failure;
 							}
 						} else {
-							int j;
 							int may_throw = 0;
 							bool left_frame = 0;
 
 							if (!zend_jit_return(&ctx, opline, op_array,
 									op1_info, OP1_REG_ADDR())) {
 								goto jit_failure;
 							}
-							if (op_array->last_var > 100) {
+							if (op_array->last_var_to_free > 100) {
 								/* To many CVs to unroll */
 								if (!zend_jit_free_cvs(&ctx)) {
 									goto jit_failure;
 								}
 								left_frame = 1;
 							}
 							if (!left_frame) {
-								for (j = 0 ; j < op_array->last_var; j++) {
+								for (uint32_t j = 0; j < op_array->last_var_to_free; j++) {
 									uint32_t info;
 									uint8_t type;