Mercurial

comparison third_party/luajit/src/lj_dispatch.c @ 178:94705b5986b3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[ThirdParty] Added WRK and luajit for load testing.
author MrJuneJune <me@mrjunejune.com>
date Thu, 22 Jan 2026 20:10:30 -0800
parents
children
comparison
equal deleted inserted replaced
177:24fe8ff94056 178:94705b5986b3
1 /*
2 ** Instruction dispatch handling.
3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 #define lj_dispatch_c
7 #define LUA_CORE
8
9 #include "lj_obj.h"
10 #include "lj_err.h"
11 #include "lj_buf.h"
12 #include "lj_func.h"
13 #include "lj_str.h"
14 #include "lj_tab.h"
15 #include "lj_meta.h"
16 #include "lj_debug.h"
17 #include "lj_state.h"
18 #include "lj_frame.h"
19 #include "lj_bc.h"
20 #include "lj_ff.h"
21 #include "lj_strfmt.h"
22 #if LJ_HASJIT
23 #include "lj_jit.h"
24 #endif
25 #if LJ_HASFFI
26 #include "lj_ccallback.h"
27 #endif
28 #include "lj_trace.h"
29 #include "lj_dispatch.h"
30 #if LJ_HASPROFILE
31 #include "lj_profile.h"
32 #endif
33 #include "lj_vm.h"
34 #include "luajit.h"
35
36 /* Bump GG_NUM_ASMFF in lj_dispatch.h as needed. Ugly. */
37 LJ_STATIC_ASSERT(GG_NUM_ASMFF == FF_NUM_ASMFUNC);
38
39 /* -- Dispatch table management ------------------------------------------- */
40
41 #if LJ_TARGET_MIPS
42 #include <math.h>
43 LJ_FUNCA_NORET void LJ_FASTCALL lj_ffh_coroutine_wrap_err(lua_State *L,
44 lua_State *co);
45 #if !LJ_HASJIT
46 #define lj_dispatch_stitch lj_dispatch_ins
47 #endif
48 #if !LJ_HASPROFILE
49 #define lj_dispatch_profile lj_dispatch_ins
50 #endif
51
52 #define GOTFUNC(name) (ASMFunction)name,
53 static const ASMFunction dispatch_got[] = {
54 GOTDEF(GOTFUNC)
55 };
56 #undef GOTFUNC
57 #endif
58
59 /* Initialize instruction dispatch table and hot counters. */
60 void lj_dispatch_init(GG_State *GG)
61 {
62 uint32_t i;
63 ASMFunction *disp = GG->dispatch;
64 for (i = 0; i < GG_LEN_SDISP; i++)
65 disp[GG_LEN_DDISP+i] = disp[i] = makeasmfunc(lj_bc_ofs[i]);
66 for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
67 disp[i] = makeasmfunc(lj_bc_ofs[i]);
68 /* The JIT engine is off by default. luaopen_jit() turns it on. */
69 disp[BC_FORL] = disp[BC_IFORL];
70 disp[BC_ITERL] = disp[BC_IITERL];
71 /* Workaround for stable v2.1 bytecode. TODO: Replace with BC_IITERN. */
72 disp[BC_ITERN] = &lj_vm_IITERN;
73 disp[BC_LOOP] = disp[BC_ILOOP];
74 disp[BC_FUNCF] = disp[BC_IFUNCF];
75 disp[BC_FUNCV] = disp[BC_IFUNCV];
76 GG->g.bc_cfunc_ext = GG->g.bc_cfunc_int = BCINS_AD(BC_FUNCC, LUA_MINSTACK, 0);
77 for (i = 0; i < GG_NUM_ASMFF; i++)
78 GG->bcff[i] = BCINS_AD(BC__MAX+i, 0, 0);
79 #if LJ_TARGET_MIPS
80 memcpy(GG->got, dispatch_got, LJ_GOT__MAX*sizeof(ASMFunction *));
81 #endif
82 }
83
84 #if LJ_HASJIT
85 /* Initialize hotcount table. */
86 void lj_dispatch_init_hotcount(global_State *g)
87 {
88 int32_t hotloop = G2J(g)->param[JIT_P_hotloop];
89 HotCount start = (HotCount)(hotloop*HOTCOUNT_LOOP - 1);
90 HotCount *hotcount = G2GG(g)->hotcount;
91 uint32_t i;
92 for (i = 0; i < HOTCOUNT_SIZE; i++)
93 hotcount[i] = start;
94 }
95 #endif
96
97 /* Internal dispatch mode bits. */
98 #define DISPMODE_CALL 0x01 /* Override call dispatch. */
99 #define DISPMODE_RET 0x02 /* Override return dispatch. */
100 #define DISPMODE_INS 0x04 /* Override instruction dispatch. */
101 #define DISPMODE_JIT 0x10 /* JIT compiler on. */
102 #define DISPMODE_REC 0x20 /* Recording active. */
103 #define DISPMODE_PROF 0x40 /* Profiling active. */
104
105 /* Update dispatch table depending on various flags. */
106 void lj_dispatch_update(global_State *g)
107 {
108 uint8_t oldmode = g->dispatchmode;
109 uint8_t mode = 0;
110 #if LJ_HASJIT
111 mode |= (G2J(g)->flags & JIT_F_ON) ? DISPMODE_JIT : 0;
112 mode |= G2J(g)->state != LJ_TRACE_IDLE ?
113 (DISPMODE_REC|DISPMODE_INS|DISPMODE_CALL) : 0;
114 #endif
115 #if LJ_HASPROFILE
116 mode |= (g->hookmask & HOOK_PROFILE) ? (DISPMODE_PROF|DISPMODE_INS) : 0;
117 #endif
118 mode |= (g->hookmask & (LUA_MASKLINE|LUA_MASKCOUNT)) ? DISPMODE_INS : 0;
119 mode |= (g->hookmask & LUA_MASKCALL) ? DISPMODE_CALL : 0;
120 mode |= (g->hookmask & LUA_MASKRET) ? DISPMODE_RET : 0;
121 if (oldmode != mode) { /* Mode changed? */
122 ASMFunction *disp = G2GG(g)->dispatch;
123 ASMFunction f_forl, f_iterl, f_itern, f_loop, f_funcf, f_funcv;
124 g->dispatchmode = mode;
125
126 /* Hotcount if JIT is on, but not while recording. */
127 if ((mode & (DISPMODE_JIT|DISPMODE_REC)) == DISPMODE_JIT) {
128 f_forl = makeasmfunc(lj_bc_ofs[BC_FORL]);
129 f_iterl = makeasmfunc(lj_bc_ofs[BC_ITERL]);
130 f_itern = makeasmfunc(lj_bc_ofs[BC_ITERN]);
131 f_loop = makeasmfunc(lj_bc_ofs[BC_LOOP]);
132 f_funcf = makeasmfunc(lj_bc_ofs[BC_FUNCF]);
133 f_funcv = makeasmfunc(lj_bc_ofs[BC_FUNCV]);
134 } else { /* Otherwise use the non-hotcounting instructions. */
135 f_forl = disp[GG_LEN_DDISP+BC_IFORL];
136 f_iterl = disp[GG_LEN_DDISP+BC_IITERL];
137 f_itern = &lj_vm_IITERN;
138 f_loop = disp[GG_LEN_DDISP+BC_ILOOP];
139 f_funcf = makeasmfunc(lj_bc_ofs[BC_IFUNCF]);
140 f_funcv = makeasmfunc(lj_bc_ofs[BC_IFUNCV]);
141 }
142 /* Init static counting instruction dispatch first (may be copied below). */
143 disp[GG_LEN_DDISP+BC_FORL] = f_forl;
144 disp[GG_LEN_DDISP+BC_ITERL] = f_iterl;
145 disp[GG_LEN_DDISP+BC_ITERN] = f_itern;
146 disp[GG_LEN_DDISP+BC_LOOP] = f_loop;
147
148 /* Set dynamic instruction dispatch. */
149 if ((oldmode ^ mode) & (DISPMODE_PROF|DISPMODE_REC|DISPMODE_INS)) {
150 /* Need to update the whole table. */
151 if (!(mode & DISPMODE_INS)) { /* No ins dispatch? */
152 /* Copy static dispatch table to dynamic dispatch table. */
153 memcpy(&disp[0], &disp[GG_LEN_DDISP], GG_LEN_SDISP*sizeof(ASMFunction));
154 /* Overwrite with dynamic return dispatch. */
155 if ((mode & DISPMODE_RET)) {
156 disp[BC_RETM] = lj_vm_rethook;
157 disp[BC_RET] = lj_vm_rethook;
158 disp[BC_RET0] = lj_vm_rethook;
159 disp[BC_RET1] = lj_vm_rethook;
160 }
161 } else {
162 /* The recording dispatch also checks for hooks. */
163 ASMFunction f = (mode & DISPMODE_PROF) ? lj_vm_profhook :
164 (mode & DISPMODE_REC) ? lj_vm_record : lj_vm_inshook;
165 uint32_t i;
166 for (i = 0; i < GG_LEN_SDISP; i++)
167 disp[i] = f;
168 }
169 } else if (!(mode & DISPMODE_INS)) {
170 /* Otherwise set dynamic counting ins. */
171 disp[BC_FORL] = f_forl;
172 disp[BC_ITERL] = f_iterl;
173 disp[BC_ITERN] = f_itern;
174 disp[BC_LOOP] = f_loop;
175 /* Set dynamic return dispatch. */
176 if ((mode & DISPMODE_RET)) {
177 disp[BC_RETM] = lj_vm_rethook;
178 disp[BC_RET] = lj_vm_rethook;
179 disp[BC_RET0] = lj_vm_rethook;
180 disp[BC_RET1] = lj_vm_rethook;
181 } else {
182 disp[BC_RETM] = disp[GG_LEN_DDISP+BC_RETM];
183 disp[BC_RET] = disp[GG_LEN_DDISP+BC_RET];
184 disp[BC_RET0] = disp[GG_LEN_DDISP+BC_RET0];
185 disp[BC_RET1] = disp[GG_LEN_DDISP+BC_RET1];
186 }
187 }
188
189 /* Set dynamic call dispatch. */
190 if ((oldmode ^ mode) & DISPMODE_CALL) { /* Update the whole table? */
191 uint32_t i;
192 if ((mode & DISPMODE_CALL) == 0) { /* No call hooks? */
193 for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
194 disp[i] = makeasmfunc(lj_bc_ofs[i]);
195 } else {
196 for (i = GG_LEN_SDISP; i < GG_LEN_DDISP; i++)
197 disp[i] = lj_vm_callhook;
198 }
199 }
200 if (!(mode & DISPMODE_CALL)) { /* Overwrite dynamic counting ins. */
201 disp[BC_FUNCF] = f_funcf;
202 disp[BC_FUNCV] = f_funcv;
203 }
204
205 #if LJ_HASJIT
206 /* Reset hotcounts for JIT off to on transition. */
207 if ((mode & DISPMODE_JIT) && !(oldmode & DISPMODE_JIT))
208 lj_dispatch_init_hotcount(g);
209 #endif
210 }
211 }
212
213 /* -- JIT mode setting ---------------------------------------------------- */
214
215 #if LJ_HASJIT
216 /* Set JIT mode for a single prototype. */
217 static void setptmode(global_State *g, GCproto *pt, int mode)
218 {
219 if ((mode & LUAJIT_MODE_ON)) { /* (Re-)enable JIT compilation. */
220 pt->flags &= ~PROTO_NOJIT;
221 lj_trace_reenableproto(pt); /* Unpatch all ILOOP etc. bytecodes. */
222 } else { /* Flush and/or disable JIT compilation. */
223 if (!(mode & LUAJIT_MODE_FLUSH))
224 pt->flags |= PROTO_NOJIT;
225 lj_trace_flushproto(g, pt); /* Flush all traces of prototype. */
226 }
227 }
228
229 /* Recursively set the JIT mode for all children of a prototype. */
230 static void setptmode_all(global_State *g, GCproto *pt, int mode)
231 {
232 ptrdiff_t i;
233 if (!(pt->flags & PROTO_CHILD)) return;
234 for (i = -(ptrdiff_t)pt->sizekgc; i < 0; i++) {
235 GCobj *o = proto_kgc(pt, i);
236 if (o->gch.gct == ~LJ_TPROTO) {
237 setptmode(g, gco2pt(o), mode);
238 setptmode_all(g, gco2pt(o), mode);
239 }
240 }
241 }
242 #endif
243
244 /* Public API function: control the JIT engine. */
245 int luaJIT_setmode(lua_State *L, int idx, int mode)
246 {
247 global_State *g = G(L);
248 int mm = mode & LUAJIT_MODE_MASK;
249 lj_trace_abort(g); /* Abort recording on any state change. */
250 /* Avoid pulling the rug from under our own feet. */
251 if ((g->hookmask & HOOK_GC))
252 lj_err_caller(L, LJ_ERR_NOGCMM);
253 switch (mm) {
254 #if LJ_HASJIT
255 case LUAJIT_MODE_ENGINE:
256 if ((mode & LUAJIT_MODE_FLUSH)) {
257 lj_trace_flushall(L);
258 } else {
259 if (!(mode & LUAJIT_MODE_ON))
260 G2J(g)->flags &= ~(uint32_t)JIT_F_ON;
261 else
262 G2J(g)->flags |= (uint32_t)JIT_F_ON;
263 lj_dispatch_update(g);
264 }
265 break;
266 case LUAJIT_MODE_FUNC:
267 case LUAJIT_MODE_ALLFUNC:
268 case LUAJIT_MODE_ALLSUBFUNC: {
269 cTValue *tv = idx == 0 ? frame_prev(L->base-1)-LJ_FR2 :
270 idx > 0 ? L->base + (idx-1) : L->top + idx;
271 GCproto *pt;
272 if ((idx == 0 || tvisfunc(tv)) && isluafunc(&gcval(tv)->fn))
273 pt = funcproto(&gcval(tv)->fn); /* Cannot use funcV() for frame slot. */
274 else if (tvisproto(tv))
275 pt = protoV(tv);
276 else
277 return 0; /* Failed. */
278 if (mm != LUAJIT_MODE_ALLSUBFUNC)
279 setptmode(g, pt, mode);
280 if (mm != LUAJIT_MODE_FUNC)
281 setptmode_all(g, pt, mode);
282 break;
283 }
284 case LUAJIT_MODE_TRACE:
285 if (!(mode & LUAJIT_MODE_FLUSH))
286 return 0; /* Failed. */
287 lj_trace_flush(G2J(g), idx);
288 break;
289 #else
290 case LUAJIT_MODE_ENGINE:
291 case LUAJIT_MODE_FUNC:
292 case LUAJIT_MODE_ALLFUNC:
293 case LUAJIT_MODE_ALLSUBFUNC:
294 UNUSED(idx);
295 if ((mode & LUAJIT_MODE_ON))
296 return 0; /* Failed. */
297 break;
298 #endif
299 case LUAJIT_MODE_WRAPCFUNC:
300 if ((mode & LUAJIT_MODE_ON)) {
301 if (idx != 0) {
302 cTValue *tv = idx > 0 ? L->base + (idx-1) : L->top + idx;
303 if (tvislightud(tv))
304 g->wrapf = (lua_CFunction)lightudV(g, tv);
305 else
306 return 0; /* Failed. */
307 } else {
308 return 0; /* Failed. */
309 }
310 g->bc_cfunc_ext = BCINS_AD(BC_FUNCCW, 0, 0);
311 } else {
312 g->bc_cfunc_ext = BCINS_AD(BC_FUNCC, 0, 0);
313 }
314 break;
315 default:
316 return 0; /* Failed. */
317 }
318 return 1; /* OK. */
319 }
320
321 /* Enforce (dynamic) linker error for version mismatches. See luajit.c. */
322 LUA_API void LUAJIT_VERSION_SYM(void)
323 {
324 }
325
326 /* -- Hooks --------------------------------------------------------------- */
327
328 /* This function can be called asynchronously (e.g. during a signal). */
329 LUA_API int lua_sethook(lua_State *L, lua_Hook func, int mask, int count)
330 {
331 global_State *g = G(L);
332 mask &= HOOK_EVENTMASK;
333 if (func == NULL || mask == 0) { mask = 0; func = NULL; } /* Consistency. */
334 g->hookf = func;
335 g->hookcount = g->hookcstart = (int32_t)count;
336 g->hookmask = (uint8_t)((g->hookmask & ~HOOK_EVENTMASK) | mask);
337 lj_trace_abort(g); /* Abort recording on any hook change. */
338 lj_dispatch_update(g);
339 return 1;
340 }
341
342 LUA_API lua_Hook lua_gethook(lua_State *L)
343 {
344 return G(L)->hookf;
345 }
346
347 LUA_API int lua_gethookmask(lua_State *L)
348 {
349 return G(L)->hookmask & HOOK_EVENTMASK;
350 }
351
352 LUA_API int lua_gethookcount(lua_State *L)
353 {
354 return (int)G(L)->hookcstart;
355 }
356
357 /* Call a hook. */
358 static void callhook(lua_State *L, int event, BCLine line)
359 {
360 global_State *g = G(L);
361 lua_Hook hookf = g->hookf;
362 if (hookf && !hook_active(g)) {
363 lua_Debug ar;
364 lj_trace_abort(g); /* Abort recording on any hook call. */
365 ar.event = event;
366 ar.currentline = line;
367 /* Top frame, nextframe = NULL. */
368 ar.i_ci = (int)((L->base-1) - tvref(L->stack));
369 lj_state_checkstack(L, 1+LUA_MINSTACK);
370 #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
371 lj_profile_hook_enter(g);
372 #else
373 hook_enter(g);
374 #endif
375 hookf(L, &ar);
376 lj_assertG(hook_active(g), "active hook flag removed");
377 setgcref(g->cur_L, obj2gco(L));
378 #if LJ_HASPROFILE && !LJ_PROFILE_SIGPROF
379 lj_profile_hook_leave(g);
380 #else
381 hook_leave(g);
382 #endif
383 }
384 }
385
386 /* -- Dispatch callbacks -------------------------------------------------- */
387
388 /* Calculate number of used stack slots in the current frame. */
389 static BCReg cur_topslot(GCproto *pt, const BCIns *pc, uint32_t nres)
390 {
391 BCIns ins = pc[-1];
392 if (bc_op(ins) == BC_UCLO)
393 ins = pc[bc_j(ins)];
394 switch (bc_op(ins)) {
395 case BC_CALLM: case BC_CALLMT: return bc_a(ins) + bc_c(ins) + nres-1+1+LJ_FR2;
396 case BC_RETM: return bc_a(ins) + bc_d(ins) + nres-1;
397 case BC_TSETM: return bc_a(ins) + nres-1;
398 default: return pt->framesize;
399 }
400 }
401
402 /* Instruction dispatch. Used by instr/line/return hooks or when recording. */
403 void LJ_FASTCALL lj_dispatch_ins(lua_State *L, const BCIns *pc)
404 {
405 ERRNO_SAVE
406 GCfunc *fn = curr_func(L);
407 GCproto *pt = funcproto(fn);
408 void *cf = cframe_raw(L->cframe);
409 const BCIns *oldpc = cframe_pc(cf);
410 global_State *g = G(L);
411 BCReg slots;
412 setcframe_pc(cf, pc);
413 slots = cur_topslot(pt, pc, cframe_multres_n(cf));
414 L->top = L->base + slots; /* Fix top. */
415 #if LJ_HASJIT
416 {
417 jit_State *J = G2J(g);
418 if (J->state != LJ_TRACE_IDLE) {
419 #ifdef LUA_USE_ASSERT
420 ptrdiff_t delta = L->top - L->base;
421 #endif
422 J->L = L;
423 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
424 lj_assertG(L->top - L->base == delta,
425 "unbalanced stack after tracing of instruction");
426 }
427 }
428 #endif
429 if ((g->hookmask & LUA_MASKCOUNT) && g->hookcount == 0) {
430 g->hookcount = g->hookcstart;
431 callhook(L, LUA_HOOKCOUNT, -1);
432 L->top = L->base + slots; /* Fix top again. */
433 }
434 if ((g->hookmask & LUA_MASKLINE)) {
435 BCPos npc = proto_bcpos(pt, pc) - 1;
436 BCPos opc = proto_bcpos(pt, oldpc) - 1;
437 BCLine line = lj_debug_line(pt, npc);
438 if (pc <= oldpc || opc >= pt->sizebc || line != lj_debug_line(pt, opc)) {
439 callhook(L, LUA_HOOKLINE, line);
440 L->top = L->base + slots; /* Fix top again. */
441 }
442 }
443 if ((g->hookmask & LUA_MASKRET) && bc_isret(bc_op(pc[-1])))
444 callhook(L, LUA_HOOKRET, -1);
445 ERRNO_RESTORE
446 }
447
448 /* Initialize call. Ensure stack space and return # of missing parameters. */
449 static int call_init(lua_State *L, GCfunc *fn)
450 {
451 if (isluafunc(fn)) {
452 GCproto *pt = funcproto(fn);
453 int numparams = pt->numparams;
454 int gotparams = (int)(L->top - L->base);
455 int need = pt->framesize;
456 if ((pt->flags & PROTO_VARARG)) need += 1+gotparams;
457 lj_state_checkstack(L, (MSize)need);
458 numparams -= gotparams;
459 return numparams >= 0 ? numparams : 0;
460 } else {
461 lj_state_checkstack(L, LUA_MINSTACK);
462 return 0;
463 }
464 }
465
466 /* Call dispatch. Used by call hooks, hot calls or when recording. */
467 ASMFunction LJ_FASTCALL lj_dispatch_call(lua_State *L, const BCIns *pc)
468 {
469 ERRNO_SAVE
470 GCfunc *fn = curr_func(L);
471 BCOp op;
472 global_State *g = G(L);
473 #if LJ_HASJIT
474 jit_State *J = G2J(g);
475 #endif
476 int missing = call_init(L, fn);
477 #if LJ_HASJIT
478 J->L = L;
479 if ((uintptr_t)pc & 1) { /* Marker for hot call. */
480 #ifdef LUA_USE_ASSERT
481 ptrdiff_t delta = L->top - L->base;
482 #endif
483 pc = (const BCIns *)((uintptr_t)pc & ~(uintptr_t)1);
484 lj_trace_hot(J, pc);
485 lj_assertG(L->top - L->base == delta,
486 "unbalanced stack after hot call");
487 goto out;
488 } else if (J->state != LJ_TRACE_IDLE &&
489 !(g->hookmask & (HOOK_GC|HOOK_VMEVENT))) {
490 #ifdef LUA_USE_ASSERT
491 ptrdiff_t delta = L->top - L->base;
492 #endif
493 /* Record the FUNC* bytecodes, too. */
494 lj_trace_ins(J, pc-1); /* The interpreter bytecode PC is offset by 1. */
495 lj_assertG(L->top - L->base == delta,
496 "unbalanced stack after hot instruction");
497 }
498 #endif
499 if ((g->hookmask & LUA_MASKCALL)) {
500 int i;
501 for (i = 0; i < missing; i++) /* Add missing parameters. */
502 setnilV(L->top++);
503 callhook(L, LUA_HOOKCALL, -1);
504 /* Preserve modifications of missing parameters by lua_setlocal(). */
505 while (missing-- > 0 && tvisnil(L->top - 1))
506 L->top--;
507 }
508 #if LJ_HASJIT
509 out:
510 #endif
511 op = bc_op(pc[-1]); /* Get FUNC* op. */
512 #if LJ_HASJIT
513 /* Use the non-hotcounting variants if JIT is off or while recording. */
514 if ((!(J->flags & JIT_F_ON) || J->state != LJ_TRACE_IDLE) &&
515 (op == BC_FUNCF || op == BC_FUNCV))
516 op = (BCOp)((int)op+(int)BC_IFUNCF-(int)BC_FUNCF);
517 #endif
518 ERRNO_RESTORE
519 return makeasmfunc(lj_bc_ofs[op]); /* Return static dispatch target. */
520 }
521
522 #if LJ_HASJIT
523 /* Stitch a new trace. */
524 void LJ_FASTCALL lj_dispatch_stitch(jit_State *J, const BCIns *pc)
525 {
526 ERRNO_SAVE
527 lua_State *L = J->L;
528 void *cf = cframe_raw(L->cframe);
529 const BCIns *oldpc = cframe_pc(cf);
530 setcframe_pc(cf, pc);
531 /* Before dispatch, have to bias PC by 1. */
532 L->top = L->base + cur_topslot(curr_proto(L), pc+1, cframe_multres_n(cf));
533 lj_trace_stitch(J, pc-1); /* Point to the CALL instruction. */
534 setcframe_pc(cf, oldpc);
535 ERRNO_RESTORE
536 }
537 #endif
538
539 #if LJ_HASPROFILE
540 /* Profile dispatch. */
541 void LJ_FASTCALL lj_dispatch_profile(lua_State *L, const BCIns *pc)
542 {
543 ERRNO_SAVE
544 GCfunc *fn = curr_func(L);
545 GCproto *pt = funcproto(fn);
546 void *cf = cframe_raw(L->cframe);
547 const BCIns *oldpc = cframe_pc(cf);
548 global_State *g;
549 setcframe_pc(cf, pc);
550 L->top = L->base + cur_topslot(pt, pc, cframe_multres_n(cf));
551 lj_profile_interpreter(L);
552 setcframe_pc(cf, oldpc);
553 g = G(L);
554 setgcref(g->cur_L, obj2gco(L));
555 setvmstate(g, INTERP);
556 ERRNO_RESTORE
557 }
558 #endif
559