Mercurial

comparison third_party/luajit/src/vm_arm64.dasc @ 178:94705b5986b3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[ThirdParty] Added WRK and luajit for load testing.
author MrJuneJune <me@mrjunejune.com>
date Thu, 22 Jan 2026 20:10:30 -0800
parents
children
comparison
equal deleted inserted replaced
177:24fe8ff94056 178:94705b5986b3
1 |// Low-level VM code for ARM64 CPUs.
2 |// Bytecode interpreter, fast functions and helper functions.
3 |// Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4 |
5 |.arch arm64
6 |.section code_op, code_sub
7 |
8 |.actionlist build_actionlist
9 |.globals GLOB_
10 |.globalnames globnames
11 |.externnames extnames
12 |
13 |// Note: The ragged indentation of the instructions is intentional.
14 |// The starting columns indicate data dependencies.
15 |
16 |//-----------------------------------------------------------------------
17 |
18 |// ARM64 registers and the AAPCS64 ABI 1.0 at a glance:
19 |//
20 |// x0-x17 temp, x19-x28 callee-saved, x29 fp, x30 lr
21 |// x18 is reserved on most platforms. Don't use it, save it or restore it.
22 |// x31 doesn't exist. Register number 31 either means xzr/wzr (zero) or sp,
23 |// depending on the instruction.
24 |// v0-v7 temp, v8-v15 callee-saved (only d8-d15 preserved), v16-v31 temp
25 |//
26 |// x0-x7/v0-v7 hold parameters and results.
27 |
28 |// Fixed register assignments for the interpreter.
29 |
30 |// The following must be C callee-save.
31 |.define BASE, x19 // Base of current Lua stack frame.
32 |.define KBASE, x20 // Constants of current Lua function.
33 |.define PC, x21 // Next PC.
34 |.define GLREG, x22 // Global state.
35 |.define LREG, x23 // Register holding lua_State (also in SAVE_L).
36 |.define TISNUM, x24 // Constant LJ_TISNUM << 47.
37 |.define TISNUMhi, x25 // Constant LJ_TISNUM << 15.
38 |.define TISNIL, x26 // Constant -1LL.
39 |.define fp, x29 // Yes, we have to maintain a frame pointer.
40 |
41 |.define ST_INTERP, w26 // Constant -1.
42 |
43 |// The following temporaries are not saved across C calls, except for RA/RC.
44 |.define RA, x27
45 |.define RC, x28
46 |.define RB, x17
47 |.define RAw, w27
48 |.define RCw, w28
49 |.define RBw, w17
50 |.define INS, x16
51 |.define INSw, w16
52 |.define ITYPE, x15
53 |.define TMP0, x8
54 |.define TMP1, x9
55 |.define TMP2, x10
56 |.define TMP3, x11
57 |.define TMP0w, w8
58 |.define TMP1w, w9
59 |.define TMP2w, w10
60 |.define TMP3w, w11
61 |
62 |// Calling conventions. Also used as temporaries.
63 |.define CARG1, x0
64 |.define CARG2, x1
65 |.define CARG3, x2
66 |.define CARG4, x3
67 |.define CARG5, x4
68 |.define CARG1w, w0
69 |.define CARG2w, w1
70 |.define CARG3w, w2
71 |.define CARG4w, w3
72 |.define CARG5w, w4
73 |
74 |.define FARG1, d0
75 |.define FARG2, d1
76 |
77 |.define CRET1, x0
78 |.define CRET1w, w0
79 |
80 |//-----------------------------------------------------------------------
81 |
82 |// ARM64e pointer authentication codes (PAC).
83 |.if PAUTH
84 |.macro sp_auth; pacibsp; .endmacro
85 |.macro br_auth, reg; braaz reg; .endmacro
86 |.macro blr_auth, reg; blraaz reg; .endmacro
87 |.macro ret_auth; retab; .endmacro
88 |.else
89 |.macro sp_auth; .endmacro
90 |.macro br_auth, reg; br reg; .endmacro
91 |.macro blr_auth, reg; blr reg; .endmacro
92 |.macro ret_auth; ret; .endmacro
93 |.endif
94 |
95 |//-----------------------------------------------------------------------
96 |
97 |// Stack layout while in interpreter. Must match with lj_frame.h.
98 |
99 |.define CFRAME_SPACE, 208
100 |//----- 16 byte aligned, <-- sp entering interpreter
101 |.define SAVE_FP_LR_, 192
102 |.define SAVE_GPR_, 112 // 112+10*8: 64 bit GPR saves
103 |.define SAVE_FPR_, 48 // 48+8*8: 64 bit FPR saves
104 |// Unused [sp, #44] // 32 bit values
105 |.define SAVE_NRES, [sp, #40]
106 |.define SAVE_ERRF, [sp, #36]
107 |.define SAVE_MULTRES, [sp, #32]
108 |.define TMPD, [sp, #24] // 64 bit values
109 |.define SAVE_L, [sp, #16]
110 |.define SAVE_PC, [sp, #8]
111 |.define SAVE_CFRAME, [sp, #0]
112 |//----- 16 byte aligned, <-- sp while in interpreter.
113 |
114 |.define TMPDofs, #24
115 |
116 |.macro save_, gpr1, gpr2, fpr1, fpr2
117 | stp d..fpr2, d..fpr1, [sp, # SAVE_FPR_+(14-fpr1)*8]
118 | stp x..gpr2, x..gpr1, [sp, # SAVE_GPR_+(27-gpr1)*8]
119 |.endmacro
120 |.macro rest_, gpr1, gpr2, fpr1, fpr2
121 | ldp d..fpr2, d..fpr1, [sp, # SAVE_FPR_+(14-fpr1)*8]
122 | ldp x..gpr2, x..gpr1, [sp, # SAVE_GPR_+(27-gpr1)*8]
123 |.endmacro
124 |
125 |.macro saveregs
126 | sp_auth
127 | sub sp, sp, # CFRAME_SPACE
128 | stp fp, lr, [sp, # SAVE_FP_LR_]
129 | add fp, sp, # SAVE_FP_LR_
130 | stp x20, x19, [sp, # SAVE_GPR_+(27-19)*8]
131 | save_ 21, 22, 8, 9
132 | save_ 23, 24, 10, 11
133 | save_ 25, 26, 12, 13
134 | save_ 27, 28, 14, 15
135 |.endmacro
136 |.macro restoreregs
137 | ldp x20, x19, [sp, # SAVE_GPR_+(27-19)*8]
138 | rest_ 21, 22, 8, 9
139 | rest_ 23, 24, 10, 11
140 | rest_ 25, 26, 12, 13
141 | rest_ 27, 28, 14, 15
142 | ldp fp, lr, [sp, # SAVE_FP_LR_]
143 | add sp, sp, # CFRAME_SPACE
144 |.endmacro
145 |
146 |// Type definitions. Some of these are only used for documentation.
147 |.type L, lua_State, LREG
148 |.type GL, global_State, GLREG
149 |.type TVALUE, TValue
150 |.type GCOBJ, GCobj
151 |.type STR, GCstr
152 |.type TAB, GCtab
153 |.type LFUNC, GCfuncL
154 |.type CFUNC, GCfuncC
155 |.type PROTO, GCproto
156 |.type UPVAL, GCupval
157 |.type NODE, Node
158 |.type NARGS8, int
159 |.type TRACE, GCtrace
160 |.type SBUF, SBuf
161 |
162 |//-----------------------------------------------------------------------
163 |
164 |// Trap for not-yet-implemented parts.
165 |.macro NYI; brk; .endmacro
166 |
167 |//-----------------------------------------------------------------------
168 |
169 |// Access to frame relative to BASE.
170 |.define FRAME_FUNC, #-16
171 |.define FRAME_PC, #-8
172 |
173 |// Endian-specific defines.
174 |.if ENDIAN_LE
175 |.define LO, 0
176 |.define OFS_RD, 2
177 |.define OFS_RB, 3
178 |.define OFS_RA, 1
179 |.define OFS_OP, 0
180 |.else
181 |.define LO, 4
182 |.define OFS_RD, 0
183 |.define OFS_RB, 0
184 |.define OFS_RA, 2
185 |.define OFS_OP, 3
186 |.endif
187 |
188 |.macro decode_RA, dst, ins; ubfx dst, ins, #8, #8; .endmacro
189 |.macro decode_RB, dst, ins; ubfx dst, ins, #24, #8; .endmacro
190 |.macro decode_RC, dst, ins; ubfx dst, ins, #16, #8; .endmacro
191 |.macro decode_RD, dst, ins; ubfx dst, ins, #16, #16; .endmacro
192 |.macro decode_RC8RD, dst, src; ubfiz dst, src, #3, #8; .endmacro
193 |
194 |// Instruction decode+dispatch.
195 |.macro ins_NEXT
196 | ldr INSw, [PC], #4
197 | add TMP1, GL, INS, uxtb #3
198 | decode_RA RA, INS
199 | ldr TMP0, [TMP1, #GG_G2DISP]
200 | decode_RD RC, INS
201 | br_auth TMP0
202 |.endmacro
203 |
204 |// Instruction footer.
205 |.if 1
206 | // Replicated dispatch. Less unpredictable branches, but higher I-Cache use.
207 | .define ins_next, ins_NEXT
208 | .define ins_next_, ins_NEXT
209 |.else
210 | // Common dispatch. Lower I-Cache use, only one (very) unpredictable branch.
211 | // Affects only certain kinds of benchmarks (and only with -j off).
212 | .macro ins_next
213 | b ->ins_next
214 | .endmacro
215 | .macro ins_next_
216 | ->ins_next:
217 | ins_NEXT
218 | .endmacro
219 |.endif
220 |
221 |// Call decode and dispatch.
222 |.macro ins_callt
223 | // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
224 | ldr PC, LFUNC:CARG3->pc
225 | ldr INSw, [PC], #4
226 | add TMP1, GL, INS, uxtb #3
227 | decode_RA RA, INS
228 | ldr TMP0, [TMP1, #GG_G2DISP]
229 | add RA, BASE, RA, lsl #3
230 | br_auth TMP0
231 |.endmacro
232 |
233 |.macro ins_call
234 | // BASE = new base, CARG3 = LFUNC/CFUNC, RC = nargs*8, PC = caller PC
235 | str PC, [BASE, FRAME_PC]
236 | ins_callt
237 |.endmacro
238 |
239 |//-----------------------------------------------------------------------
240 |
241 |// Macros to check the TValue type and extract the GCobj. Branch on failure.
242 |.macro checktp, reg, tp, target
243 | asr ITYPE, reg, #47
244 | cmn ITYPE, #-tp
245 | and reg, reg, #LJ_GCVMASK
246 | bne target
247 |.endmacro
248 |.macro checktp, dst, reg, tp, target
249 | asr ITYPE, reg, #47
250 | cmn ITYPE, #-tp
251 | and dst, reg, #LJ_GCVMASK
252 | bne target
253 |.endmacro
254 |.macro checkstr, reg, target; checktp reg, LJ_TSTR, target; .endmacro
255 |.macro checktab, reg, target; checktp reg, LJ_TTAB, target; .endmacro
256 |.macro checkfunc, reg, target; checktp reg, LJ_TFUNC, target; .endmacro
257 |.macro checkint, reg, target
258 | cmp TISNUMhi, reg, lsr #32
259 | bne target
260 |.endmacro
261 |.macro checknum, reg, target
262 | cmp TISNUMhi, reg, lsr #32
263 | bls target
264 |.endmacro
265 |.macro checknumber, reg, target
266 | cmp TISNUMhi, reg, lsr #32
267 | blo target
268 |.endmacro
269 |
270 |.macro mov_false, reg; movn reg, #0x8000, lsl #32; .endmacro
271 |.macro mov_true, reg; movn reg, #0x0001, lsl #48; .endmacro
272 |
273 #define GL_J(field) (GG_G2J + (int)offsetof(jit_State, field))
274 |
275 #define PC2PROTO(field) ((int)offsetof(GCproto, field)-(int)sizeof(GCproto))
276 |
277 |.macro hotcheck, delta
278 | lsr CARG1, PC, #1
279 | and CARG1, CARG1, #126
280 | add CARG1, CARG1, #GG_G2DISP+GG_DISP2HOT
281 | ldrh CARG2w, [GL, CARG1]
282 | subs CARG2, CARG2, #delta
283 | strh CARG2w, [GL, CARG1]
284 |.endmacro
285 |
286 |.macro hotloop
287 | hotcheck HOTCOUNT_LOOP
288 | blo ->vm_hotloop
289 |.endmacro
290 |
291 |.macro hotcall
292 | hotcheck HOTCOUNT_CALL
293 | blo ->vm_hotcall
294 |.endmacro
295 |
296 |// Set current VM state.
297 |.macro mv_vmstate, reg, st; movn reg, #LJ_VMST_..st; .endmacro
298 |.macro st_vmstate, reg; str reg, GL->vmstate; .endmacro
299 |
300 |// Move table write barrier back. Overwrites mark and tmp.
301 |.macro barrierback, tab, mark, tmp
302 | ldr tmp, GL->gc.grayagain
303 | and mark, mark, #~LJ_GC_BLACK // black2gray(tab)
304 | str tab, GL->gc.grayagain
305 | strb mark, tab->marked
306 | str tmp, tab->gclist
307 |.endmacro
308 |
309 |//-----------------------------------------------------------------------
310
311 #if !LJ_DUALNUM
312 #error "Only dual-number mode supported for ARM64 target"
313 #endif
314
315 /* Generate subroutines used by opcodes and other parts of the VM. */
316 /* The .code_sub section should be last to help static branch prediction. */
317 static void build_subroutines(BuildCtx *ctx)
318 {
319 |.code_sub
320 |
321 |//-----------------------------------------------------------------------
322 |//-- Return handling ----------------------------------------------------
323 |//-----------------------------------------------------------------------
324 |
325 |->vm_returnp:
326 | // See vm_return. Also: RB = previous base.
327 | tbz PC, #2, ->cont_dispatch // (PC & FRAME_P) == 0?
328 |
329 | // Return from pcall or xpcall fast func.
330 | ldr PC, [RB, FRAME_PC] // Fetch PC of previous frame.
331 | mov_true TMP0
332 | mov BASE, RB
333 | // Prepending may overwrite the pcall frame, so do it at the end.
334 | str TMP0, [RA, #-8]! // Prepend true to results.
335 |
336 |->vm_returnc:
337 | adds RC, RC, #8 // RC = (nresults+1)*8.
338 | mov CRET1, #LUA_YIELD
339 | beq ->vm_unwind_c_eh
340 | str RCw, SAVE_MULTRES
341 | ands CARG1, PC, #FRAME_TYPE
342 | beq ->BC_RET_Z // Handle regular return to Lua.
343 |
344 |->vm_return:
345 | // BASE = base, RA = resultptr, RC/MULTRES = (nresults+1)*8, PC = return
346 | // CARG1 = PC & FRAME_TYPE
347 | and RB, PC, #~FRAME_TYPEP
348 | cmp CARG1, #FRAME_C
349 | sub RB, BASE, RB // RB = previous base.
350 | bne ->vm_returnp
351 |
352 | str RB, L->base
353 | ldrsw CARG2, SAVE_NRES // CARG2 = nresults+1.
354 | mv_vmstate TMP0w, C
355 | sub BASE, BASE, #16
356 | subs TMP2, RC, #8
357 | st_vmstate TMP0w
358 | beq >2
359 |1:
360 | subs TMP2, TMP2, #8
361 | ldr TMP0, [RA], #8
362 | str TMP0, [BASE], #8
363 | bne <1
364 |2:
365 | cmp RC, CARG2, lsl #3 // More/less results wanted?
366 | bne >6
367 |3:
368 | str BASE, L->top // Store new top.
369 |
370 |->vm_leave_cp:
371 | ldr RC, SAVE_CFRAME // Restore previous C frame.
372 | mov CRET1, #0 // Ok return status for vm_pcall.
373 | str RC, L->cframe
374 |
375 |->vm_leave_unw:
376 | restoreregs
377 | ret_auth
378 |
379 |6:
380 | bgt >7 // Less results wanted?
381 | // More results wanted. Check stack size and fill up results with nil.
382 | ldr CARG3, L->maxstack
383 | cmp BASE, CARG3
384 | bhs >8
385 | str TISNIL, [BASE], #8
386 | add RC, RC, #8
387 | b <2
388 |
389 |7: // Less results wanted.
390 | cbz CARG2, <3 // LUA_MULTRET+1 case?
391 | sub CARG1, RC, CARG2, lsl #3
392 | sub BASE, BASE, CARG1 // Shrink top.
393 | b <3
394 |
395 |8: // Corner case: need to grow stack for filling up results.
396 | // This can happen if:
397 | // - A C function grows the stack (a lot).
398 | // - The GC shrinks the stack in between.
399 | // - A return back from a lua_call() with (high) nresults adjustment.
400 | str BASE, L->top // Save current top held in BASE (yes).
401 | mov CARG1, L
402 | bl extern lj_state_growstack // (lua_State *L, int n)
403 | ldr BASE, L->top // Need the (realloced) L->top in BASE.
404 | ldrsw CARG2, SAVE_NRES
405 | b <2
406 |
407 |->vm_unwind_c: // Unwind C stack, return from vm_pcall.
408 | // (void *cframe, int errcode)
409 | mov sp, CARG1
410 | mov CRET1, CARG2
411 |->vm_unwind_c_eh: // Landing pad for external unwinder.
412 | ldr L, SAVE_L
413 | mv_vmstate TMP0w, C
414 | ldr GL, L->glref
415 | st_vmstate TMP0w
416 | b ->vm_leave_unw
417 |
418 |->vm_unwind_ff: // Unwind C stack, return from ff pcall.
419 | // (void *cframe)
420 | and sp, CARG1, #CFRAME_RAWMASK
421 |->vm_unwind_ff_eh: // Landing pad for external unwinder.
422 | ldr L, SAVE_L
423 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
424 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
425 | movn TISNIL, #0
426 | mov RC, #16 // 2 results: false + error message.
427 | ldr BASE, L->base
428 | ldr GL, L->glref // Setup pointer to global state.
429 | mov_false TMP0
430 | sub RA, BASE, #8 // Results start at BASE-8.
431 | ldr PC, [BASE, FRAME_PC] // Fetch PC of previous frame.
432 | str TMP0, [BASE, #-8] // Prepend false to error message.
433 | st_vmstate ST_INTERP
434 | b ->vm_returnc
435 |
436 |//-----------------------------------------------------------------------
437 |//-- Grow stack for calls -----------------------------------------------
438 |//-----------------------------------------------------------------------
439 |
440 |->vm_growstack_c: // Grow stack for C function.
441 | // CARG1 = L
442 | mov CARG2, #LUA_MINSTACK
443 | b >2
444 |
445 |->vm_growstack_l: // Grow stack for Lua function.
446 | // BASE = new base, RA = BASE+framesize*8, RC = nargs*8, PC = first PC
447 | add RC, BASE, RC
448 | sub RA, RA, BASE
449 | mov CARG1, L
450 | stp BASE, RC, L->base
451 | add PC, PC, #4 // Must point after first instruction.
452 | lsr CARG2, RA, #3
453 |2:
454 | // L->base = new base, L->top = top
455 | str PC, SAVE_PC
456 | bl extern lj_state_growstack // (lua_State *L, int n)
457 | ldp BASE, RC, L->base
458 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
459 | sub NARGS8:RC, RC, BASE
460 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
461 | // BASE = new base, RB = LFUNC/CFUNC, RC = nargs*8, FRAME_PC(BASE) = PC
462 | ins_callt // Just retry the call.
463 |
464 |//-----------------------------------------------------------------------
465 |//-- Entry points into the assembler VM ---------------------------------
466 |//-----------------------------------------------------------------------
467 |
468 |->vm_resume: // Setup C frame and resume thread.
469 | // (lua_State *L, TValue *base, int nres1 = 0, ptrdiff_t ef = 0)
470 | saveregs
471 | mov L, CARG1
472 | ldr GL, L->glref // Setup pointer to global state.
473 | mov BASE, CARG2
474 | str L, SAVE_L
475 | mov PC, #FRAME_CP
476 | str wzr, SAVE_NRES
477 | add TMP0, sp, #CFRAME_RESUME
478 | ldrb TMP1w, L->status
479 | str wzr, SAVE_ERRF
480 | str L, SAVE_PC // Any value outside of bytecode is ok.
481 | str xzr, SAVE_CFRAME
482 | str TMP0, L->cframe
483 | cbz TMP1w, >3
484 |
485 | // Resume after yield (like a return).
486 | str L, GL->cur_L
487 | mov RA, BASE
488 | ldp BASE, CARG1, L->base
489 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
490 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
491 | ldr PC, [BASE, FRAME_PC]
492 | strb wzr, L->status
493 | movn TISNIL, #0
494 | sub RC, CARG1, BASE
495 | ands CARG1, PC, #FRAME_TYPE
496 | add RC, RC, #8
497 | st_vmstate ST_INTERP
498 | str RCw, SAVE_MULTRES
499 | beq ->BC_RET_Z
500 | b ->vm_return
501 |
502 |->vm_pcall: // Setup protected C frame and enter VM.
503 | // (lua_State *L, TValue *base, int nres1, ptrdiff_t ef)
504 | saveregs
505 | mov PC, #FRAME_CP
506 | str CARG4w, SAVE_ERRF
507 | b >1
508 |
509 |->vm_call: // Setup C frame and enter VM.
510 | // (lua_State *L, TValue *base, int nres1)
511 | saveregs
512 | mov PC, #FRAME_C
513 |
514 |1: // Entry point for vm_pcall above (PC = ftype).
515 | ldr RC, L:CARG1->cframe
516 | str CARG3w, SAVE_NRES
517 | mov L, CARG1
518 | str CARG1, SAVE_L
519 | ldr GL, L->glref // Setup pointer to global state.
520 | mov BASE, CARG2
521 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
522 | add TMP0, sp, #0
523 | str RC, SAVE_CFRAME
524 | str TMP0, L->cframe // Add our C frame to cframe chain.
525 |
526 |3: // Entry point for vm_cpcall/vm_resume (BASE = base, PC = ftype).
527 | str L, GL->cur_L
528 | ldp RB, CARG1, L->base // RB = old base (for vmeta_call).
529 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
530 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
531 | add PC, PC, BASE
532 | movn TISNIL, #0
533 | sub PC, PC, RB // PC = frame delta + frame type
534 | sub NARGS8:RC, CARG1, BASE
535 | st_vmstate ST_INTERP
536 |
537 |->vm_call_dispatch:
538 | // RB = old base, BASE = new base, RC = nargs*8, PC = caller PC
539 | ldr CARG3, [BASE, FRAME_FUNC]
540 | checkfunc CARG3, ->vmeta_call
541 |
542 |->vm_call_dispatch_f:
543 | ins_call
544 | // BASE = new base, CARG3 = func, RC = nargs*8, PC = caller PC
545 |
546 |->vm_cpcall: // Setup protected C frame, call C.
547 | // (lua_State *L, lua_CFunction func, void *ud, lua_CPFunction cp)
548 | saveregs
549 | mov L, CARG1
550 | ldr RA, L:CARG1->stack
551 | str CARG1, SAVE_L
552 | ldr GL, L->glref // Setup pointer to global state.
553 | ldr RB, L->top
554 | str CARG1, SAVE_PC // Any value outside of bytecode is ok.
555 | ldr RC, L->cframe
556 | sub RA, RA, RB // Compute -savestack(L, L->top).
557 | str RAw, SAVE_NRES // Neg. delta means cframe w/o frame.
558 | str wzr, SAVE_ERRF // No error function.
559 | add TMP0, sp, #0
560 | str RC, SAVE_CFRAME
561 | str TMP0, L->cframe // Add our C frame to cframe chain.
562 | str L, GL->cur_L
563 | blr_auth CARG4 // (lua_State *L, lua_CFunction func, void *ud)
564 | mov BASE, CRET1
565 | mov PC, #FRAME_CP
566 | cbnz BASE, <3 // Else continue with the call.
567 | b ->vm_leave_cp // No base? Just remove C frame.
568 |
569 |//-----------------------------------------------------------------------
570 |//-- Metamethod handling ------------------------------------------------
571 |//-----------------------------------------------------------------------
572 |
573 |//-- Continuation dispatch ----------------------------------------------
574 |
575 |->cont_dispatch:
576 | // BASE = meta base, RA = resultptr, RC = (nresults+1)*8
577 | ldr LFUNC:CARG3, [RB, FRAME_FUNC]
578 | ldr CARG1, [BASE, #-32] // Get continuation.
579 | mov CARG4, BASE
580 | mov BASE, RB // Restore caller BASE.
581 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
582 |.if FFI
583 | cmp CARG1, #1
584 |.endif
585 | ldr PC, [CARG4, #-24] // Restore PC from [cont|PC].
586 | add TMP0, RA, RC
587 | str TISNIL, [TMP0, #-8] // Ensure one valid arg.
588 |.if FFI
589 | bls >1
590 |.endif
591 | ldr CARG3, LFUNC:CARG3->pc
592 | ldr KBASE, [CARG3, #PC2PROTO(k)]
593 | // BASE = base, RA = resultptr, CARG4 = meta base
594 | br_auth CARG1
595 |
596 |.if FFI
597 |1:
598 | beq ->cont_ffi_callback // cont = 1: return from FFI callback.
599 | // cont = 0: tailcall from C function.
600 | sub CARG4, CARG4, #32
601 | sub RC, CARG4, BASE
602 | b ->vm_call_tail
603 |.endif
604 |
605 |->cont_cat: // RA = resultptr, CARG4 = meta base
606 | ldr INSw, [PC, #-4]
607 | sub CARG2, CARG4, #32
608 | ldr TMP0, [RA]
609 | str BASE, L->base
610 | decode_RB RB, INS
611 | decode_RA RA, INS
612 | add TMP1, BASE, RB, lsl #3
613 | subs TMP1, CARG2, TMP1
614 | beq >1
615 | str TMP0, [CARG2]
616 | lsr CARG3, TMP1, #3
617 | b ->BC_CAT_Z
618 |
619 |1:
620 | str TMP0, [BASE, RA, lsl #3]
621 | b ->cont_nop
622 |
623 |//-- Table indexing metamethods -----------------------------------------
624 |
625 |->vmeta_tgets1:
626 | movn CARG4, #~LJ_TSTR
627 | add CARG2, BASE, RB, lsl #3
628 | add CARG4, STR:RC, CARG4, lsl #47
629 | b >2
630 |
631 |->vmeta_tgets:
632 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
633 | str CARG2, GL->tmptv
634 | add CARG2, GL, #offsetof(global_State, tmptv)
635 |2:
636 | add CARG3, sp, TMPDofs
637 | str CARG4, TMPD
638 | b >1
639 |
640 |->vmeta_tgetb: // RB = table, RC = index
641 | add RC, RC, TISNUM
642 | add CARG2, BASE, RB, lsl #3
643 | add CARG3, sp, TMPDofs
644 | str RC, TMPD
645 | b >1
646 |
647 |->vmeta_tgetv: // RB = table, RC = key
648 | add CARG2, BASE, RB, lsl #3
649 | add CARG3, BASE, RC, lsl #3
650 |1:
651 | str BASE, L->base
652 | mov CARG1, L
653 | str PC, SAVE_PC
654 | bl extern lj_meta_tget // (lua_State *L, TValue *o, TValue *k)
655 | // Returns TValue * (finished) or NULL (metamethod).
656 | cbz CRET1, >3
657 | ldr TMP0, [CRET1]
658 | str TMP0, [BASE, RA, lsl #3]
659 | ins_next
660 |
661 |3: // Call __index metamethod.
662 | // BASE = base, L->top = new base, stack = cont/func/t/k
663 | sub TMP1, BASE, #FRAME_CONT
664 | ldr BASE, L->top
665 | mov NARGS8:RC, #16 // 2 args for func(t, k).
666 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
667 | str PC, [BASE, #-24] // [cont|PC]
668 | sub PC, BASE, TMP1
669 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
670 | b ->vm_call_dispatch_f
671 |
672 |->vmeta_tgetr:
673 | sxtw CARG2, TMP1w
674 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
675 | // Returns cTValue * or NULL.
676 | mov TMP0, TISNIL
677 | cbz CRET1, ->BC_TGETR_Z
678 | ldr TMP0, [CRET1]
679 | b ->BC_TGETR_Z
680 |
681 |//-----------------------------------------------------------------------
682 |
683 |->vmeta_tsets1:
684 | movn CARG4, #~LJ_TSTR
685 | add CARG2, BASE, RB, lsl #3
686 | add CARG4, STR:RC, CARG4, lsl #47
687 | b >2
688 |
689 |->vmeta_tsets:
690 | movk CARG2, #(LJ_TTAB>>1)&0xffff, lsl #48
691 | str CARG2, GL->tmptv
692 | add CARG2, GL, #offsetof(global_State, tmptv)
693 |2:
694 | add CARG3, sp, TMPDofs
695 | str CARG4, TMPD
696 | b >1
697 |
698 |->vmeta_tsetb: // RB = table, RC = index
699 | add RC, RC, TISNUM
700 | add CARG2, BASE, RB, lsl #3
701 | add CARG3, sp, TMPDofs
702 | str RC, TMPD
703 | b >1
704 |
705 |->vmeta_tsetv:
706 | add CARG2, BASE, RB, lsl #3
707 | add CARG3, BASE, RC, lsl #3
708 |1:
709 | str BASE, L->base
710 | mov CARG1, L
711 | str PC, SAVE_PC
712 | bl extern lj_meta_tset // (lua_State *L, TValue *o, TValue *k)
713 | // Returns TValue * (finished) or NULL (metamethod).
714 | ldr TMP0, [BASE, RA, lsl #3]
715 | cbz CRET1, >3
716 | // NOBARRIER: lj_meta_tset ensures the table is not black.
717 | str TMP0, [CRET1]
718 | ins_next
719 |
720 |3: // Call __newindex metamethod.
721 | // BASE = base, L->top = new base, stack = cont/func/t/k/(v)
722 | sub TMP1, BASE, #FRAME_CONT
723 | ldr BASE, L->top
724 | mov NARGS8:RC, #24 // 3 args for func(t, k, v).
725 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
726 | str TMP0, [BASE, #16] // Copy value to third argument.
727 | str PC, [BASE, #-24] // [cont|PC]
728 | sub PC, BASE, TMP1
729 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
730 | b ->vm_call_dispatch_f
731 |
732 |->vmeta_tsetr:
733 | sxtw CARG3, TMP1w
734 | str BASE, L->base
735 | mov CARG1, L
736 | str PC, SAVE_PC
737 | bl extern lj_tab_setinth // (lua_State *L, GCtab *t, int32_t key)
738 | // Returns TValue *.
739 | b ->BC_TSETR_Z
740 |
741 |//-- Comparison metamethods ---------------------------------------------
742 |
743 |->vmeta_comp:
744 | add CARG2, BASE, RA, lsl #3
745 | sub PC, PC, #4
746 | add CARG3, BASE, RC, lsl #3
747 | str BASE, L->base
748 | mov CARG1, L
749 | str PC, SAVE_PC
750 | uxtb CARG4w, INSw
751 | bl extern lj_meta_comp // (lua_State *L, TValue *o1, *o2, int op)
752 | // Returns 0/1 or TValue * (metamethod).
753 |3:
754 | cmp CRET1, #1
755 | bhi ->vmeta_binop
756 |4:
757 | ldrh RBw, [PC, # OFS_RD]
758 | add PC, PC, #4
759 | add RB, PC, RB, lsl #2
760 | sub RB, RB, #0x20000
761 | csel PC, PC, RB, lo
762 |->cont_nop:
763 | ins_next
764 |
765 |->cont_ra: // RA = resultptr
766 | ldr INSw, [PC, #-4]
767 | ldr TMP0, [RA]
768 | decode_RA TMP1, INS
769 | str TMP0, [BASE, TMP1, lsl #3]
770 | b ->cont_nop
771 |
772 |->cont_condt: // RA = resultptr
773 | ldr TMP0, [RA]
774 | mov_true TMP1
775 | cmp TMP1, TMP0 // Branch if result is true.
776 | b <4
777 |
778 |->cont_condf: // RA = resultptr
779 | ldr TMP0, [RA]
780 | mov_false TMP1
781 | cmp TMP0, TMP1 // Branch if result is false.
782 | b <4
783 |
784 |->vmeta_equal:
785 | // CARG2, CARG3, CARG4 are already set by BC_ISEQV/BC_ISNEV.
786 | and TAB:CARG3, CARG3, #LJ_GCVMASK
787 | sub PC, PC, #4
788 | str BASE, L->base
789 | mov CARG1, L
790 | str PC, SAVE_PC
791 | bl extern lj_meta_equal // (lua_State *L, GCobj *o1, *o2, int ne)
792 | // Returns 0/1 or TValue * (metamethod).
793 | b <3
794 |
795 |->vmeta_equal_cd:
796 |.if FFI
797 | sub PC, PC, #4
798 | str BASE, L->base
799 | mov CARG1, L
800 | mov CARG2, INS
801 | str PC, SAVE_PC
802 | bl extern lj_meta_equal_cd // (lua_State *L, BCIns op)
803 | // Returns 0/1 or TValue * (metamethod).
804 | b <3
805 |.endif
806 |
807 |->vmeta_istype:
808 | sub PC, PC, #4
809 | str BASE, L->base
810 | mov CARG1, L
811 | mov CARG2, RA
812 | mov CARG3, RC
813 | str PC, SAVE_PC
814 | bl extern lj_meta_istype // (lua_State *L, BCReg ra, BCReg tp)
815 | b ->cont_nop
816 |
817 |//-- Arithmetic metamethods ---------------------------------------------
818 |
819 |->vmeta_arith_vn:
820 | add CARG3, BASE, RB, lsl #3
821 | add CARG4, KBASE, RC, lsl #3
822 | b >1
823 |
824 |->vmeta_arith_nv:
825 | add CARG4, BASE, RB, lsl #3
826 | add CARG3, KBASE, RC, lsl #3
827 | b >1
828 |
829 |->vmeta_unm:
830 | add CARG3, BASE, RC, lsl #3
831 | mov CARG4, CARG3
832 | b >1
833 |
834 |->vmeta_arith_vv:
835 | add CARG3, BASE, RB, lsl #3
836 | add CARG4, BASE, RC, lsl #3
837 |1:
838 | uxtb CARG5w, INSw
839 | add CARG2, BASE, RA, lsl #3
840 | str BASE, L->base
841 | mov CARG1, L
842 | str PC, SAVE_PC
843 | bl extern lj_meta_arith // (lua_State *L, TValue *ra,*rb,*rc, BCReg op)
844 | // Returns NULL (finished) or TValue * (metamethod).
845 | cbz CRET1, ->cont_nop
846 |
847 | // Call metamethod for binary op.
848 |->vmeta_binop:
849 | // BASE = old base, CRET1 = new base, stack = cont/func/o1/o2
850 | sub TMP1, CRET1, BASE
851 | str PC, [CRET1, #-24] // [cont|PC]
852 | add PC, TMP1, #FRAME_CONT
853 | mov BASE, CRET1
854 | mov NARGS8:RC, #16 // 2 args for func(o1, o2).
855 | b ->vm_call_dispatch
856 |
857 |->vmeta_len:
858 | add CARG2, BASE, RC, lsl #3
859 #if LJ_52
860 | mov TAB:RC, TAB:CARG1 // Save table (ignored for other types).
861 #endif
862 | str BASE, L->base
863 | mov CARG1, L
864 | str PC, SAVE_PC
865 | bl extern lj_meta_len // (lua_State *L, TValue *o)
866 | // Returns NULL (retry) or TValue * (metamethod base).
867 #if LJ_52
868 | cbnz CRET1, ->vmeta_binop // Binop call for compatibility.
869 | mov TAB:CARG1, TAB:RC
870 | b ->BC_LEN_Z
871 #else
872 | b ->vmeta_binop // Binop call for compatibility.
873 #endif
874 |
875 |//-- Call metamethod ----------------------------------------------------
876 |
877 |->vmeta_call: // Resolve and call __call metamethod.
878 | // RB = old base, BASE = new base, RC = nargs*8
879 | mov CARG1, L
880 | str RB, L->base // This is the callers base!
881 | sub CARG2, BASE, #16
882 | str PC, SAVE_PC
883 | add CARG3, BASE, NARGS8:RC
884 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
885 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Guaranteed to be a function here.
886 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
887 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
888 | ins_call
889 |
890 |->vmeta_callt: // Resolve __call for BC_CALLT.
891 | // BASE = old base, RA = new base, RC = nargs*8
892 | mov CARG1, L
893 | str BASE, L->base
894 | sub CARG2, RA, #16
895 | str PC, SAVE_PC
896 | add CARG3, RA, NARGS8:RC
897 | bl extern lj_meta_call // (lua_State *L, TValue *func, TValue *top)
898 | ldr TMP1, [RA, FRAME_FUNC] // Guaranteed to be a function here.
899 | ldr PC, [BASE, FRAME_PC]
900 | add NARGS8:RC, NARGS8:RC, #8 // Got one more argument now.
901 | and LFUNC:CARG3, TMP1, #LJ_GCVMASK
902 | b ->BC_CALLT2_Z
903 |
904 |//-- Argument coercion for 'for' statement ------------------------------
905 |
906 |->vmeta_for:
907 | mov CARG1, L
908 | str BASE, L->base
909 | mov CARG2, RA
910 | str PC, SAVE_PC
911 | bl extern lj_meta_for // (lua_State *L, TValue *base)
912 | ldr INSw, [PC, #-4]
913 |.if JIT
914 | uxtb TMP0w, INSw
915 |.endif
916 | decode_RA RA, INS
917 | decode_RD RC, INS
918 |.if JIT
919 | cmp TMP0, #BC_JFORI
920 | beq =>BC_JFORI
921 |.endif
922 | b =>BC_FORI
923 |
924 |//-----------------------------------------------------------------------
925 |//-- Fast functions -----------------------------------------------------
926 |//-----------------------------------------------------------------------
927 |
928 |.macro .ffunc, name
929 |->ff_ .. name:
930 |.endmacro
931 |
932 |.macro .ffunc_1, name
933 |->ff_ .. name:
934 | ldr CARG1, [BASE]
935 | cmp NARGS8:RC, #8
936 | blo ->fff_fallback
937 |.endmacro
938 |
939 |.macro .ffunc_2, name
940 |->ff_ .. name:
941 | ldp CARG1, CARG2, [BASE]
942 | cmp NARGS8:RC, #16
943 | blo ->fff_fallback
944 |.endmacro
945 |
946 |.macro .ffunc_n, name
947 | .ffunc name
948 | ldr CARG1, [BASE]
949 | cmp NARGS8:RC, #8
950 | ldr FARG1, [BASE]
951 | blo ->fff_fallback
952 | checknum CARG1, ->fff_fallback
953 |.endmacro
954 |
955 |.macro .ffunc_nn, name
956 | .ffunc name
957 | ldp CARG1, CARG2, [BASE]
958 | cmp NARGS8:RC, #16
959 | ldp FARG1, FARG2, [BASE]
960 | blo ->fff_fallback
961 | checknum CARG1, ->fff_fallback
962 | checknum CARG2, ->fff_fallback
963 |.endmacro
964 |
965 |// Inlined GC threshold check. Caveat: uses CARG1 and CARG2.
966 |.macro ffgccheck
967 | ldp CARG1, CARG2, GL->gc.total // Assumes threshold follows total.
968 | cmp CARG1, CARG2
969 | blt >1
970 | bl ->fff_gcstep
971 |1:
972 |.endmacro
973 |
974 |//-- Base library: checks -----------------------------------------------
975 |
976 |.ffunc_1 assert
977 | ldr PC, [BASE, FRAME_PC]
978 | mov_false TMP1
979 | cmp CARG1, TMP1
980 | bhs ->fff_fallback
981 | str CARG1, [BASE, #-16]
982 | sub RB, BASE, #8
983 | subs RA, NARGS8:RC, #8
984 | add RC, NARGS8:RC, #8 // Compute (nresults+1)*8.
985 | cbz RA, ->fff_res // Done if exactly 1 argument.
986 |1:
987 | ldr CARG1, [RB, #16]
988 | sub RA, RA, #8
989 | str CARG1, [RB], #8
990 | cbnz RA, <1
991 | b ->fff_res
992 |
993 |.ffunc_1 type
994 | mov TMP0, #~LJ_TISNUM
995 | asr ITYPE, CARG1, #47
996 | cmn ITYPE, #~LJ_TISNUM
997 | csinv TMP1, TMP0, ITYPE, lo
998 | add TMP1, TMP1, #offsetof(GCfuncC, upvalue)/8
999 | ldr CARG1, [CFUNC:CARG3, TMP1, lsl #3]
1000 | b ->fff_restv
1001 |
1002 |//-- Base library: getters and setters ---------------------------------
1003 |
1004 |.ffunc_1 getmetatable
1005 | asr ITYPE, CARG1, #47
1006 | cmn ITYPE, #-LJ_TTAB
1007 | ccmn ITYPE, #-LJ_TUDATA, #4, ne
1008 | and TAB:CARG1, CARG1, #LJ_GCVMASK
1009 | bne >6
1010 |1: // Field metatable must be at same offset for GCtab and GCudata!
1011 | ldr TAB:RB, TAB:CARG1->metatable
1012 |2:
1013 | mov CARG1, TISNIL
1014 | ldr STR:RC, GL->gcroot[GCROOT_MMNAME+MM_metatable]
1015 | cbz TAB:RB, ->fff_restv
1016 | ldr TMP1w, TAB:RB->hmask
1017 | ldr TMP2w, STR:RC->sid
1018 | ldr NODE:CARG3, TAB:RB->node
1019 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
1020 | add TMP1, TMP1, TMP1, lsl #1
1021 | movn CARG4, #~LJ_TSTR
1022 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
1023 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
1024 |3: // Rearranged logic, because we expect _not_ to find the key.
1025 | ldp CARG1, TMP0, NODE:CARG3->val
1026 | ldr NODE:CARG3, NODE:CARG3->next
1027 | cmp TMP0, CARG4
1028 | beq >5
1029 | cbnz NODE:CARG3, <3
1030 |4:
1031 | mov CARG1, RB // Use metatable as default result.
1032 | movk CARG1, #(LJ_TTAB>>1)&0xffff, lsl #48
1033 | b ->fff_restv
1034 |5:
1035 | cmp TMP0, TISNIL
1036 | bne ->fff_restv
1037 | b <4
1038 |
1039 |6:
1040 | movn TMP0, #~LJ_TISNUM
1041 | cmp ITYPE, TMP0
1042 | csel ITYPE, ITYPE, TMP0, hs
1043 | sub TMP1, GL, ITYPE, lsl #3
1044 | ldr TAB:RB, [TMP1, #offsetof(global_State, gcroot[GCROOT_BASEMT])-8]
1045 | b <2
1046 |
1047 |.ffunc_2 setmetatable
1048 | // Fast path: no mt for table yet and not clearing the mt.
1049 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1050 | ldr TAB:TMP0, TAB:TMP1->metatable
1051 | asr ITYPE, CARG2, #47
1052 | ldrb TMP2w, TAB:TMP1->marked
1053 | cmn ITYPE, #-LJ_TTAB
1054 | and TAB:CARG2, CARG2, #LJ_GCVMASK
1055 | ccmp TAB:TMP0, #0, #0, eq
1056 | bne ->fff_fallback
1057 | str TAB:CARG2, TAB:TMP1->metatable
1058 | tbz TMP2w, #2, ->fff_restv // isblack(table)
1059 | barrierback TAB:TMP1, TMP2w, TMP0
1060 | b ->fff_restv
1061 |
1062 |.ffunc rawget
1063 | ldr CARG2, [BASE]
1064 | cmp NARGS8:RC, #16
1065 | blo ->fff_fallback
1066 | checktab CARG2, ->fff_fallback
1067 | mov CARG1, L
1068 | add CARG3, BASE, #8
1069 | bl extern lj_tab_get // (lua_State *L, GCtab *t, cTValue *key)
1070 | // Returns cTValue *.
1071 | ldr CARG1, [CRET1]
1072 | b ->fff_restv
1073 |
1074 |//-- Base library: conversions ------------------------------------------
1075 |
1076 |.ffunc tonumber
1077 | // Only handles the number case inline (without a base argument).
1078 | ldr CARG1, [BASE]
1079 | cmp NARGS8:RC, #8
1080 | bne ->fff_fallback
1081 | checknumber CARG1, ->fff_fallback
1082 | b ->fff_restv
1083 |
1084 |.ffunc_1 tostring
1085 | // Only handles the string or number case inline.
1086 | asr ITYPE, CARG1, #47
1087 | cmn ITYPE, #-LJ_TSTR
1088 | // A __tostring method in the string base metatable is ignored.
1089 | beq ->fff_restv
1090 | // Handle numbers inline, unless a number base metatable is present.
1091 | ldr TMP1, GL->gcroot[GCROOT_BASEMT_NUM]
1092 | str BASE, L->base
1093 | cmn ITYPE, #-LJ_TISNUM
1094 | ccmp TMP1, #0, #0, ls
1095 | str PC, SAVE_PC // Redundant (but a defined value).
1096 | bne ->fff_fallback
1097 | ffgccheck
1098 | mov CARG1, L
1099 | mov CARG2, BASE
1100 | bl extern lj_strfmt_number // (lua_State *L, cTValue *o)
1101 | // Returns GCstr *.
1102 | movn TMP1, #~LJ_TSTR
1103 | ldr BASE, L->base
1104 | add CARG1, CARG1, TMP1, lsl #47
1105 | b ->fff_restv
1106 |
1107 |//-- Base library: iterators -------------------------------------------
1108 |
1109 |.ffunc_1 next
1110 | checktp CARG1, LJ_TTAB, ->fff_fallback
1111 | str TISNIL, [BASE, NARGS8:RC] // Set missing 2nd arg to nil.
1112 | ldr PC, [BASE, FRAME_PC]
1113 | add CARG2, BASE, #8
1114 | sub CARG3, BASE, #16
1115 | bl extern lj_tab_next // (GCtab *t, cTValue *key, TValue *o)
1116 | // Returns 1=found, 0=end, -1=error.
1117 | mov RC, #(2+1)*8
1118 | tbnz CRET1w, #31, ->fff_fallback // Invalid key.
1119 | cbnz CRET1, ->fff_res // Found key/value.
1120 | // End of traversal: return nil.
1121 | str TISNIL, [BASE, #-16]
1122 | b ->fff_res1
1123 |
1124 |.ffunc_1 pairs
1125 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1126 #if LJ_52
1127 | ldr TAB:CARG2, TAB:TMP1->metatable
1128 #endif
1129 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1130 | ldr PC, [BASE, FRAME_PC]
1131 #if LJ_52
1132 | cbnz TAB:CARG2, ->fff_fallback
1133 #endif
1134 | mov RC, #(3+1)*8
1135 | stp CARG1, TISNIL, [BASE, #-8]
1136 | str CFUNC:CARG4, [BASE, #-16]
1137 | b ->fff_res
1138 |
1139 |.ffunc_2 ipairs_aux
1140 | checktab CARG1, ->fff_fallback
1141 | checkint CARG2, ->fff_fallback
1142 | ldr TMP1w, TAB:CARG1->asize
1143 | ldr CARG3, TAB:CARG1->array
1144 | ldr TMP0w, TAB:CARG1->hmask
1145 | add CARG2w, CARG2w, #1
1146 | cmp CARG2w, TMP1w
1147 | ldr PC, [BASE, FRAME_PC]
1148 | add TMP2, CARG2, TISNUM
1149 | mov RC, #(0+1)*8
1150 | str TMP2, [BASE, #-16]
1151 | bhs >2 // Not in array part?
1152 | ldr TMP0, [CARG3, CARG2, lsl #3]
1153 |1:
1154 | mov TMP1, #(2+1)*8
1155 | cmp TMP0, TISNIL
1156 | str TMP0, [BASE, #-8]
1157 | csel RC, RC, TMP1, eq
1158 | b ->fff_res
1159 |2: // Check for empty hash part first. Otherwise call C function.
1160 | cbz TMP0w, ->fff_res
1161 | bl extern lj_tab_getinth // (GCtab *t, int32_t key)
1162 | // Returns cTValue * or NULL.
1163 | cbz CRET1, ->fff_res
1164 | ldr TMP0, [CRET1]
1165 | b <1
1166 |
1167 |.ffunc_1 ipairs
1168 | checktp TMP1, CARG1, LJ_TTAB, ->fff_fallback
1169 #if LJ_52
1170 | ldr TAB:CARG2, TAB:TMP1->metatable
1171 #endif
1172 | ldr CFUNC:CARG4, CFUNC:CARG3->upvalue[0]
1173 | ldr PC, [BASE, FRAME_PC]
1174 #if LJ_52
1175 | cbnz TAB:CARG2, ->fff_fallback
1176 #endif
1177 | mov RC, #(3+1)*8
1178 | stp CARG1, TISNUM, [BASE, #-8]
1179 | str CFUNC:CARG4, [BASE, #-16]
1180 | b ->fff_res
1181 |
1182 |//-- Base library: catch errors ----------------------------------------
1183 |
1184 |.ffunc pcall
1185 | cmp NARGS8:RC, #8
1186 | ldrb TMP0w, GL->hookmask
1187 | blo ->fff_fallback
1188 | sub NARGS8:RC, NARGS8:RC, #8
1189 | mov RB, BASE
1190 | add BASE, BASE, #16
1191 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1192 | add PC, TMP0, #16+FRAME_PCALL
1193 | beq ->vm_call_dispatch
1194 |1:
1195 | add TMP2, BASE, NARGS8:RC
1196 |2:
1197 | ldr TMP0, [TMP2, #-16]
1198 | str TMP0, [TMP2, #-8]!
1199 | cmp TMP2, BASE
1200 | bne <2
1201 | b ->vm_call_dispatch
1202 |
1203 |.ffunc xpcall
1204 | ldp CARG1, CARG2, [BASE]
1205 | ldrb TMP0w, GL->hookmask
1206 | subs NARGS8:TMP1, NARGS8:RC, #16
1207 | blo ->fff_fallback
1208 | mov RB, BASE
1209 | asr ITYPE, CARG2, #47
1210 | ubfx TMP0w, TMP0w, #HOOK_ACTIVE_SHIFT, #1
1211 | cmn ITYPE, #-LJ_TFUNC
1212 | add PC, TMP0, #24+FRAME_PCALL
1213 | bne ->fff_fallback // Traceback must be a function.
1214 | mov NARGS8:RC, NARGS8:TMP1
1215 | add BASE, BASE, #24
1216 | stp CARG2, CARG1, [RB] // Swap function and traceback.
1217 | cbz NARGS8:RC, ->vm_call_dispatch
1218 | b <1
1219 |
1220 |//-- Coroutine library --------------------------------------------------
1221 |
1222 |.macro coroutine_resume_wrap, resume
1223 |.if resume
1224 |.ffunc_1 coroutine_resume
1225 | checktp CARG1, LJ_TTHREAD, ->fff_fallback
1226 |.else
1227 |.ffunc coroutine_wrap_aux
1228 | ldr L:CARG1, CFUNC:CARG3->upvalue[0].gcr
1229 | and L:CARG1, CARG1, #LJ_GCVMASK
1230 |.endif
1231 | ldr PC, [BASE, FRAME_PC]
1232 | str BASE, L->base
1233 | ldp RB, CARG2, L:CARG1->base
1234 | ldrb TMP1w, L:CARG1->status
1235 | add TMP0, CARG2, TMP1
1236 | str PC, SAVE_PC
1237 | cmp TMP0, RB
1238 | beq ->fff_fallback
1239 | cmp TMP1, #LUA_YIELD
1240 | add TMP0, CARG2, #8
1241 | csel CARG2, CARG2, TMP0, hs
1242 | ldr CARG4, L:CARG1->maxstack
1243 | add CARG3, CARG2, NARGS8:RC
1244 | ldr RB, L:CARG1->cframe
1245 | ccmp CARG3, CARG4, #2, ls
1246 | ccmp RB, #0, #2, ls
1247 | bhi ->fff_fallback
1248 |.if resume
1249 | sub CARG3, CARG3, #8 // Keep resumed thread in stack for GC.
1250 | add BASE, BASE, #8
1251 | sub NARGS8:RC, NARGS8:RC, #8
1252 |.endif
1253 | str CARG3, L:CARG1->top
1254 | str BASE, L->top
1255 | cbz NARGS8:RC, >3
1256 |2: // Move args to coroutine.
1257 | ldr TMP0, [BASE, RB]
1258 | cmp RB, NARGS8:RC
1259 | str TMP0, [CARG2, RB]
1260 | add RB, RB, #8
1261 | bne <2
1262 |3:
1263 | mov CARG3, #0
1264 | mov L:RA, L:CARG1
1265 | mov CARG4, #0
1266 | bl ->vm_resume // (lua_State *L, TValue *base, 0, 0)
1267 | // Returns thread status.
1268 |4:
1269 | ldp CARG3, CARG4, L:RA->base
1270 | cmp CRET1, #LUA_YIELD
1271 | ldr BASE, L->base
1272 | str L, GL->cur_L
1273 | st_vmstate ST_INTERP
1274 | bhi >8
1275 | sub RC, CARG4, CARG3
1276 | ldr CARG1, L->maxstack
1277 | add CARG2, BASE, RC
1278 | cbz RC, >6 // No results?
1279 | cmp CARG2, CARG1
1280 | mov RB, #0
1281 | bhi >9 // Need to grow stack?
1282 |
1283 | sub CARG4, RC, #8
1284 | str CARG3, L:RA->top // Clear coroutine stack.
1285 |5: // Move results from coroutine.
1286 | ldr TMP0, [CARG3, RB]
1287 | cmp RB, CARG4
1288 | str TMP0, [BASE, RB]
1289 | add RB, RB, #8
1290 | bne <5
1291 |6:
1292 |.if resume
1293 | mov_true TMP1
1294 | add RC, RC, #16
1295 |7:
1296 | str TMP1, [BASE, #-8] // Prepend true/false to results.
1297 | sub RA, BASE, #8
1298 |.else
1299 | mov RA, BASE
1300 | add RC, RC, #8
1301 |.endif
1302 | ands CARG1, PC, #FRAME_TYPE
1303 | str PC, SAVE_PC
1304 | str RCw, SAVE_MULTRES
1305 | beq ->BC_RET_Z
1306 | b ->vm_return
1307 |
1308 |8: // Coroutine returned with error (at co->top-1).
1309 |.if resume
1310 | ldr TMP0, [CARG4, #-8]!
1311 | mov_false TMP1
1312 | mov RC, #(2+1)*8
1313 | str CARG4, L:RA->top // Remove error from coroutine stack.
1314 | str TMP0, [BASE] // Copy error message.
1315 | b <7
1316 |.else
1317 | mov CARG1, L
1318 | mov CARG2, L:RA
1319 | bl extern lj_ffh_coroutine_wrap_err // (lua_State *L, lua_State *co)
1320 | // Never returns.
1321 |.endif
1322 |
1323 |9: // Handle stack expansion on return from yield.
1324 | mov CARG1, L
1325 | lsr CARG2, RC, #3
1326 | bl extern lj_state_growstack // (lua_State *L, int n)
1327 | mov CRET1, #0
1328 | b <4
1329 |.endmacro
1330 |
1331 | coroutine_resume_wrap 1 // coroutine.resume
1332 | coroutine_resume_wrap 0 // coroutine.wrap
1333 |
1334 |.ffunc coroutine_yield
1335 | ldr TMP0, L->cframe
1336 | add TMP1, BASE, NARGS8:RC
1337 | mov CRET1, #LUA_YIELD
1338 | stp BASE, TMP1, L->base
1339 | tbz TMP0, #0, ->fff_fallback
1340 | str xzr, L->cframe
1341 | strb CRET1w, L->status
1342 | b ->vm_leave_unw
1343 |
1344 |//-- Math library -------------------------------------------------------
1345 |
1346 |.macro math_round, func, round
1347 | .ffunc math_ .. func
1348 | ldr CARG1, [BASE]
1349 | cmp NARGS8:RC, #8
1350 | ldr d0, [BASE]
1351 | blo ->fff_fallback
1352 | cmp TISNUMhi, CARG1, lsr #32
1353 | beq ->fff_restv
1354 | blo ->fff_fallback
1355 | round d0, d0
1356 | b ->fff_resn
1357 |.endmacro
1358 |
1359 | math_round floor, frintm
1360 | math_round ceil, frintp
1361 |
1362 |.ffunc_1 math_abs
1363 | checknumber CARG1, ->fff_fallback
1364 | and CARG1, CARG1, #U64x(7fffffff,ffffffff)
1365 | bne ->fff_restv
1366 | eor CARG2w, CARG1w, CARG1w, asr #31
1367 | movz CARG3, #0x41e0, lsl #48 // 2^31.
1368 | subs CARG1w, CARG2w, CARG1w, asr #31
1369 | add CARG1, CARG1, TISNUM
1370 | csel CARG1, CARG1, CARG3, pl
1371 | // Fallthrough.
1372 |
1373 |->fff_restv:
1374 | // CARG1 = TValue result.
1375 | ldr PC, [BASE, FRAME_PC]
1376 | str CARG1, [BASE, #-16]
1377 |->fff_res1:
1378 | // PC = return.
1379 | mov RC, #(1+1)*8
1380 |->fff_res:
1381 | // RC = (nresults+1)*8, PC = return.
1382 | ands CARG1, PC, #FRAME_TYPE
1383 | str RCw, SAVE_MULTRES
1384 | sub RA, BASE, #16
1385 | bne ->vm_return
1386 | ldr INSw, [PC, #-4]
1387 | decode_RB RB, INS
1388 |5:
1389 | cmp RC, RB, lsl #3 // More results expected?
1390 | blo >6
1391 | decode_RA TMP1, INS
1392 | // Adjust BASE. KBASE is assumed to be set for the calling frame.
1393 | sub BASE, RA, TMP1, lsl #3
1394 | ins_next
1395 |
1396 |6: // Fill up results with nil.
1397 | add TMP1, RA, RC
1398 | add RC, RC, #8
1399 | str TISNIL, [TMP1, #-8]
1400 | b <5
1401 |
1402 |.macro math_extern, func
1403 | .ffunc_n math_ .. func
1404 | bl extern func
1405 | b ->fff_resn
1406 |.endmacro
1407 |
1408 |.macro math_extern2, func
1409 | .ffunc_nn math_ .. func
1410 | bl extern func
1411 | b ->fff_resn
1412 |.endmacro
1413 |
1414 |.ffunc_n math_sqrt
1415 | fsqrt d0, d0
1416 |->fff_resn:
1417 | ldr PC, [BASE, FRAME_PC]
1418 | str d0, [BASE, #-16]
1419 | b ->fff_res1
1420 |
1421 |.ffunc math_log
1422 | ldr CARG1, [BASE]
1423 | cmp NARGS8:RC, #8
1424 | ldr FARG1, [BASE]
1425 | bne ->fff_fallback // Need exactly 1 argument.
1426 | checknum CARG1, ->fff_fallback
1427 | bl extern log
1428 | b ->fff_resn
1429 |
1430 | math_extern log10
1431 | math_extern exp
1432 | math_extern sin
1433 | math_extern cos
1434 | math_extern tan
1435 | math_extern asin
1436 | math_extern acos
1437 | math_extern atan
1438 | math_extern sinh
1439 | math_extern cosh
1440 | math_extern tanh
1441 | math_extern2 pow
1442 | math_extern2 atan2
1443 | math_extern2 fmod
1444 |
1445 |.ffunc_2 math_ldexp
1446 | ldr FARG1, [BASE]
1447 | checknum CARG1, ->fff_fallback
1448 | checkint CARG2, ->fff_fallback
1449 | sxtw CARG1, CARG2w
1450 | bl extern ldexp // (double x, int exp)
1451 | b ->fff_resn
1452 |
1453 |.ffunc_n math_frexp
1454 | add CARG1, sp, TMPDofs
1455 | bl extern frexp
1456 | ldr CARG2w, TMPD
1457 | ldr PC, [BASE, FRAME_PC]
1458 | str d0, [BASE, #-16]
1459 | mov RC, #(2+1)*8
1460 | add CARG2, CARG2, TISNUM
1461 | str CARG2, [BASE, #-8]
1462 | b ->fff_res
1463 |
1464 |.ffunc_n math_modf
1465 | sub CARG1, BASE, #16
1466 | ldr PC, [BASE, FRAME_PC]
1467 | bl extern modf
1468 | mov RC, #(2+1)*8
1469 | str d0, [BASE, #-8]
1470 | b ->fff_res
1471 |
1472 |.macro math_minmax, name, cond, fcond
1473 | .ffunc_1 name
1474 | add RB, BASE, RC
1475 | add RA, BASE, #8
1476 | checkint CARG1, >4
1477 |1: // Handle integers.
1478 | ldr CARG2, [RA]
1479 | cmp RA, RB
1480 | bhs ->fff_restv
1481 | checkint CARG2, >3
1482 | cmp CARG1w, CARG2w
1483 | add RA, RA, #8
1484 | csel CARG1, CARG2, CARG1, cond
1485 | b <1
1486 |3: // Convert intermediate result to number and continue below.
1487 | scvtf d0, CARG1w
1488 | blo ->fff_fallback
1489 | ldr d1, [RA]
1490 | b >6
1491 |
1492 |4:
1493 | ldr d0, [BASE]
1494 | blo ->fff_fallback
1495 |5: // Handle numbers.
1496 | ldr CARG2, [RA]
1497 | ldr d1, [RA]
1498 | cmp RA, RB
1499 | bhs ->fff_resn
1500 | checknum CARG2, >7
1501 |6:
1502 | fcmp d0, d1
1503 | add RA, RA, #8
1504 | fcsel d0, d1, d0, fcond
1505 | b <5
1506 |7: // Convert integer to number and continue above.
1507 | scvtf d1, CARG2w
1508 | blo ->fff_fallback
1509 | b <6
1510 |.endmacro
1511 |
1512 | math_minmax math_min, gt, pl
1513 | math_minmax math_max, lt, le
1514 |
1515 |//-- String library -----------------------------------------------------
1516 |
1517 |.ffunc string_byte // Only handle the 1-arg case here.
1518 | ldp PC, CARG1, [BASE, FRAME_PC]
1519 | cmp NARGS8:RC, #8
1520 | asr ITYPE, CARG1, #47
1521 | ccmn ITYPE, #-LJ_TSTR, #0, eq
1522 | and STR:CARG1, CARG1, #LJ_GCVMASK
1523 | bne ->fff_fallback
1524 | ldrb TMP0w, STR:CARG1[1] // Access is always ok (NUL at end).
1525 | ldr CARG3w, STR:CARG1->len
1526 | add TMP0, TMP0, TISNUM
1527 | str TMP0, [BASE, #-16]
1528 | mov RC, #(0+1)*8
1529 | cbz CARG3, ->fff_res
1530 | b ->fff_res1
1531 |
1532 |.ffunc string_char // Only handle the 1-arg case here.
1533 | ffgccheck
1534 | ldp PC, CARG1, [BASE, FRAME_PC]
1535 | cmp CARG1w, #255
1536 | ccmp NARGS8:RC, #8, #0, ls // Need exactly 1 argument.
1537 | bne ->fff_fallback
1538 | checkint CARG1, ->fff_fallback
1539 | mov CARG3, #1
1540 | // Point to the char inside the integer in the stack slot.
1541 |.if ENDIAN_LE
1542 | mov CARG2, BASE
1543 |.else
1544 | add CARG2, BASE, #7
1545 |.endif
1546 |->fff_newstr:
1547 | // CARG2 = str, CARG3 = len.
1548 | str BASE, L->base
1549 | mov CARG1, L
1550 | str PC, SAVE_PC
1551 | bl extern lj_str_new // (lua_State *L, char *str, size_t l)
1552 |->fff_resstr:
1553 | // Returns GCstr *.
1554 | ldr BASE, L->base
1555 | movn TMP1, #~LJ_TSTR
1556 | add CARG1, CARG1, TMP1, lsl #47
1557 | b ->fff_restv
1558 |
1559 |.ffunc string_sub
1560 | ffgccheck
1561 | ldr CARG1, [BASE]
1562 | ldr CARG3, [BASE, #16]
1563 | cmp NARGS8:RC, #16
1564 | movn RB, #0
1565 | beq >1
1566 | blo ->fff_fallback
1567 | checkint CARG3, ->fff_fallback
1568 | sxtw RB, CARG3w
1569 |1:
1570 | ldr CARG2, [BASE, #8]
1571 | checkstr CARG1, ->fff_fallback
1572 | ldr TMP1w, STR:CARG1->len
1573 | checkint CARG2, ->fff_fallback
1574 | sxtw CARG2, CARG2w
1575 | // CARG1 = str, TMP1 = str->len, CARG2 = start, RB = end
1576 | add TMP2, RB, TMP1
1577 | cmp RB, #0
1578 | add TMP0, CARG2, TMP1
1579 | csinc RB, RB, TMP2, ge // if (end < 0) end += len+1
1580 | cmp CARG2, #0
1581 | csinc CARG2, CARG2, TMP0, ge // if (start < 0) start += len+1
1582 | cmp RB, #0
1583 | csel RB, RB, xzr, ge // if (end < 0) end = 0
1584 | cmp CARG2, #1
1585 | csinc CARG2, CARG2, xzr, ge // if (start < 1) start = 1
1586 | cmp RB, TMP1
1587 | csel RB, RB, TMP1, le // if (end > len) end = len
1588 | add CARG1, STR:CARG1, #sizeof(GCstr)-1
1589 | subs CARG3, RB, CARG2 // len = end - start
1590 | add CARG2, CARG1, CARG2
1591 | add CARG3, CARG3, #1 // len += 1
1592 | bge ->fff_newstr
1593 | add STR:CARG1, GL, #offsetof(global_State, strempty)
1594 | movn TMP1, #~LJ_TSTR
1595 | add CARG1, CARG1, TMP1, lsl #47
1596 | b ->fff_restv
1597 |
1598 |.macro ffstring_op, name
1599 | .ffunc string_ .. name
1600 | ffgccheck
1601 | ldr CARG2, [BASE]
1602 | cmp NARGS8:RC, #8
1603 | asr ITYPE, CARG2, #47
1604 | ccmn ITYPE, #-LJ_TSTR, #0, hs
1605 | and STR:CARG2, CARG2, #LJ_GCVMASK
1606 | bne ->fff_fallback
1607 | ldr TMP0, GL->tmpbuf.b
1608 | add SBUF:CARG1, GL, #offsetof(global_State, tmpbuf)
1609 | str BASE, L->base
1610 | str PC, SAVE_PC
1611 | str L, GL->tmpbuf.L
1612 | str TMP0, GL->tmpbuf.w
1613 | bl extern lj_buf_putstr_ .. name
1614 | bl extern lj_buf_tostr
1615 | b ->fff_resstr
1616 |.endmacro
1617 |
1618 |ffstring_op reverse
1619 |ffstring_op lower
1620 |ffstring_op upper
1621 |
1622 |//-- Bit library --------------------------------------------------------
1623 |
1624 |// FP number to bit conversion for soft-float. Clobbers CARG1-CARG3
1625 |->vm_tobit_fb:
1626 | bls ->fff_fallback
1627 | add CARG2, CARG1, CARG1
1628 | mov CARG3, #1076
1629 | sub CARG3, CARG3, CARG2, lsr #53
1630 | cmp CARG3, #53
1631 | bhi >1
1632 | and CARG2, CARG2, #U64x(001fffff,ffffffff)
1633 | orr CARG2, CARG2, #U64x(00200000,00000000)
1634 | cmp CARG1, #0
1635 | lsr CARG2, CARG2, CARG3
1636 | cneg CARG1w, CARG2w, mi
1637 | br lr
1638 |1:
1639 | mov CARG1w, #0
1640 | br lr
1641 |
1642 |.macro .ffunc_bit, name
1643 | .ffunc_1 bit_..name
1644 | adr lr, >1
1645 | checkint CARG1, ->vm_tobit_fb
1646 |1:
1647 |.endmacro
1648 |
1649 |.macro .ffunc_bit_op, name, ins
1650 | .ffunc_bit name
1651 | mov RA, #8
1652 | mov TMP0w, CARG1w
1653 | adr lr, >2
1654 |1:
1655 | ldr CARG1, [BASE, RA]
1656 | cmp RA, NARGS8:RC
1657 | add RA, RA, #8
1658 | bge >9
1659 | checkint CARG1, ->vm_tobit_fb
1660 |2:
1661 | ins TMP0w, TMP0w, CARG1w
1662 | b <1
1663 |.endmacro
1664 |
1665 |.ffunc_bit_op band, and
1666 |.ffunc_bit_op bor, orr
1667 |.ffunc_bit_op bxor, eor
1668 |
1669 |.ffunc_bit tobit
1670 | mov TMP0w, CARG1w
1671 |9: // Label reused by .ffunc_bit_op users.
1672 | add CARG1, TMP0, TISNUM
1673 | b ->fff_restv
1674 |
1675 |.ffunc_bit bswap
1676 | rev TMP0w, CARG1w
1677 | add CARG1, TMP0, TISNUM
1678 | b ->fff_restv
1679 |
1680 |.ffunc_bit bnot
1681 | mvn TMP0w, CARG1w
1682 | add CARG1, TMP0, TISNUM
1683 | b ->fff_restv
1684 |
1685 |.macro .ffunc_bit_sh, name, ins, shmod
1686 | .ffunc bit_..name
1687 | ldp TMP0, CARG1, [BASE]
1688 | cmp NARGS8:RC, #16
1689 | blo ->fff_fallback
1690 | adr lr, >1
1691 | checkint CARG1, ->vm_tobit_fb
1692 |1:
1693 |.if shmod == 0
1694 | mov TMP1, CARG1
1695 |.else
1696 | neg TMP1, CARG1
1697 |.endif
1698 | mov CARG1, TMP0
1699 | adr lr, >2
1700 | checkint CARG1, ->vm_tobit_fb
1701 |2:
1702 | ins TMP0w, CARG1w, TMP1w
1703 | add CARG1, TMP0, TISNUM
1704 | b ->fff_restv
1705 |.endmacro
1706 |
1707 |.ffunc_bit_sh lshift, lsl, 0
1708 |.ffunc_bit_sh rshift, lsr, 0
1709 |.ffunc_bit_sh arshift, asr, 0
1710 |.ffunc_bit_sh rol, ror, 1
1711 |.ffunc_bit_sh ror, ror, 0
1712 |
1713 |//-----------------------------------------------------------------------
1714 |
1715 |->fff_fallback: // Call fast function fallback handler.
1716 | // BASE = new base, RC = nargs*8
1717 | ldp CFUNC:CARG3, PC, [BASE, FRAME_FUNC] // Fallback may overwrite PC.
1718 | ldr TMP2, L->maxstack
1719 | add TMP1, BASE, NARGS8:RC
1720 | stp BASE, TMP1, L->base
1721 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1722 | add TMP1, TMP1, #8*LUA_MINSTACK
1723 | ldr CARG3, CFUNC:CARG3->f
1724 | str PC, SAVE_PC // Redundant (but a defined value).
1725 | cmp TMP1, TMP2
1726 | mov CARG1, L
1727 | bhi >5 // Need to grow stack.
1728 | blr_auth CARG3 // (lua_State *L)
1729 | // Either throws an error, or recovers and returns -1, 0 or nresults+1.
1730 | ldr BASE, L->base
1731 | cmp CRET1w, #0
1732 | lsl RC, CRET1, #3
1733 | sub RA, BASE, #16
1734 | bgt ->fff_res // Returned nresults+1?
1735 |1: // Returned 0 or -1: retry fast path.
1736 | ldr CARG1, L->top
1737 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1738 | sub NARGS8:RC, CARG1, BASE
1739 | bne ->vm_call_tail // Returned -1?
1740 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1741 | ins_callt // Returned 0: retry fast path.
1742 |
1743 |// Reconstruct previous base for vmeta_call during tailcall.
1744 |->vm_call_tail:
1745 | ands TMP0, PC, #FRAME_TYPE
1746 | and TMP1, PC, #~FRAME_TYPEP
1747 | bne >3
1748 | ldrb RAw, [PC, #-4+OFS_RA]
1749 | lsl RA, RA, #3
1750 | add TMP1, RA, #16
1751 |3:
1752 | sub RB, BASE, TMP1
1753 | b ->vm_call_dispatch // Resolve again for tailcall.
1754 |
1755 |5: // Grow stack for fallback handler.
1756 | mov CARG2, #LUA_MINSTACK
1757 | bl extern lj_state_growstack // (lua_State *L, int n)
1758 | ldr BASE, L->base
1759 | cmp CARG1, CARG1 // Set zero-flag to force retry.
1760 | b <1
1761 |
1762 |->fff_gcstep: // Call GC step function.
1763 | // BASE = new base, RC = nargs*8
1764 | sp_auth
1765 | add CARG2, BASE, NARGS8:RC // Calculate L->top.
1766 | mov RA, lr
1767 | stp BASE, CARG2, L->base
1768 | str PC, SAVE_PC // Redundant (but a defined value).
1769 | mov CARG1, L
1770 | bl extern lj_gc_step // (lua_State *L)
1771 | ldp BASE, CARG2, L->base
1772 | ldr CFUNC:CARG3, [BASE, FRAME_FUNC]
1773 | mov lr, RA // Help return address predictor.
1774 | sub NARGS8:RC, CARG2, BASE // Calculate nargs*8.
1775 | and CFUNC:CARG3, CARG3, #LJ_GCVMASK
1776 | ret_auth
1777 |
1778 |//-----------------------------------------------------------------------
1779 |//-- Special dispatch targets -------------------------------------------
1780 |//-----------------------------------------------------------------------
1781 |
1782 |->vm_record: // Dispatch target for recording phase.
1783 |.if JIT
1784 | ldrb CARG1w, GL->hookmask
1785 | tst CARG1, #HOOK_VMEVENT // No recording while in vmevent.
1786 | bne >5
1787 | // Decrement the hookcount for consistency, but always do the call.
1788 | ldr CARG2w, GL->hookcount
1789 | tst CARG1, #HOOK_ACTIVE
1790 | bne >1
1791 | sub CARG2w, CARG2w, #1
1792 | tst CARG1, #LUA_MASKLINE|LUA_MASKCOUNT
1793 | beq >1
1794 | str CARG2w, GL->hookcount
1795 | b >1
1796 |.endif
1797 |
1798 |->vm_rethook: // Dispatch target for return hooks.
1799 | ldrb TMP2w, GL->hookmask
1800 | tbz TMP2w, #HOOK_ACTIVE_SHIFT, >1 // Hook already active?
1801 |5: // Re-dispatch to static ins.
1802 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1803 | br_auth TMP0
1804 |
1805 |->vm_inshook: // Dispatch target for instr/line hooks.
1806 | ldrb TMP2w, GL->hookmask
1807 | ldr TMP3w, GL->hookcount
1808 | tbnz TMP2w, #HOOK_ACTIVE_SHIFT, <5 // Hook already active?
1809 | tst TMP2w, #LUA_MASKLINE|LUA_MASKCOUNT
1810 | beq <5
1811 | sub TMP3w, TMP3w, #1
1812 | str TMP3w, GL->hookcount
1813 | cbz TMP3w, >1
1814 | tbz TMP2w, #LUA_HOOKLINE, <5
1815 |1:
1816 | mov CARG1, L
1817 | str BASE, L->base
1818 | mov CARG2, PC
1819 | // SAVE_PC must hold the _previous_ PC. The callee updates it with PC.
1820 | bl extern lj_dispatch_ins // (lua_State *L, const BCIns *pc)
1821 |3:
1822 | ldr BASE, L->base
1823 |4: // Re-dispatch to static ins.
1824 | ldr INSw, [PC, #-4]
1825 | add TMP1, GL, INS, uxtb #3
1826 | decode_RA RA, INS
1827 | ldr TMP0, [TMP1, #GG_G2DISP+GG_DISP2STATIC]
1828 | decode_RD RC, INS
1829 | br_auth TMP0
1830 |
1831 |->cont_hook: // Continue from hook yield.
1832 | ldr CARG1, [CARG4, #-40]
1833 | add PC, PC, #4
1834 | str CARG1w, SAVE_MULTRES // Restore MULTRES for *M ins.
1835 | b <4
1836 |
1837 |->vm_hotloop: // Hot loop counter underflow.
1838 |.if JIT
1839 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC] // Same as curr_topL(L).
1840 | add CARG1, GL, #GG_G2DISP+GG_DISP2J
1841 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1842 | str PC, SAVE_PC
1843 | ldr CARG3, LFUNC:CARG3->pc
1844 | mov CARG2, PC
1845 | str L, [GL, #GL_J(L)]
1846 | ldrb CARG3w, [CARG3, #PC2PROTO(framesize)]
1847 | str BASE, L->base
1848 | add CARG3, BASE, CARG3, lsl #3
1849 | str CARG3, L->top
1850 | bl extern lj_trace_hot // (jit_State *J, const BCIns *pc)
1851 | b <3
1852 |.endif
1853 |
1854 |->vm_callhook: // Dispatch target for call hooks.
1855 | mov CARG2, PC
1856 |.if JIT
1857 | b >1
1858 |.endif
1859 |
1860 |->vm_hotcall: // Hot call counter underflow.
1861 |.if JIT
1862 | orr CARG2, PC, #1
1863 |1:
1864 |.endif
1865 | add TMP1, BASE, NARGS8:RC
1866 | str PC, SAVE_PC
1867 | mov CARG1, L
1868 | sub RA, RA, BASE
1869 | stp BASE, TMP1, L->base
1870 | bl extern lj_dispatch_call // (lua_State *L, const BCIns *pc)
1871 | // Returns ASMFunction.
1872 | ldp BASE, TMP1, L->base
1873 | str xzr, SAVE_PC // Invalidate for subsequent line hook.
1874 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
1875 | add RA, BASE, RA
1876 | sub NARGS8:RC, TMP1, BASE
1877 | ldr INSw, [PC, #-4]
1878 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
1879 | br_auth CRET1
1880 |
1881 |->cont_stitch: // Trace stitching.
1882 |.if JIT
1883 | // RA = resultptr, CARG4 = meta base
1884 | ldr RBw, SAVE_MULTRES
1885 | ldr INSw, [PC, #-4]
1886 | ldr TRACE:CARG3, [CARG4, #-40] // Save previous trace.
1887 | subs RB, RB, #8
1888 | decode_RA RC, INS // Call base.
1889 | and CARG3, CARG3, #LJ_GCVMASK
1890 | beq >2
1891 |1: // Move results down.
1892 | ldr CARG1, [RA]
1893 | add RA, RA, #8
1894 | subs RB, RB, #8
1895 | str CARG1, [BASE, RC, lsl #3]
1896 | add RC, RC, #1
1897 | bne <1
1898 |2:
1899 | decode_RA RA, INS
1900 | decode_RB RB, INS
1901 | add RA, RA, RB
1902 |3:
1903 | cmp RA, RC
1904 | bhi >9 // More results wanted?
1905 |
1906 | ldrh RAw, TRACE:CARG3->traceno
1907 | ldrh RCw, TRACE:CARG3->link
1908 | cmp RCw, RAw
1909 | beq ->cont_nop // Blacklisted.
1910 | cmp RCw, #0
1911 | bne =>BC_JLOOP // Jump to stitched trace.
1912 |
1913 | // Stitch a new trace to the previous trace.
1914 | mov CARG1, #GL_J(exitno)
1915 | str RAw, [GL, CARG1]
1916 | mov CARG1, #GL_J(L)
1917 | str L, [GL, CARG1]
1918 | str BASE, L->base
1919 | add CARG1, GL, #GG_G2J
1920 | mov CARG2, PC
1921 | bl extern lj_dispatch_stitch // (jit_State *J, const BCIns *pc)
1922 | ldr BASE, L->base
1923 | b ->cont_nop
1924 |
1925 |9: // Fill up results with nil.
1926 | str TISNIL, [BASE, RC, lsl #3]
1927 | add RC, RC, #1
1928 | b <3
1929 |.endif
1930 |
1931 |->vm_profhook: // Dispatch target for profiler hook.
1932 #if LJ_HASPROFILE
1933 | mov CARG1, L
1934 | str BASE, L->base
1935 | mov CARG2, PC
1936 | bl extern lj_dispatch_profile // (lua_State *L, const BCIns *pc)
1937 | // HOOK_PROFILE is off again, so re-dispatch to dynamic instruction.
1938 | ldr BASE, L->base
1939 | sub PC, PC, #4
1940 | b ->cont_nop
1941 #endif
1942 |
1943 |//-----------------------------------------------------------------------
1944 |//-- Trace exit handler -------------------------------------------------
1945 |//-----------------------------------------------------------------------
1946 |
1947 |.macro savex_, a, b
1948 | stp d..a, d..b, [sp, #a*8]
1949 | stp x..a, x..b, [sp, #32*8+a*8]
1950 |.endmacro
1951 |
1952 |->vm_exit_handler:
1953 |.if JIT
1954 | sub sp, sp, #(64*8)
1955 | savex_, 0, 1
1956 | savex_, 2, 3
1957 | savex_, 4, 5
1958 | savex_, 6, 7
1959 | savex_, 8, 9
1960 | savex_, 10, 11
1961 | savex_, 12, 13
1962 | savex_, 14, 15
1963 | savex_, 16, 17
1964 | savex_, 18, 19
1965 | savex_, 20, 21
1966 | savex_, 22, 23
1967 | savex_, 24, 25
1968 | savex_, 26, 27
1969 | savex_, 28, 29
1970 | stp d30, d31, [sp, #30*8]
1971 | ldr CARG1, [sp, #64*8] // Load original value of lr.
1972 | add CARG3, sp, #64*8 // Recompute original value of sp.
1973 | mv_vmstate CARG4w, EXIT
1974 | stp xzr, CARG3, [sp, #62*8] // Store 0/sp in RID_LR/RID_SP.
1975 | sub CARG1, CARG1, lr
1976 | ldr L, GL->cur_L
1977 | lsr CARG1, CARG1, #2
1978 | ldr BASE, GL->jit_base
1979 | sub CARG1, CARG1, #2
1980 | ldr CARG2w, [lr] // Load trace number.
1981 | st_vmstate CARG4w
1982 |.if ENDIAN_BE
1983 | rev32 CARG2, CARG2
1984 |.endif
1985 | str BASE, L->base
1986 | ubfx CARG2w, CARG2w, #5, #16
1987 | str CARG1w, [GL, #GL_J(exitno)]
1988 | str CARG2w, [GL, #GL_J(parent)]
1989 | str L, [GL, #GL_J(L)]
1990 | str xzr, GL->jit_base
1991 | add CARG1, GL, #GG_G2J
1992 | mov CARG2, sp
1993 | bl extern lj_trace_exit // (jit_State *J, ExitState *ex)
1994 | // Returns MULTRES (unscaled) or negated error code.
1995 | ldr CARG2, L->cframe
1996 | ldr BASE, L->base
1997 | and sp, CARG2, #CFRAME_RAWMASK
1998 | ldr PC, SAVE_PC // Get SAVE_PC.
1999 | str L, SAVE_L // Set SAVE_L (on-trace resume/yield).
2000 | b >1
2001 |.endif
2002 |
2003 |->vm_exit_interp:
2004 | // CARG1 = MULTRES or negated error code, BASE, PC and GL set.
2005 |.if JIT
2006 | ldr L, SAVE_L
2007 |1:
2008 | cmn CARG1w, #LUA_ERRERR
2009 | bhs >9 // Check for error from exit.
2010 | lsl RC, CARG1, #3
2011 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2012 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
2013 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
2014 | movn TISNIL, #0
2015 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2016 | str RCw, SAVE_MULTRES
2017 | str BASE, L->base
2018 | ldr CARG2, LFUNC:CARG2->pc
2019 | str xzr, GL->jit_base
2020 | mv_vmstate CARG4w, INTERP
2021 | ldr KBASE, [CARG2, #PC2PROTO(k)]
2022 | // Modified copy of ins_next which handles function header dispatch, too.
2023 | ldrb RBw, [PC, # OFS_OP]
2024 | ldr INSw, [PC], #4
2025 | st_vmstate CARG4w
2026 | cmn CARG1w, #17 // Static dispatch?
2027 | beq >5
2028 | cmp RBw, #BC_FUNCC+2 // Fast function?
2029 | add TMP1, GL, INS, uxtb #3
2030 | bhs >4
2031 |2:
2032 | cmp RBw, #BC_FUNCF // Function header?
2033 | add TMP0, GL, RB, uxtb #3
2034 | ldr RB, [TMP0, #GG_G2DISP]
2035 | decode_RA RA, INS
2036 | lsr TMP0, INS, #16
2037 | csel RC, TMP0, RC, lo
2038 | blo >3
2039 | ldr CARG3, [BASE, FRAME_FUNC]
2040 | sub RC, RC, #8
2041 | add RA, BASE, RA, lsl #3 // Yes: RA = BASE+framesize*8, RC = nargs*8
2042 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2043 |3:
2044 | br_auth RB
2045 |
2046 |4: // Check frame below fast function.
2047 | ldr CARG1, [BASE, FRAME_PC]
2048 | ands CARG2, CARG1, #FRAME_TYPE
2049 | bne <2 // Trace stitching continuation?
2050 | // Otherwise set KBASE for Lua function below fast function.
2051 | ldr CARG3w, [CARG1, #-4]
2052 | decode_RA CARG1, CARG3
2053 | sub CARG2, BASE, CARG1, lsl #3
2054 | ldr LFUNC:CARG3, [CARG2, #-32]
2055 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2056 | ldr CARG3, LFUNC:CARG3->pc
2057 | ldr KBASE, [CARG3, #PC2PROTO(k)]
2058 | b <2
2059 |
2060 |5: // Dispatch to static entry of original ins replaced by BC_JLOOP.
2061 | ldr RA, [GL, #GL_J(trace)]
2062 | decode_RD RC, INS
2063 | ldr TRACE:RA, [RA, RC, lsl #3]
2064 | ldr INSw, TRACE:RA->startins
2065 | add TMP0, GL, INS, uxtb #3
2066 | decode_RA RA, INS
2067 | ldr RB, [TMP0, #GG_G2DISP+GG_DISP2STATIC]
2068 | decode_RD RC, INS
2069 | br_auth RB
2070 |
2071 |9: // Rethrow error from the right C frame.
2072 | neg CARG2w, CARG1w
2073 | mov CARG1, L
2074 | bl extern lj_err_trace // (lua_State *L, int errcode)
2075 |.endif
2076 |
2077 |//-----------------------------------------------------------------------
2078 |//-- Math helper functions ----------------------------------------------
2079 |//-----------------------------------------------------------------------
2080 |
2081 | // int lj_vm_modi(int dividend, int divisor);
2082 |->vm_modi:
2083 | eor CARG4w, CARG1w, CARG2w
2084 | cmp CARG4w, #0
2085 | eor CARG3w, CARG1w, CARG1w, asr #31
2086 | eor CARG4w, CARG2w, CARG2w, asr #31
2087 | sub CARG3w, CARG3w, CARG1w, asr #31
2088 | sub CARG4w, CARG4w, CARG2w, asr #31
2089 | udiv CARG1w, CARG3w, CARG4w
2090 | msub CARG1w, CARG1w, CARG4w, CARG3w
2091 | ccmp CARG1w, #0, #4, mi
2092 | sub CARG3w, CARG1w, CARG4w
2093 | csel CARG1w, CARG1w, CARG3w, eq
2094 | eor CARG3w, CARG1w, CARG2w
2095 | cmp CARG3w, #0
2096 | cneg CARG1w, CARG1w, mi
2097 | ret
2098 |
2099 |//-----------------------------------------------------------------------
2100 |//-- Miscellaneous functions --------------------------------------------
2101 |//-----------------------------------------------------------------------
2102 |
2103 |.define NEXT_TAB, TAB:CARG1
2104 |.define NEXT_RES, CARG1
2105 |.define NEXT_IDX, CARG2w
2106 |.define NEXT_LIM, CARG3w
2107 |.define NEXT_TMP0, TMP0
2108 |.define NEXT_TMP0w, TMP0w
2109 |.define NEXT_TMP1, TMP1
2110 |.define NEXT_TMP1w, TMP1w
2111 |.define NEXT_RES_PTR, sp
2112 |.define NEXT_RES_VAL, [sp]
2113 |.define NEXT_RES_KEY, [sp, #8]
2114 |
2115 |// TValue *lj_vm_next(GCtab *t, uint32_t idx)
2116 |// Next idx returned in CRET2w.
2117 |->vm_next:
2118 |.if JIT
2119 | ldr NEXT_LIM, NEXT_TAB->asize
2120 | ldr NEXT_TMP1, NEXT_TAB->array
2121 |1: // Traverse array part.
2122 | subs NEXT_TMP0w, NEXT_IDX, NEXT_LIM
2123 | bhs >5 // Index points after array part?
2124 | ldr NEXT_TMP0, [NEXT_TMP1, NEXT_IDX, uxtw #3]
2125 | cmn NEXT_TMP0, #-LJ_TNIL
2126 | cinc NEXT_IDX, NEXT_IDX, eq
2127 | beq <1 // Skip holes in array part.
2128 | str NEXT_TMP0, NEXT_RES_VAL
2129 | movz NEXT_TMP0w, #(LJ_TISNUM>>1)&0xffff, lsl #16
2130 | stp NEXT_IDX, NEXT_TMP0w, NEXT_RES_KEY
2131 | add NEXT_IDX, NEXT_IDX, #1
2132 | mov NEXT_RES, NEXT_RES_PTR
2133 |4:
2134 | ret
2135 |
2136 |5: // Traverse hash part.
2137 | ldr NEXT_TMP1w, NEXT_TAB->hmask
2138 | ldr NODE:NEXT_RES, NEXT_TAB->node
2139 | add NEXT_TMP0w, NEXT_TMP0w, NEXT_TMP0w, lsl #1
2140 | add NEXT_LIM, NEXT_LIM, NEXT_TMP1w
2141 | add NODE:NEXT_RES, NODE:NEXT_RES, NEXT_TMP0w, uxtw #3
2142 |6:
2143 | cmp NEXT_IDX, NEXT_LIM
2144 | bhi >9
2145 | ldr NEXT_TMP0, NODE:NEXT_RES->val
2146 | cmn NEXT_TMP0, #-LJ_TNIL
2147 | add NEXT_IDX, NEXT_IDX, #1
2148 | bne <4
2149 | // Skip holes in hash part.
2150 | add NODE:NEXT_RES, NODE:NEXT_RES, #sizeof(Node)
2151 | b <6
2152 |
2153 |9: // End of iteration. Set the key to nil (not the value).
2154 | movn NEXT_TMP0, #0
2155 | str NEXT_TMP0, NEXT_RES_KEY
2156 | mov NEXT_RES, NEXT_RES_PTR
2157 | ret
2158 |.endif
2159 |
2160 |//-----------------------------------------------------------------------
2161 |//-- FFI helper functions -----------------------------------------------
2162 |//-----------------------------------------------------------------------
2163 |
2164 |// Handler for callback functions.
2165 |// Saveregs already performed. Callback slot number in [sp], g in r12.
2166 |->vm_ffi_callback:
2167 |.if FFI
2168 |.type CTSTATE, CTState, PC
2169 | saveregs
2170 | ldr CTSTATE, GL:x10->ctype_state
2171 | mov GL, x10
2172 | add x10, sp, # CFRAME_SPACE
2173 | str w9, CTSTATE->cb.slot
2174 | stp x0, x1, CTSTATE->cb.gpr[0]
2175 | stp d0, d1, CTSTATE->cb.fpr[0]
2176 | stp x2, x3, CTSTATE->cb.gpr[2]
2177 | stp d2, d3, CTSTATE->cb.fpr[2]
2178 | stp x4, x5, CTSTATE->cb.gpr[4]
2179 | stp d4, d5, CTSTATE->cb.fpr[4]
2180 | stp x6, x7, CTSTATE->cb.gpr[6]
2181 | stp d6, d7, CTSTATE->cb.fpr[6]
2182 | str x10, CTSTATE->cb.stack
2183 | mov CARG1, CTSTATE
2184 | str CTSTATE, SAVE_PC // Any value outside of bytecode is ok.
2185 | mov CARG2, sp
2186 | bl extern lj_ccallback_enter // (CTState *cts, void *cf)
2187 | // Returns lua_State *.
2188 | ldp BASE, RC, L:CRET1->base
2189 | movz TISNUM, #(LJ_TISNUM>>1)&0xffff, lsl #48
2190 | movz TISNUMhi, #(LJ_TISNUM>>1)&0xffff, lsl #16
2191 | movn TISNIL, #0
2192 | mov L, CRET1
2193 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2194 | sub RC, RC, BASE
2195 | st_vmstate ST_INTERP
2196 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2197 | ins_callt
2198 |.endif
2199 |
2200 |->cont_ffi_callback: // Return from FFI callback.
2201 |.if FFI
2202 | ldr CTSTATE, GL->ctype_state
2203 | stp BASE, CARG4, L->base
2204 | str L, CTSTATE->L
2205 | mov CARG1, CTSTATE
2206 | mov CARG2, RA
2207 | bl extern lj_ccallback_leave // (CTState *cts, TValue *o)
2208 | ldp x0, x1, CTSTATE->cb.gpr[0]
2209 | ldp d0, d1, CTSTATE->cb.fpr[0]
2210 | b ->vm_leave_unw
2211 |.endif
2212 |
2213 |->vm_ffi_call: // Call C function via FFI.
2214 | // Caveat: needs special frame unwinding, see below.
2215 |.if FFI
2216 | .type CCSTATE, CCallState, x19
2217 | sp_auth
2218 | stp x20, CCSTATE, [sp, #-32]!
2219 | stp fp, lr, [sp, #16]
2220 | add fp, sp, #16
2221 | mov CCSTATE, x0
2222 | ldr TMP0w, CCSTATE:x0->spadj
2223 | ldrb TMP1w, CCSTATE->nsp
2224 | add TMP2, CCSTATE, #offsetof(CCallState, stack)
2225 | subs TMP1, TMP1, #1
2226 | ldr TMP3, CCSTATE->func
2227 | sub sp, sp, TMP0
2228 | bmi >2
2229 |1: // Copy stack slots
2230 | ldr TMP0, [TMP2, TMP1, lsl #3]
2231 | str TMP0, [sp, TMP1, lsl #3]
2232 | subs TMP1, TMP1, #1
2233 | bpl <1
2234 |2:
2235 | ldp x0, x1, CCSTATE->gpr[0]
2236 | ldp d0, d1, CCSTATE->fpr[0]
2237 | ldp x2, x3, CCSTATE->gpr[2]
2238 | ldp d2, d3, CCSTATE->fpr[2]
2239 | ldp x4, x5, CCSTATE->gpr[4]
2240 | ldp d4, d5, CCSTATE->fpr[4]
2241 | ldp x6, x7, CCSTATE->gpr[6]
2242 | ldp d6, d7, CCSTATE->fpr[6]
2243 | ldr x8, CCSTATE->retp
2244 | blr_auth TMP3
2245 | sub sp, fp, #16
2246 | stp x0, x1, CCSTATE->gpr[0]
2247 | stp d0, d1, CCSTATE->fpr[0]
2248 | stp d2, d3, CCSTATE->fpr[2]
2249 | ldp fp, lr, [sp, #16]
2250 | ldp x20, CCSTATE, [sp], #32
2251 | ret_auth
2252 |.endif
2253 |// Note: vm_ffi_call must be the last function in this object file!
2254 |
2255 |//-----------------------------------------------------------------------
2256 }
2257
2258 /* Generate the code for a single instruction. */
2259 static void build_ins(BuildCtx *ctx, BCOp op, int defop)
2260 {
2261 int vk = 0;
2262 |=>defop:
2263
2264 switch (op) {
2265
2266 /* -- Comparison ops ---------------------------------------------------- */
2267
2268 /* Remember: all ops branch for a true comparison, fall through otherwise. */
2269
2270 case BC_ISLT: case BC_ISGE: case BC_ISLE: case BC_ISGT:
2271 | // RA = src1, RC = src2, JMP with RC = target
2272 | ldr CARG1, [BASE, RA, lsl #3]
2273 | ldrh RBw, [PC, # OFS_RD]
2274 | ldr CARG2, [BASE, RC, lsl #3]
2275 | add PC, PC, #4
2276 | add RB, PC, RB, lsl #2
2277 | sub RB, RB, #0x20000
2278 | checkint CARG1, >3
2279 | checkint CARG2, >4
2280 | cmp CARG1w, CARG2w
2281 if (op == BC_ISLT) {
2282 | csel PC, RB, PC, lt
2283 } else if (op == BC_ISGE) {
2284 | csel PC, RB, PC, ge
2285 } else if (op == BC_ISLE) {
2286 | csel PC, RB, PC, le
2287 } else {
2288 | csel PC, RB, PC, gt
2289 }
2290 |1:
2291 | ins_next
2292 |
2293 |3: // RA not int.
2294 | ldr FARG1, [BASE, RA, lsl #3]
2295 | blo ->vmeta_comp
2296 | ldr FARG2, [BASE, RC, lsl #3]
2297 | cmp TISNUMhi, CARG2, lsr #32
2298 | bhi >5
2299 | bne ->vmeta_comp
2300 | // RA number, RC int.
2301 | scvtf FARG2, CARG2w
2302 | b >5
2303 |
2304 |4: // RA int, RC not int
2305 | ldr FARG2, [BASE, RC, lsl #3]
2306 | blo ->vmeta_comp
2307 | // RA int, RC number.
2308 | scvtf FARG1, CARG1w
2309 |
2310 |5: // RA number, RC number
2311 | fcmp FARG1, FARG2
2312 | // To preserve NaN semantics GE/GT branch on unordered, but LT/LE don't.
2313 if (op == BC_ISLT) {
2314 | csel PC, RB, PC, lo
2315 } else if (op == BC_ISGE) {
2316 | csel PC, RB, PC, hs
2317 } else if (op == BC_ISLE) {
2318 | csel PC, RB, PC, ls
2319 } else {
2320 | csel PC, RB, PC, hi
2321 }
2322 | b <1
2323 break;
2324
2325 case BC_ISEQV: case BC_ISNEV:
2326 vk = op == BC_ISEQV;
2327 | // RA = src1, RC = src2, JMP with RC = target
2328 | ldr CARG1, [BASE, RA, lsl #3]
2329 | add RC, BASE, RC, lsl #3
2330 | ldrh RBw, [PC, # OFS_RD]
2331 | ldr CARG3, [RC]
2332 | add PC, PC, #4
2333 | add RB, PC, RB, lsl #2
2334 | sub RB, RB, #0x20000
2335 | asr ITYPE, CARG3, #47
2336 | cmn ITYPE, #-LJ_TISNUM
2337 if (vk) {
2338 | bls ->BC_ISEQN_Z
2339 } else {
2340 | bls ->BC_ISNEN_Z
2341 }
2342 | // RC is not a number.
2343 | asr TMP0, CARG1, #47
2344 |.if FFI
2345 | // Check if RC or RA is a cdata.
2346 | cmn ITYPE, #-LJ_TCDATA
2347 | ccmn TMP0, #-LJ_TCDATA, #4, ne
2348 | beq ->vmeta_equal_cd
2349 |.endif
2350 | cmp CARG1, CARG3
2351 | bne >2
2352 | // Tag and value are equal.
2353 if (vk) {
2354 |->BC_ISEQV_Z:
2355 | mov PC, RB // Perform branch.
2356 }
2357 |1:
2358 | ins_next
2359 |
2360 |2: // Check if the tags are the same and it's a table or userdata.
2361 | cmp ITYPE, TMP0
2362 | ccmn ITYPE, #-LJ_TISTABUD, #2, eq
2363 if (vk) {
2364 | bhi <1
2365 } else {
2366 | bhi ->BC_ISEQV_Z // Reuse code from opposite instruction.
2367 }
2368 | // Different tables or userdatas. Need to check __eq metamethod.
2369 | // Field metatable must be at same offset for GCtab and GCudata!
2370 | and TAB:CARG2, CARG1, #LJ_GCVMASK
2371 | ldr TAB:TMP2, TAB:CARG2->metatable
2372 if (vk) {
2373 | cbz TAB:TMP2, <1 // No metatable?
2374 | ldrb TMP1w, TAB:TMP2->nomm
2375 | mov CARG4, #0 // ne = 0
2376 | tbnz TMP1w, #MM_eq, <1 // 'no __eq' flag set: done.
2377 } else {
2378 | cbz TAB:TMP2, ->BC_ISEQV_Z // No metatable?
2379 | ldrb TMP1w, TAB:TMP2->nomm
2380 | mov CARG4, #1 // ne = 1.
2381 | tbnz TMP1w, #MM_eq, ->BC_ISEQV_Z // 'no __eq' flag set: done.
2382 }
2383 | b ->vmeta_equal
2384 break;
2385
2386 case BC_ISEQS: case BC_ISNES:
2387 vk = op == BC_ISEQS;
2388 | // RA = src, RC = str_const (~), JMP with RC = target
2389 | ldr CARG1, [BASE, RA, lsl #3]
2390 | mvn RC, RC
2391 | ldrh RBw, [PC, # OFS_RD]
2392 | ldr CARG2, [KBASE, RC, lsl #3]
2393 | add PC, PC, #4
2394 | movn TMP0, #~LJ_TSTR
2395 |.if FFI
2396 | asr ITYPE, CARG1, #47
2397 |.endif
2398 | add RB, PC, RB, lsl #2
2399 | add CARG2, CARG2, TMP0, lsl #47
2400 | sub RB, RB, #0x20000
2401 |.if FFI
2402 | cmn ITYPE, #-LJ_TCDATA
2403 | beq ->vmeta_equal_cd
2404 |.endif
2405 | cmp CARG1, CARG2
2406 if (vk) {
2407 | csel PC, RB, PC, eq
2408 } else {
2409 | csel PC, RB, PC, ne
2410 }
2411 | ins_next
2412 break;
2413
2414 case BC_ISEQN: case BC_ISNEN:
2415 vk = op == BC_ISEQN;
2416 | // RA = src, RC = num_const (~), JMP with RC = target
2417 | ldr CARG1, [BASE, RA, lsl #3]
2418 | add RC, KBASE, RC, lsl #3
2419 | ldrh RBw, [PC, # OFS_RD]
2420 | ldr CARG3, [RC]
2421 | add PC, PC, #4
2422 | add RB, PC, RB, lsl #2
2423 | sub RB, RB, #0x20000
2424 if (vk) {
2425 |->BC_ISEQN_Z:
2426 } else {
2427 |->BC_ISNEN_Z:
2428 }
2429 | checkint CARG1, >4
2430 | checkint CARG3, >6
2431 | cmp CARG1w, CARG3w
2432 |1:
2433 if (vk) {
2434 | csel PC, RB, PC, eq
2435 |2:
2436 } else {
2437 |2:
2438 | csel PC, RB, PC, ne
2439 }
2440 |3:
2441 | ins_next
2442 |
2443 |4: // RA not int.
2444 |.if FFI
2445 | blo >7
2446 |.else
2447 | blo <2
2448 |.endif
2449 | ldr FARG1, [BASE, RA, lsl #3]
2450 | ldr FARG2, [RC]
2451 | cmp TISNUMhi, CARG3, lsr #32
2452 | bne >5
2453 | // RA number, RC int.
2454 | scvtf FARG2, CARG3w
2455 |5:
2456 | // RA number, RC number.
2457 | fcmp FARG1, FARG2
2458 | b <1
2459 |
2460 |6: // RA int, RC number
2461 | ldr FARG2, [RC]
2462 | scvtf FARG1, CARG1w
2463 | fcmp FARG1, FARG2
2464 | b <1
2465 |
2466 |.if FFI
2467 |7:
2468 | asr ITYPE, CARG1, #47
2469 | cmn ITYPE, #-LJ_TCDATA
2470 | bne <2
2471 | b ->vmeta_equal_cd
2472 |.endif
2473 break;
2474
2475 case BC_ISEQP: case BC_ISNEP:
2476 vk = op == BC_ISEQP;
2477 | // RA = src, RC = primitive_type (~), JMP with RC = target
2478 | ldr TMP0, [BASE, RA, lsl #3]
2479 | ldrh RBw, [PC, # OFS_RD]
2480 | add PC, PC, #4
2481 | add RC, RC, #1
2482 | add RB, PC, RB, lsl #2
2483 |.if FFI
2484 | asr ITYPE, TMP0, #47
2485 | cmn ITYPE, #-LJ_TCDATA
2486 | beq ->vmeta_equal_cd
2487 | cmn RC, ITYPE
2488 |.else
2489 | cmn RC, TMP0, asr #47
2490 |.endif
2491 | sub RB, RB, #0x20000
2492 if (vk) {
2493 | csel PC, RB, PC, eq
2494 } else {
2495 | csel PC, RB, PC, ne
2496 }
2497 | ins_next
2498 break;
2499
2500 /* -- Unary test and copy ops ------------------------------------------- */
2501
2502 case BC_ISTC: case BC_ISFC: case BC_IST: case BC_ISF:
2503 | // RA = dst or unused, RC = src, JMP with RC = target
2504 | ldrh RBw, [PC, # OFS_RD]
2505 | ldr TMP0, [BASE, RC, lsl #3]
2506 | add PC, PC, #4
2507 | mov_false TMP1
2508 | add RB, PC, RB, lsl #2
2509 | cmp TMP0, TMP1
2510 | sub RB, RB, #0x20000
2511 if (op == BC_ISTC || op == BC_IST) {
2512 if (op == BC_ISTC) {
2513 | csel RA, RA, RC, lo
2514 }
2515 | csel PC, RB, PC, lo
2516 } else {
2517 if (op == BC_ISFC) {
2518 | csel RA, RA, RC, hs
2519 }
2520 | csel PC, RB, PC, hs
2521 }
2522 if (op == BC_ISTC || op == BC_ISFC) {
2523 | str TMP0, [BASE, RA, lsl #3]
2524 }
2525 | ins_next
2526 break;
2527
2528 case BC_ISTYPE:
2529 | // RA = src, RC = -type
2530 | ldr TMP0, [BASE, RA, lsl #3]
2531 | cmn RC, TMP0, asr #47
2532 | bne ->vmeta_istype
2533 | ins_next
2534 break;
2535 case BC_ISNUM:
2536 | // RA = src, RC = -(TISNUM-1)
2537 | ldr TMP0, [BASE, RA]
2538 | checknum TMP0, ->vmeta_istype
2539 | ins_next
2540 break;
2541
2542 /* -- Unary ops --------------------------------------------------------- */
2543
2544 case BC_MOV:
2545 | // RA = dst, RC = src
2546 | ldr TMP0, [BASE, RC, lsl #3]
2547 | str TMP0, [BASE, RA, lsl #3]
2548 | ins_next
2549 break;
2550 case BC_NOT:
2551 | // RA = dst, RC = src
2552 | ldr TMP0, [BASE, RC, lsl #3]
2553 | mov_false TMP1
2554 | mov_true TMP2
2555 | cmp TMP0, TMP1
2556 | csel TMP0, TMP1, TMP2, lo
2557 | str TMP0, [BASE, RA, lsl #3]
2558 | ins_next
2559 break;
2560 case BC_UNM:
2561 | // RA = dst, RC = src
2562 | ldr TMP0, [BASE, RC, lsl #3]
2563 | asr ITYPE, TMP0, #47
2564 | cmn ITYPE, #-LJ_TISNUM
2565 | bhi ->vmeta_unm
2566 | eor TMP0, TMP0, #U64x(80000000,00000000)
2567 | bne >5
2568 | negs TMP0w, TMP0w
2569 | movz CARG3, #0x41e0, lsl #48 // 2^31.
2570 | add TMP0, TMP0, TISNUM
2571 | csel TMP0, TMP0, CARG3, vc
2572 |5:
2573 | str TMP0, [BASE, RA, lsl #3]
2574 | ins_next
2575 break;
2576 case BC_LEN:
2577 | // RA = dst, RC = src
2578 | ldr CARG1, [BASE, RC, lsl #3]
2579 | asr ITYPE, CARG1, #47
2580 | cmn ITYPE, #-LJ_TSTR
2581 | and CARG1, CARG1, #LJ_GCVMASK
2582 | bne >2
2583 | ldr CARG1w, STR:CARG1->len
2584 |1:
2585 | add CARG1, CARG1, TISNUM
2586 | str CARG1, [BASE, RA, lsl #3]
2587 | ins_next
2588 |
2589 |2:
2590 | cmn ITYPE, #-LJ_TTAB
2591 | bne ->vmeta_len
2592 #if LJ_52
2593 | ldr TAB:CARG2, TAB:CARG1->metatable
2594 | cbnz TAB:CARG2, >9
2595 |3:
2596 #endif
2597 |->BC_LEN_Z:
2598 | bl extern lj_tab_len // (GCtab *t)
2599 | // Returns uint32_t (but less than 2^31).
2600 | b <1
2601 |
2602 #if LJ_52
2603 |9:
2604 | ldrb TMP1w, TAB:CARG2->nomm
2605 | tbnz TMP1w, #MM_len, <3 // 'no __len' flag set: done.
2606 | b ->vmeta_len
2607 #endif
2608 break;
2609
2610 /* -- Binary ops -------------------------------------------------------- */
2611
2612 |.macro ins_arithcheck_int, target
2613 | checkint CARG1, target
2614 | checkint CARG2, target
2615 |.endmacro
2616 |
2617 |.macro ins_arithcheck_num, target
2618 | checknum CARG1, target
2619 | checknum CARG2, target
2620 |.endmacro
2621 |
2622 |.macro ins_arithcheck_nzdiv, target
2623 | cbz CARG2w, target
2624 |.endmacro
2625 |
2626 |.macro ins_arithhead
2627 ||vk = ((int)op - BC_ADDVN) / (BC_ADDNV-BC_ADDVN);
2628 ||if (vk == 1) {
2629 | and RC, RC, #255
2630 | decode_RB RB, INS
2631 ||} else {
2632 | decode_RB RB, INS
2633 | and RC, RC, #255
2634 ||}
2635 |.endmacro
2636 |
2637 |.macro ins_arithload, reg1, reg2
2638 | // RA = dst, RB = src1, RC = src2 | num_const
2639 ||switch (vk) {
2640 ||case 0:
2641 | ldr reg1, [BASE, RB, lsl #3]
2642 | ldr reg2, [KBASE, RC, lsl #3]
2643 || break;
2644 ||case 1:
2645 | ldr reg1, [KBASE, RC, lsl #3]
2646 | ldr reg2, [BASE, RB, lsl #3]
2647 || break;
2648 ||default:
2649 | ldr reg1, [BASE, RB, lsl #3]
2650 | ldr reg2, [BASE, RC, lsl #3]
2651 || break;
2652 ||}
2653 |.endmacro
2654 |
2655 |.macro ins_arithfallback, ins
2656 ||switch (vk) {
2657 ||case 0:
2658 | ins ->vmeta_arith_vn
2659 || break;
2660 ||case 1:
2661 | ins ->vmeta_arith_nv
2662 || break;
2663 ||default:
2664 | ins ->vmeta_arith_vv
2665 || break;
2666 ||}
2667 |.endmacro
2668 |
2669 |.macro ins_arithmod, res, reg1, reg2
2670 | fdiv d2, reg1, reg2
2671 | frintm d2, d2
2672 | // Cannot use fmsub, because FMA is not enabled by default.
2673 | fmul d2, d2, reg2
2674 | fsub res, reg1, d2
2675 |.endmacro
2676 |
2677 |.macro ins_arithdn, intins, fpins
2678 | ins_arithhead
2679 | ins_arithload CARG1, CARG2
2680 | ins_arithcheck_int >5
2681 |.if "intins" == "smull"
2682 | smull CARG1, CARG1w, CARG2w
2683 | cmp CARG1, CARG1, sxtw
2684 | mov CARG1w, CARG1w
2685 | ins_arithfallback bne
2686 |.elif "intins" == "ins_arithmodi"
2687 | ins_arithfallback ins_arithcheck_nzdiv
2688 | bl ->vm_modi
2689 |.else
2690 | intins CARG1w, CARG1w, CARG2w
2691 | ins_arithfallback bvs
2692 |.endif
2693 | add CARG1, CARG1, TISNUM
2694 | str CARG1, [BASE, RA, lsl #3]
2695 |4:
2696 | ins_next
2697 |
2698 |5: // FP variant.
2699 | ins_arithload FARG1, FARG2
2700 | ins_arithfallback ins_arithcheck_num
2701 | fpins FARG1, FARG1, FARG2
2702 | str FARG1, [BASE, RA, lsl #3]
2703 | b <4
2704 |.endmacro
2705 |
2706 |.macro ins_arithfp, fpins
2707 | ins_arithhead
2708 | ins_arithload CARG1, CARG2
2709 | ins_arithload FARG1, FARG2
2710 | ins_arithfallback ins_arithcheck_num
2711 |.if "fpins" == "fpow"
2712 | bl extern pow
2713 |.else
2714 | fpins FARG1, FARG1, FARG2
2715 |.endif
2716 | str FARG1, [BASE, RA, lsl #3]
2717 | ins_next
2718 |.endmacro
2719
2720 case BC_ADDVN: case BC_ADDNV: case BC_ADDVV:
2721 | ins_arithdn adds, fadd
2722 break;
2723 case BC_SUBVN: case BC_SUBNV: case BC_SUBVV:
2724 | ins_arithdn subs, fsub
2725 break;
2726 case BC_MULVN: case BC_MULNV: case BC_MULVV:
2727 | ins_arithdn smull, fmul
2728 break;
2729 case BC_DIVVN: case BC_DIVNV: case BC_DIVVV:
2730 | ins_arithfp fdiv
2731 break;
2732 case BC_MODVN: case BC_MODNV: case BC_MODVV:
2733 | ins_arithdn ins_arithmodi, ins_arithmod
2734 break;
2735 case BC_POW:
2736 | // NYI: (partial) integer arithmetic.
2737 | ins_arithfp fpow
2738 break;
2739
2740 case BC_CAT:
2741 | decode_RB RB, INS
2742 | and RC, RC, #255
2743 | // RA = dst, RB = src_start, RC = src_end
2744 | str BASE, L->base
2745 | sub CARG3, RC, RB
2746 | add CARG2, BASE, RC, lsl #3
2747 |->BC_CAT_Z:
2748 | // RA = dst, CARG2 = top-1, CARG3 = left
2749 | mov CARG1, L
2750 | str PC, SAVE_PC
2751 | bl extern lj_meta_cat // (lua_State *L, TValue *top, int left)
2752 | // Returns NULL (finished) or TValue * (metamethod).
2753 | ldrb RBw, [PC, #-4+OFS_RB]
2754 | ldr BASE, L->base
2755 | cbnz CRET1, ->vmeta_binop
2756 | ldr TMP0, [BASE, RB, lsl #3]
2757 | str TMP0, [BASE, RA, lsl #3] // Copy result to RA.
2758 | ins_next
2759 break;
2760
2761 /* -- Constant ops ------------------------------------------------------ */
2762
2763 case BC_KSTR:
2764 | // RA = dst, RC = str_const (~)
2765 | mvn RC, RC
2766 | ldr TMP0, [KBASE, RC, lsl #3]
2767 | movn TMP1, #~LJ_TSTR
2768 | add TMP0, TMP0, TMP1, lsl #47
2769 | str TMP0, [BASE, RA, lsl #3]
2770 | ins_next
2771 break;
2772 case BC_KCDATA:
2773 |.if FFI
2774 | // RA = dst, RC = cdata_const (~)
2775 | mvn RC, RC
2776 | ldr TMP0, [KBASE, RC, lsl #3]
2777 | movn TMP1, #~LJ_TCDATA
2778 | add TMP0, TMP0, TMP1, lsl #47
2779 | str TMP0, [BASE, RA, lsl #3]
2780 | ins_next
2781 |.endif
2782 break;
2783 case BC_KSHORT:
2784 | // RA = dst, RC = int16_literal
2785 | sxth RCw, RCw
2786 | add TMP0, RC, TISNUM
2787 | str TMP0, [BASE, RA, lsl #3]
2788 | ins_next
2789 break;
2790 case BC_KNUM:
2791 | // RA = dst, RC = num_const
2792 | ldr TMP0, [KBASE, RC, lsl #3]
2793 | str TMP0, [BASE, RA, lsl #3]
2794 | ins_next
2795 break;
2796 case BC_KPRI:
2797 | // RA = dst, RC = primitive_type (~)
2798 | mvn TMP0, RC, lsl #47
2799 | str TMP0, [BASE, RA, lsl #3]
2800 | ins_next
2801 break;
2802 case BC_KNIL:
2803 | // RA = base, RC = end
2804 | add RA, BASE, RA, lsl #3
2805 | add RC, BASE, RC, lsl #3
2806 | str TISNIL, [RA], #8
2807 |1:
2808 | cmp RA, RC
2809 | str TISNIL, [RA], #8
2810 | blt <1
2811 | ins_next_
2812 break;
2813
2814 /* -- Upvalue and function ops ------------------------------------------ */
2815
2816 case BC_UGET:
2817 | // RA = dst, RC = uvnum
2818 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2819 | add RC, RC, #offsetof(GCfuncL, uvptr)/8
2820 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2821 | ldr UPVAL:CARG2, [LFUNC:CARG2, RC, lsl #3]
2822 | ldr CARG2, UPVAL:CARG2->v
2823 | ldr TMP0, [CARG2]
2824 | str TMP0, [BASE, RA, lsl #3]
2825 | ins_next
2826 break;
2827 case BC_USETV:
2828 | // RA = uvnum, RC = src
2829 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2830 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2831 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2832 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2833 | ldr CARG3, [BASE, RC, lsl #3]
2834 | ldr CARG2, UPVAL:CARG1->v
2835 | ldrb TMP2w, UPVAL:CARG1->marked
2836 | ldrb TMP0w, UPVAL:CARG1->closed
2837 | asr ITYPE, CARG3, #47
2838 | str CARG3, [CARG2]
2839 | add ITYPE, ITYPE, #-LJ_TISGCV
2840 | tst TMP2w, #LJ_GC_BLACK // isblack(uv)
2841 | ccmp TMP0w, #0, #4, ne // && uv->closed
2842 | ccmn ITYPE, #-(LJ_TNUMX - LJ_TISGCV), #0, ne // && tvisgcv(v)
2843 | bhi >2
2844 |1:
2845 | ins_next
2846 |
2847 |2: // Check if new value is white.
2848 | and GCOBJ:CARG3, CARG3, #LJ_GCVMASK
2849 | ldrb TMP1w, GCOBJ:CARG3->gch.marked
2850 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2851 | beq <1
2852 | // Crossed a write barrier. Move the barrier forward.
2853 | mov CARG1, GL
2854 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2855 | b <1
2856 break;
2857 case BC_USETS:
2858 | // RA = uvnum, RC = str_const (~)
2859 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2860 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2861 | mvn RC, RC
2862 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2863 | ldr UPVAL:CARG1, [LFUNC:CARG2, RA, lsl #3]
2864 | ldr STR:CARG3, [KBASE, RC, lsl #3]
2865 | movn TMP0, #~LJ_TSTR
2866 | ldr CARG2, UPVAL:CARG1->v
2867 | ldrb TMP2w, UPVAL:CARG1->marked
2868 | add TMP0, STR:CARG3, TMP0, lsl #47
2869 | ldrb TMP1w, STR:CARG3->marked
2870 | str TMP0, [CARG2]
2871 | tbnz TMP2w, #2, >2 // isblack(uv)
2872 |1:
2873 | ins_next
2874 |
2875 |2: // Check if string is white and ensure upvalue is closed.
2876 | ldrb TMP0w, UPVAL:CARG1->closed
2877 | tst TMP1w, #LJ_GC_WHITES // iswhite(str)
2878 | ccmp TMP0w, #0, #4, ne
2879 | beq <1
2880 | // Crossed a write barrier. Move the barrier forward.
2881 | mov CARG1, GL
2882 | bl extern lj_gc_barrieruv // (global_State *g, TValue *tv)
2883 | b <1
2884 break;
2885 case BC_USETN:
2886 | // RA = uvnum, RC = num_const
2887 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2888 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2889 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2890 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2891 | ldr TMP0, [KBASE, RC, lsl #3]
2892 | ldr CARG2, UPVAL:CARG2->v
2893 | str TMP0, [CARG2]
2894 | ins_next
2895 break;
2896 case BC_USETP:
2897 | // RA = uvnum, RC = primitive_type (~)
2898 | ldr LFUNC:CARG2, [BASE, FRAME_FUNC]
2899 | add RA, RA, #offsetof(GCfuncL, uvptr)/8
2900 | and LFUNC:CARG2, CARG2, #LJ_GCVMASK
2901 | ldr UPVAL:CARG2, [LFUNC:CARG2, RA, lsl #3]
2902 | mvn TMP0, RC, lsl #47
2903 | ldr CARG2, UPVAL:CARG2->v
2904 | str TMP0, [CARG2]
2905 | ins_next
2906 break;
2907
2908 case BC_UCLO:
2909 | // RA = level, RC = target
2910 | ldr CARG3, L->openupval
2911 | add RC, PC, RC, lsl #2
2912 | str BASE, L->base
2913 | sub PC, RC, #0x20000
2914 | cbz CARG3, >1
2915 | mov CARG1, L
2916 | add CARG2, BASE, RA, lsl #3
2917 | bl extern lj_func_closeuv // (lua_State *L, TValue *level)
2918 | ldr BASE, L->base
2919 |1:
2920 | ins_next
2921 break;
2922
2923 case BC_FNEW:
2924 | // RA = dst, RC = proto_const (~) (holding function prototype)
2925 | mvn RC, RC
2926 | str BASE, L->base
2927 | ldr LFUNC:CARG3, [BASE, FRAME_FUNC]
2928 | str PC, SAVE_PC
2929 | ldr CARG2, [KBASE, RC, lsl #3]
2930 | mov CARG1, L
2931 | and LFUNC:CARG3, CARG3, #LJ_GCVMASK
2932 | // (lua_State *L, GCproto *pt, GCfuncL *parent)
2933 | bl extern lj_func_newL_gc
2934 | // Returns GCfuncL *.
2935 | ldr BASE, L->base
2936 | movn TMP0, #~LJ_TFUNC
2937 | add CRET1, CRET1, TMP0, lsl #47
2938 | str CRET1, [BASE, RA, lsl #3]
2939 | ins_next
2940 break;
2941
2942 /* -- Table ops --------------------------------------------------------- */
2943
2944 case BC_TNEW:
2945 case BC_TDUP:
2946 | // RA = dst, RC = (hbits|asize) | tab_const (~)
2947 | ldp CARG3, CARG4, GL->gc.total // Assumes threshold follows total.
2948 | str BASE, L->base
2949 | str PC, SAVE_PC
2950 | mov CARG1, L
2951 | cmp CARG3, CARG4
2952 | bhs >5
2953 |1:
2954 if (op == BC_TNEW) {
2955 | and CARG2, RC, #0x7ff
2956 | lsr CARG3, RC, #11
2957 | cmp CARG2, #0x7ff
2958 | mov TMP0, #0x801
2959 | csel CARG2, CARG2, TMP0, ne
2960 | bl extern lj_tab_new // (lua_State *L, int32_t asize, uint32_t hbits)
2961 | // Returns GCtab *.
2962 } else {
2963 | mvn RC, RC
2964 | ldr CARG2, [KBASE, RC, lsl #3]
2965 | bl extern lj_tab_dup // (lua_State *L, Table *kt)
2966 | // Returns GCtab *.
2967 }
2968 | ldr BASE, L->base
2969 | movk CRET1, #(LJ_TTAB>>1)&0xffff, lsl #48
2970 | str CRET1, [BASE, RA, lsl #3]
2971 | ins_next
2972 |
2973 |5:
2974 | bl extern lj_gc_step_fixtop // (lua_State *L)
2975 | mov CARG1, L
2976 | b <1
2977 break;
2978
2979 case BC_GGET:
2980 | // RA = dst, RC = str_const (~)
2981 case BC_GSET:
2982 | // RA = src, RC = str_const (~)
2983 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
2984 | mvn RC, RC
2985 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
2986 | ldr TAB:CARG2, LFUNC:CARG1->env
2987 | ldr STR:RC, [KBASE, RC, lsl #3]
2988 if (op == BC_GGET) {
2989 | b ->BC_TGETS_Z
2990 } else {
2991 | b ->BC_TSETS_Z
2992 }
2993 break;
2994
2995 case BC_TGETV:
2996 | decode_RB RB, INS
2997 | and RC, RC, #255
2998 | // RA = dst, RB = table, RC = key
2999 | ldr CARG2, [BASE, RB, lsl #3]
3000 | ldr TMP1, [BASE, RC, lsl #3]
3001 | checktab CARG2, ->vmeta_tgetv
3002 | checkint TMP1, >9 // Integer key?
3003 | ldr CARG3, TAB:CARG2->array
3004 | ldr CARG1w, TAB:CARG2->asize
3005 | add CARG3, CARG3, TMP1, uxtw #3
3006 | cmp TMP1w, CARG1w // In array part?
3007 | bhs ->vmeta_tgetv
3008 | ldr TMP0, [CARG3]
3009 | cmp TMP0, TISNIL
3010 | beq >5
3011 |1:
3012 | str TMP0, [BASE, RA, lsl #3]
3013 | ins_next
3014 |
3015 |5: // Check for __index if table value is nil.
3016 | ldr TAB:CARG1, TAB:CARG2->metatable
3017 | cbz TAB:CARG1, <1 // No metatable: done.
3018 | ldrb TMP1w, TAB:CARG1->nomm
3019 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
3020 | b ->vmeta_tgetv
3021 |
3022 |9:
3023 | asr ITYPE, TMP1, #47
3024 | cmn ITYPE, #-LJ_TSTR // String key?
3025 | bne ->vmeta_tgetv
3026 | and STR:RC, TMP1, #LJ_GCVMASK
3027 | b ->BC_TGETS_Z
3028 break;
3029 case BC_TGETS:
3030 | decode_RB RB, INS
3031 | and RC, RC, #255
3032 | // RA = dst, RB = table, RC = str_const (~)
3033 | ldr CARG2, [BASE, RB, lsl #3]
3034 | mvn RC, RC
3035 | ldr STR:RC, [KBASE, RC, lsl #3]
3036 | checktab CARG2, ->vmeta_tgets1
3037 |->BC_TGETS_Z:
3038 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = dst
3039 | ldr TMP1w, TAB:CARG2->hmask
3040 | ldr TMP2w, STR:RC->sid
3041 | ldr NODE:CARG3, TAB:CARG2->node
3042 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
3043 | add TMP1, TMP1, TMP1, lsl #1
3044 | movn CARG4, #~LJ_TSTR
3045 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
3046 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
3047 |1:
3048 | ldp TMP0, CARG1, NODE:CARG3->val
3049 | ldr NODE:CARG3, NODE:CARG3->next
3050 | cmp CARG1, CARG4
3051 | bne >4
3052 | cmp TMP0, TISNIL
3053 | beq >5
3054 |3:
3055 | str TMP0, [BASE, RA, lsl #3]
3056 | ins_next
3057 |
3058 |4: // Follow hash chain.
3059 | cbnz NODE:CARG3, <1
3060 | // End of hash chain: key not found, nil result.
3061 | mov TMP0, TISNIL
3062 |
3063 |5: // Check for __index if table value is nil.
3064 | ldr TAB:CARG1, TAB:CARG2->metatable
3065 | cbz TAB:CARG1, <3 // No metatable: done.
3066 | ldrb TMP1w, TAB:CARG1->nomm
3067 | tbnz TMP1w, #MM_index, <3 // 'no __index' flag set: done.
3068 | b ->vmeta_tgets
3069 break;
3070 case BC_TGETB:
3071 | decode_RB RB, INS
3072 | and RC, RC, #255
3073 | // RA = dst, RB = table, RC = index
3074 | ldr CARG2, [BASE, RB, lsl #3]
3075 | checktab CARG2, ->vmeta_tgetb
3076 | ldr CARG3, TAB:CARG2->array
3077 | ldr CARG1w, TAB:CARG2->asize
3078 | add CARG3, CARG3, RC, lsl #3
3079 | cmp RCw, CARG1w // In array part?
3080 | bhs ->vmeta_tgetb
3081 | ldr TMP0, [CARG3]
3082 | cmp TMP0, TISNIL
3083 | beq >5
3084 |1:
3085 | str TMP0, [BASE, RA, lsl #3]
3086 | ins_next
3087 |
3088 |5: // Check for __index if table value is nil.
3089 | ldr TAB:CARG1, TAB:CARG2->metatable
3090 | cbz TAB:CARG1, <1 // No metatable: done.
3091 | ldrb TMP1w, TAB:CARG1->nomm
3092 | tbnz TMP1w, #MM_index, <1 // 'no __index' flag set: done.
3093 | b ->vmeta_tgetb
3094 break;
3095 case BC_TGETR:
3096 | decode_RB RB, INS
3097 | and RC, RC, #255
3098 | // RA = dst, RB = table, RC = key
3099 | ldr CARG1, [BASE, RB, lsl #3]
3100 | ldr TMP1, [BASE, RC, lsl #3]
3101 | and TAB:CARG1, CARG1, #LJ_GCVMASK
3102 | ldr CARG3, TAB:CARG1->array
3103 | ldr TMP2w, TAB:CARG1->asize
3104 | add CARG3, CARG3, TMP1w, uxtw #3
3105 | cmp TMP1w, TMP2w // In array part?
3106 | bhs ->vmeta_tgetr
3107 | ldr TMP0, [CARG3]
3108 |->BC_TGETR_Z:
3109 | str TMP0, [BASE, RA, lsl #3]
3110 | ins_next
3111 break;
3112
3113 case BC_TSETV:
3114 | decode_RB RB, INS
3115 | and RC, RC, #255
3116 | // RA = src, RB = table, RC = key
3117 | ldr CARG2, [BASE, RB, lsl #3]
3118 | ldr TMP1, [BASE, RC, lsl #3]
3119 | checktab CARG2, ->vmeta_tsetv
3120 | checkint TMP1, >9 // Integer key?
3121 | ldr CARG3, TAB:CARG2->array
3122 | ldr CARG1w, TAB:CARG2->asize
3123 | add CARG3, CARG3, TMP1, uxtw #3
3124 | cmp TMP1w, CARG1w // In array part?
3125 | bhs ->vmeta_tsetv
3126 | ldr TMP1, [CARG3]
3127 | ldr TMP0, [BASE, RA, lsl #3]
3128 | ldrb TMP2w, TAB:CARG2->marked
3129 | cmp TMP1, TISNIL // Previous value is nil?
3130 | beq >5
3131 |1:
3132 | str TMP0, [CARG3]
3133 | tbnz TMP2w, #2, >7 // isblack(table)
3134 |2:
3135 | ins_next
3136 |
3137 |5: // Check for __newindex if previous value is nil.
3138 | ldr TAB:CARG1, TAB:CARG2->metatable
3139 | cbz TAB:CARG1, <1 // No metatable: done.
3140 | ldrb TMP1w, TAB:CARG1->nomm
3141 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
3142 | b ->vmeta_tsetv
3143 |
3144 |7: // Possible table write barrier for the value. Skip valiswhite check.
3145 | barrierback TAB:CARG2, TMP2w, TMP1
3146 | b <2
3147 |
3148 |9:
3149 | asr ITYPE, TMP1, #47
3150 | cmn ITYPE, #-LJ_TSTR // String key?
3151 | bne ->vmeta_tsetv
3152 | and STR:RC, TMP1, #LJ_GCVMASK
3153 | b ->BC_TSETS_Z
3154 break;
3155 case BC_TSETS:
3156 | decode_RB RB, INS
3157 | and RC, RC, #255
3158 | // RA = dst, RB = table, RC = str_const (~)
3159 | ldr CARG2, [BASE, RB, lsl #3]
3160 | mvn RC, RC
3161 | ldr STR:RC, [KBASE, RC, lsl #3]
3162 | checktab CARG2, ->vmeta_tsets1
3163 |->BC_TSETS_Z:
3164 | // TAB:CARG2 = GCtab *, STR:RC = GCstr *, RA = src
3165 | ldr TMP1w, TAB:CARG2->hmask
3166 | ldr TMP2w, STR:RC->sid
3167 | ldr NODE:CARG3, TAB:CARG2->node
3168 | and TMP1w, TMP1w, TMP2w // idx = str->sid & tab->hmask
3169 | add TMP1, TMP1, TMP1, lsl #1
3170 | movn CARG4, #~LJ_TSTR
3171 | add NODE:CARG3, NODE:CARG3, TMP1, lsl #3 // node = tab->node + idx*3*8
3172 | add CARG4, STR:RC, CARG4, lsl #47 // Tagged key to look for.
3173 | strb wzr, TAB:CARG2->nomm // Clear metamethod cache.
3174 |1:
3175 | ldp TMP1, CARG1, NODE:CARG3->val
3176 | ldr NODE:TMP3, NODE:CARG3->next
3177 | ldrb TMP2w, TAB:CARG2->marked
3178 | cmp CARG1, CARG4
3179 | bne >5
3180 | ldr TMP0, [BASE, RA, lsl #3]
3181 | cmp TMP1, TISNIL // Previous value is nil?
3182 | beq >4
3183 |2:
3184 | str TMP0, NODE:CARG3->val
3185 | tbnz TMP2w, #2, >7 // isblack(table)
3186 |3:
3187 | ins_next
3188 |
3189 |4: // Check for __newindex if previous value is nil.
3190 | ldr TAB:CARG1, TAB:CARG2->metatable
3191 | cbz TAB:CARG1, <2 // No metatable: done.
3192 | ldrb TMP1w, TAB:CARG1->nomm
3193 | tbnz TMP1w, #MM_newindex, <2 // 'no __newindex' flag set: done.
3194 | b ->vmeta_tsets
3195 |
3196 |5: // Follow hash chain.
3197 | mov NODE:CARG3, NODE:TMP3
3198 | cbnz NODE:TMP3, <1
3199 | // End of hash chain: key not found, add a new one.
3200 |
3201 | // But check for __newindex first.
3202 | ldr TAB:CARG1, TAB:CARG2->metatable
3203 | cbz TAB:CARG1, >6 // No metatable: continue.
3204 | ldrb TMP1w, TAB:CARG1->nomm
3205 | // 'no __newindex' flag NOT set: check.
3206 | tbz TMP1w, #MM_newindex, ->vmeta_tsets
3207 |6:
3208 | movn TMP1, #~LJ_TSTR
3209 | str PC, SAVE_PC
3210 | add TMP0, STR:RC, TMP1, lsl #47
3211 | str BASE, L->base
3212 | mov CARG1, L
3213 | str TMP0, TMPD
3214 | add CARG3, sp, TMPDofs
3215 | bl extern lj_tab_newkey // (lua_State *L, GCtab *t, TValue *k)
3216 | // Returns TValue *.
3217 | ldr BASE, L->base
3218 | ldr TMP0, [BASE, RA, lsl #3]
3219 | str TMP0, [CRET1]
3220 | b <3 // No 2nd write barrier needed.
3221 |
3222 |7: // Possible table write barrier for the value. Skip valiswhite check.
3223 | barrierback TAB:CARG2, TMP2w, TMP1
3224 | b <3
3225 break;
3226 case BC_TSETB:
3227 | decode_RB RB, INS
3228 | and RC, RC, #255
3229 | // RA = src, RB = table, RC = index
3230 | ldr CARG2, [BASE, RB, lsl #3]
3231 | checktab CARG2, ->vmeta_tsetb
3232 | ldr CARG3, TAB:CARG2->array
3233 | ldr CARG1w, TAB:CARG2->asize
3234 | add CARG3, CARG3, RC, lsl #3
3235 | cmp RCw, CARG1w // In array part?
3236 | bhs ->vmeta_tsetb
3237 | ldr TMP1, [CARG3]
3238 | ldr TMP0, [BASE, RA, lsl #3]
3239 | ldrb TMP2w, TAB:CARG2->marked
3240 | cmp TMP1, TISNIL // Previous value is nil?
3241 | beq >5
3242 |1:
3243 | str TMP0, [CARG3]
3244 | tbnz TMP2w, #2, >7 // isblack(table)
3245 |2:
3246 | ins_next
3247 |
3248 |5: // Check for __newindex if previous value is nil.
3249 | ldr TAB:CARG1, TAB:CARG2->metatable
3250 | cbz TAB:CARG1, <1 // No metatable: done.
3251 | ldrb TMP1w, TAB:CARG1->nomm
3252 | tbnz TMP1w, #MM_newindex, <1 // 'no __newindex' flag set: done.
3253 | b ->vmeta_tsetb
3254 |
3255 |7: // Possible table write barrier for the value. Skip valiswhite check.
3256 | barrierback TAB:CARG2, TMP2w, TMP1
3257 | b <2
3258 break;
3259 case BC_TSETR:
3260 | decode_RB RB, INS
3261 | and RC, RC, #255
3262 | // RA = src, RB = table, RC = key
3263 | ldr CARG2, [BASE, RB, lsl #3]
3264 | ldr TMP1, [BASE, RC, lsl #3]
3265 | and TAB:CARG2, CARG2, #LJ_GCVMASK
3266 | ldr CARG1, TAB:CARG2->array
3267 | ldrb TMP2w, TAB:CARG2->marked
3268 | ldr CARG4w, TAB:CARG2->asize
3269 | add CARG1, CARG1, TMP1, uxtw #3
3270 | tbnz TMP2w, #2, >7 // isblack(table)
3271 |2:
3272 | cmp TMP1w, CARG4w // In array part?
3273 | bhs ->vmeta_tsetr
3274 |->BC_TSETR_Z:
3275 | ldr TMP0, [BASE, RA, lsl #3]
3276 | str TMP0, [CARG1]
3277 | ins_next
3278 |
3279 |7: // Possible table write barrier for the value. Skip valiswhite check.
3280 | barrierback TAB:CARG2, TMP2w, TMP0
3281 | b <2
3282 break;
3283
3284 case BC_TSETM:
3285 | // RA = base (table at base-1), RC = num_const (start index)
3286 | add RA, BASE, RA, lsl #3
3287 |1:
3288 | ldr RBw, SAVE_MULTRES
3289 | ldr TAB:CARG2, [RA, #-8] // Guaranteed to be a table.
3290 | ldr TMP1, [KBASE, RC, lsl #3] // Integer constant is in lo-word.
3291 | sub RB, RB, #8
3292 | cbz RB, >4 // Nothing to copy?
3293 | and TAB:CARG2, CARG2, #LJ_GCVMASK
3294 | ldr CARG1w, TAB:CARG2->asize
3295 | add CARG3w, TMP1w, RBw, lsr #3
3296 | ldr CARG4, TAB:CARG2->array
3297 | cmp CARG3, CARG1
3298 | add RB, RA, RB
3299 | bhi >5
3300 | add TMP1, CARG4, TMP1w, uxtw #3
3301 | ldrb TMP2w, TAB:CARG2->marked
3302 |3: // Copy result slots to table.
3303 | ldr TMP0, [RA], #8
3304 | str TMP0, [TMP1], #8
3305 | cmp RA, RB
3306 | blo <3
3307 | tbnz TMP2w, #2, >7 // isblack(table)
3308 |4:
3309 | ins_next
3310 |
3311 |5: // Need to resize array part.
3312 | str BASE, L->base
3313 | mov CARG1, L
3314 | str PC, SAVE_PC
3315 | bl extern lj_tab_reasize // (lua_State *L, GCtab *t, int nasize)
3316 | // Must not reallocate the stack.
3317 | b <1
3318 |
3319 |7: // Possible table write barrier for any value. Skip valiswhite check.
3320 | barrierback TAB:CARG2, TMP2w, TMP1
3321 | b <4
3322 break;
3323
3324 /* -- Calls and vararg handling ----------------------------------------- */
3325
3326 case BC_CALLM:
3327 | // RA = base, (RB = nresults+1,) RC = extra_nargs
3328 | ldr TMP0w, SAVE_MULTRES
3329 | decode_RC8RD NARGS8:RC, RC
3330 | add NARGS8:RC, NARGS8:RC, TMP0
3331 | b ->BC_CALL_Z
3332 break;
3333 case BC_CALL:
3334 | decode_RC8RD NARGS8:RC, RC
3335 | // RA = base, (RB = nresults+1,) RC = (nargs+1)*8
3336 |->BC_CALL_Z:
3337 | mov RB, BASE // Save old BASE for vmeta_call.
3338 | add BASE, BASE, RA, lsl #3
3339 | ldr CARG3, [BASE]
3340 | sub NARGS8:RC, NARGS8:RC, #8
3341 | add BASE, BASE, #16
3342 | checkfunc CARG3, ->vmeta_call
3343 | ins_call
3344 break;
3345
3346 case BC_CALLMT:
3347 | // RA = base, (RB = 0,) RC = extra_nargs
3348 | ldr TMP0w, SAVE_MULTRES
3349 | add NARGS8:RC, TMP0, RC, lsl #3
3350 | b ->BC_CALLT1_Z
3351 break;
3352 case BC_CALLT:
3353 | lsl NARGS8:RC, RC, #3
3354 | // RA = base, (RB = 0,) RC = (nargs+1)*8
3355 |->BC_CALLT1_Z:
3356 | add RA, BASE, RA, lsl #3
3357 | ldr TMP1, [RA]
3358 | sub NARGS8:RC, NARGS8:RC, #8
3359 | add RA, RA, #16
3360 | checktp CARG3, TMP1, LJ_TFUNC, ->vmeta_callt
3361 | ldr PC, [BASE, FRAME_PC]
3362 |->BC_CALLT2_Z:
3363 | mov RB, #0
3364 | ldrb TMP2w, LFUNC:CARG3->ffid
3365 | tst PC, #FRAME_TYPE
3366 | bne >7
3367 |1:
3368 | str TMP1, [BASE, FRAME_FUNC] // Copy function down, but keep PC.
3369 | cbz NARGS8:RC, >3
3370 |2:
3371 | ldr TMP0, [RA, RB]
3372 | add TMP1, RB, #8
3373 | cmp TMP1, NARGS8:RC
3374 | str TMP0, [BASE, RB]
3375 | mov RB, TMP1
3376 | bne <2
3377 |3:
3378 | cmp TMP2, #1 // (> FF_C) Calling a fast function?
3379 | bhi >5
3380 |4:
3381 | ins_callt
3382 |
3383 |5: // Tailcall to a fast function with a Lua frame below.
3384 | ldrb RAw, [PC, #-4+OFS_RA]
3385 | sub CARG1, BASE, RA, lsl #3
3386 | ldr LFUNC:CARG1, [CARG1, #-32]
3387 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3388 | ldr CARG1, LFUNC:CARG1->pc
3389 | ldr KBASE, [CARG1, #PC2PROTO(k)]
3390 | b <4
3391 |
3392 |7: // Tailcall from a vararg function.
3393 | eor PC, PC, #FRAME_VARG
3394 | tst PC, #FRAME_TYPEP // Vararg frame below?
3395 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3396 | bne <1
3397 | sub BASE, BASE, PC
3398 | ldr PC, [BASE, FRAME_PC]
3399 | tst PC, #FRAME_TYPE
3400 | csel TMP2, RB, TMP2, ne // Clear ffid if no Lua function below.
3401 | b <1
3402 break;
3403
3404 case BC_ITERC:
3405 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3406 | add RA, BASE, RA, lsl #3
3407 | ldr CARG3, [RA, #-24]
3408 | mov RB, BASE // Save old BASE for vmeta_call.
3409 | ldp CARG1, CARG2, [RA, #-16]
3410 | add BASE, RA, #16
3411 | mov NARGS8:RC, #16 // Iterators get 2 arguments.
3412 | str CARG3, [RA] // Copy callable.
3413 | stp CARG1, CARG2, [RA, #16] // Copy state and control var.
3414 | checkfunc CARG3, ->vmeta_call
3415 | ins_call
3416 break;
3417
3418 case BC_ITERN:
3419 |.if JIT
3420 | hotloop
3421 |.endif
3422 |->vm_IITERN:
3423 | // RA = base, (RB = nresults+1, RC = nargs+1 (2+1))
3424 | add RA, BASE, RA, lsl #3
3425 | ldr TAB:RB, [RA, #-16]
3426 | ldrh TMP3w, [PC, # OFS_RD]
3427 | ldr CARG1w, [RA, #-8+LO] // Get index from control var.
3428 | add PC, PC, #4
3429 | add TMP3, PC, TMP3, lsl #2
3430 | and TAB:RB, RB, #LJ_GCVMASK
3431 | sub TMP3, TMP3, #0x20000
3432 | ldr TMP1w, TAB:RB->asize
3433 | ldr CARG2, TAB:RB->array
3434 |1: // Traverse array part.
3435 | subs RC, CARG1, TMP1
3436 | add CARG3, CARG2, CARG1, lsl #3
3437 | bhs >5 // Index points after array part?
3438 | ldr TMP0, [CARG3]
3439 | cmp TMP0, TISNIL
3440 | cinc CARG1, CARG1, eq // Skip holes in array part.
3441 | beq <1
3442 | add CARG1, CARG1, TISNUM
3443 | stp CARG1, TMP0, [RA]
3444 | add CARG1, CARG1, #1
3445 |3:
3446 | str CARG1w, [RA, #-8+LO] // Update control var.
3447 | mov PC, TMP3
3448 |4:
3449 | ins_next
3450 |
3451 |5: // Traverse hash part.
3452 | ldr TMP2w, TAB:RB->hmask
3453 | ldr NODE:RB, TAB:RB->node
3454 |6:
3455 | add CARG1, RC, RC, lsl #1
3456 | cmp RC, TMP2 // End of iteration? Branch to ITERN+1.
3457 | add NODE:CARG3, NODE:RB, CARG1, lsl #3 // node = tab->node + idx*3*8
3458 | bhi <4
3459 | ldp TMP0, CARG1, NODE:CARG3->val
3460 | cmp TMP0, TISNIL
3461 | add RC, RC, #1
3462 | beq <6 // Skip holes in hash part.
3463 | stp CARG1, TMP0, [RA]
3464 | add CARG1, RC, TMP1
3465 | b <3
3466 break;
3467
3468 case BC_ISNEXT:
3469 | // RA = base, RC = target (points to ITERN)
3470 | add RA, BASE, RA, lsl #3
3471 | ldr CFUNC:CARG1, [RA, #-24]
3472 | add RC, PC, RC, lsl #2
3473 | ldp TAB:CARG3, CARG4, [RA, #-16]
3474 | sub RC, RC, #0x20000
3475 | checkfunc CFUNC:CARG1, >5
3476 | asr TMP0, TAB:CARG3, #47
3477 | ldrb TMP1w, CFUNC:CARG1->ffid
3478 | cmn TMP0, #-LJ_TTAB
3479 | ccmp CARG4, TISNIL, #0, eq
3480 | ccmp TMP1w, #FF_next_N, #0, eq
3481 | bne >5
3482 | mov TMP0w, #0xfffe7fff // LJ_KEYINDEX
3483 | lsl TMP0, TMP0, #32
3484 | str TMP0, [RA, #-8] // Initialize control var.
3485 |1:
3486 | mov PC, RC
3487 | ins_next
3488 |
3489 |5: // Despecialize bytecode if any of the checks fail.
3490 |.if JIT
3491 | ldrb TMP2w, [RC, # OFS_OP]
3492 |.endif
3493 | mov TMP0, #BC_JMP
3494 | mov TMP1, #BC_ITERC
3495 | strb TMP0w, [PC, #-4+OFS_OP]
3496 |.if JIT
3497 | cmp TMP2w, #BC_ITERN
3498 | bne >6
3499 |.endif
3500 | strb TMP1w, [RC, # OFS_OP]
3501 | b <1
3502 |.if JIT
3503 |6: // Unpatch JLOOP.
3504 | ldr RA, [GL, #GL_J(trace)]
3505 | ldrh TMP2w, [RC, # OFS_RD]
3506 | ldr TRACE:RA, [RA, TMP2, lsl #3]
3507 | ldr TMP2w, TRACE:RA->startins
3508 | bfxil TMP2w, TMP1w, #0, #8
3509 | str TMP2w, [RC]
3510 | b <1
3511 |.endif
3512 break;
3513
3514 case BC_VARG:
3515 | decode_RB RB, INS
3516 | and RC, RC, #255
3517 | // RA = base, RB = (nresults+1), RC = numparams
3518 | ldr TMP1, [BASE, FRAME_PC]
3519 | add RC, BASE, RC, lsl #3
3520 | add RA, BASE, RA, lsl #3
3521 | add RC, RC, #FRAME_VARG
3522 | add TMP2, RA, RB, lsl #3
3523 | sub RC, RC, TMP1 // RC = vbase
3524 | // Note: RC may now be even _above_ BASE if nargs was < numparams.
3525 | sub TMP3, BASE, #16 // TMP3 = vtop
3526 | cbz RB, >5
3527 | sub TMP2, TMP2, #16
3528 |1: // Copy vararg slots to destination slots.
3529 | cmp RC, TMP3
3530 | ldr TMP0, [RC], #8
3531 | csel TMP0, TMP0, TISNIL, lo
3532 | cmp RA, TMP2
3533 | str TMP0, [RA], #8
3534 | blo <1
3535 |2:
3536 | ins_next
3537 |
3538 |5: // Copy all varargs.
3539 | ldr TMP0, L->maxstack
3540 | subs TMP2, TMP3, RC
3541 | csel RB, xzr, TMP2, le // MULTRES = (max(vtop-vbase,0)+1)*8
3542 | add RB, RB, #8
3543 | add TMP1, RA, TMP2
3544 | str RBw, SAVE_MULTRES
3545 | ble <2 // Nothing to copy.
3546 | cmp TMP1, TMP0
3547 | bhi >7
3548 |6:
3549 | ldr TMP0, [RC], #8
3550 | str TMP0, [RA], #8
3551 | cmp RC, TMP3
3552 | blo <6
3553 | b <2
3554 |
3555 |7: // Grow stack for varargs.
3556 | lsr CARG2, TMP2, #3
3557 | stp BASE, RA, L->base
3558 | mov CARG1, L
3559 | sub RC, RC, BASE // Need delta, because BASE may change.
3560 | str PC, SAVE_PC
3561 | bl extern lj_state_growstack // (lua_State *L, int n)
3562 | ldp BASE, RA, L->base
3563 | add RC, BASE, RC
3564 | sub TMP3, BASE, #16
3565 | b <6
3566 break;
3567
3568 /* -- Returns ----------------------------------------------------------- */
3569
3570 case BC_RETM:
3571 | // RA = results, RC = extra results
3572 | ldr TMP0w, SAVE_MULTRES
3573 | ldr PC, [BASE, FRAME_PC]
3574 | add RA, BASE, RA, lsl #3
3575 | add RC, TMP0, RC, lsl #3
3576 | b ->BC_RETM_Z
3577 break;
3578
3579 case BC_RET:
3580 | // RA = results, RC = nresults+1
3581 | ldr PC, [BASE, FRAME_PC]
3582 | lsl RC, RC, #3
3583 | add RA, BASE, RA, lsl #3
3584 |->BC_RETM_Z:
3585 | str RCw, SAVE_MULTRES
3586 |1:
3587 | ands CARG1, PC, #FRAME_TYPE
3588 | eor CARG2, PC, #FRAME_VARG
3589 | bne ->BC_RETV2_Z
3590 |
3591 |->BC_RET_Z:
3592 | // BASE = base, RA = resultptr, RC = (nresults+1)*8, PC = return
3593 | ldr INSw, [PC, #-4]
3594 | subs TMP1, RC, #8
3595 | sub CARG3, BASE, #16
3596 | beq >3
3597 |2:
3598 | ldr TMP0, [RA], #8
3599 | add BASE, BASE, #8
3600 | sub TMP1, TMP1, #8
3601 | str TMP0, [BASE, #-24]
3602 | cbnz TMP1, <2
3603 |3:
3604 | decode_RA RA, INS
3605 | sub CARG4, CARG3, RA, lsl #3
3606 | decode_RB RB, INS
3607 | ldr LFUNC:CARG1, [CARG4, FRAME_FUNC]
3608 |5:
3609 | cmp RC, RB, lsl #3 // More results expected?
3610 | blo >6
3611 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3612 | mov BASE, CARG4
3613 | ldr CARG2, LFUNC:CARG1->pc
3614 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3615 | ins_next
3616 |
3617 |6: // Fill up results with nil.
3618 | add BASE, BASE, #8
3619 | add RC, RC, #8
3620 | str TISNIL, [BASE, #-24]
3621 | b <5
3622 |
3623 |->BC_RETV1_Z: // Non-standard return case.
3624 | add RA, BASE, RA, lsl #3
3625 |->BC_RETV2_Z:
3626 | tst CARG2, #FRAME_TYPEP
3627 | bne ->vm_return
3628 | // Return from vararg function: relocate BASE down.
3629 | sub BASE, BASE, CARG2
3630 | ldr PC, [BASE, FRAME_PC]
3631 | b <1
3632 break;
3633
3634 case BC_RET0: case BC_RET1:
3635 | // RA = results, RC = nresults+1
3636 | ldr PC, [BASE, FRAME_PC]
3637 | lsl RC, RC, #3
3638 | str RCw, SAVE_MULTRES
3639 | ands CARG1, PC, #FRAME_TYPE
3640 | eor CARG2, PC, #FRAME_VARG
3641 | bne ->BC_RETV1_Z
3642 | ldr INSw, [PC, #-4]
3643 if (op == BC_RET1) {
3644 | ldr TMP0, [BASE, RA, lsl #3]
3645 }
3646 | sub CARG4, BASE, #16
3647 | decode_RA RA, INS
3648 | sub BASE, CARG4, RA, lsl #3
3649 if (op == BC_RET1) {
3650 | str TMP0, [CARG4], #8
3651 }
3652 | decode_RB RB, INS
3653 | ldr LFUNC:CARG1, [BASE, FRAME_FUNC]
3654 |5:
3655 | cmp RC, RB, lsl #3
3656 | blo >6
3657 | and LFUNC:CARG1, CARG1, #LJ_GCVMASK
3658 | ldr CARG2, LFUNC:CARG1->pc
3659 | ldr KBASE, [CARG2, #PC2PROTO(k)]
3660 | ins_next
3661 |
3662 |6: // Fill up results with nil.
3663 | add RC, RC, #8
3664 | str TISNIL, [CARG4], #8
3665 | b <5
3666 break;
3667
3668 /* -- Loops and branches ------------------------------------------------ */
3669
3670 |.define FOR_IDX, [RA]; .define FOR_TIDX, [RA, #4]
3671 |.define FOR_STOP, [RA, #8]; .define FOR_TSTOP, [RA, #12]
3672 |.define FOR_STEP, [RA, #16]; .define FOR_TSTEP, [RA, #20]
3673 |.define FOR_EXT, [RA, #24]; .define FOR_TEXT, [RA, #28]
3674
3675 case BC_FORL:
3676 |.if JIT
3677 | hotloop
3678 |.endif
3679 | // Fall through. Assumes BC_IFORL follows.
3680 break;
3681
3682 case BC_JFORI:
3683 case BC_JFORL:
3684 #if !LJ_HASJIT
3685 break;
3686 #endif
3687 case BC_FORI:
3688 case BC_IFORL:
3689 | // RA = base, RC = target (after end of loop or start of loop)
3690 vk = (op == BC_IFORL || op == BC_JFORL);
3691 | add RA, BASE, RA, lsl #3
3692 | ldp CARG1, CARG2, FOR_IDX // CARG1 = IDX, CARG2 = STOP
3693 | ldr CARG3, FOR_STEP // CARG3 = STEP
3694 if (op != BC_JFORL) {
3695 | add RC, PC, RC, lsl #2
3696 | sub RC, RC, #0x20000
3697 }
3698 | checkint CARG1, >5
3699 if (!vk) {
3700 | checkint CARG2, ->vmeta_for
3701 | checkint CARG3, ->vmeta_for
3702 | tbnz CARG3w, #31, >4
3703 | cmp CARG1w, CARG2w
3704 } else {
3705 | adds CARG1w, CARG1w, CARG3w
3706 | bvs >2
3707 | add TMP0, CARG1, TISNUM
3708 | tbnz CARG3w, #31, >4
3709 | cmp CARG1w, CARG2w
3710 }
3711 |1:
3712 if (op == BC_FORI) {
3713 | csel PC, RC, PC, gt
3714 } else if (op == BC_JFORI) {
3715 | mov PC, RC
3716 | ldrh RCw, [RC, #-4+OFS_RD]
3717 } else if (op == BC_IFORL) {
3718 | csel PC, RC, PC, le
3719 }
3720 if (vk) {
3721 | str TMP0, FOR_IDX
3722 | str TMP0, FOR_EXT
3723 } else {
3724 | str CARG1, FOR_EXT
3725 }
3726 if (op == BC_JFORI || op == BC_JFORL) {
3727 | ble =>BC_JLOOP
3728 }
3729 |2:
3730 | ins_next
3731 |
3732 |4: // Invert check for negative step.
3733 | cmp CARG2w, CARG1w
3734 | b <1
3735 |
3736 |5: // FP loop.
3737 | ldp d0, d1, FOR_IDX
3738 | blo ->vmeta_for
3739 if (!vk) {
3740 | checknum CARG2, ->vmeta_for
3741 | checknum CARG3, ->vmeta_for
3742 | str d0, FOR_EXT
3743 } else {
3744 | ldr d2, FOR_STEP
3745 | fadd d0, d0, d2
3746 }
3747 | tbnz CARG3, #63, >7
3748 | fcmp d0, d1
3749 |6:
3750 if (vk) {
3751 | str d0, FOR_IDX
3752 | str d0, FOR_EXT
3753 }
3754 if (op == BC_FORI) {
3755 | csel PC, RC, PC, hi
3756 } else if (op == BC_JFORI) {
3757 | ldrh RCw, [RC, #-4+OFS_RD]
3758 | bls =>BC_JLOOP
3759 } else if (op == BC_IFORL) {
3760 | csel PC, RC, PC, ls
3761 } else {
3762 | bls =>BC_JLOOP
3763 }
3764 | b <2
3765 |
3766 |7: // Invert check for negative step.
3767 | fcmp d1, d0
3768 | b <6
3769 break;
3770
3771 case BC_ITERL:
3772 |.if JIT
3773 | hotloop
3774 |.endif
3775 | // Fall through. Assumes BC_IITERL follows.
3776 break;
3777
3778 case BC_JITERL:
3779 #if !LJ_HASJIT
3780 break;
3781 #endif
3782 case BC_IITERL:
3783 | // RA = base, RC = target
3784 | ldr CARG1, [BASE, RA, lsl #3]
3785 | add TMP1, BASE, RA, lsl #3
3786 | cmp CARG1, TISNIL
3787 | beq >1 // Stop if iterator returned nil.
3788 if (op == BC_JITERL) {
3789 | str CARG1, [TMP1, #-8]
3790 | b =>BC_JLOOP
3791 } else {
3792 | add TMP0, PC, RC, lsl #2 // Otherwise save control var + branch.
3793 | sub PC, TMP0, #0x20000
3794 | str CARG1, [TMP1, #-8]
3795 }
3796 |1:
3797 | ins_next
3798 break;
3799
3800 case BC_LOOP:
3801 | // RA = base, RC = target (loop extent)
3802 | // Note: RA/RC is only used by trace recorder to determine scope/extent
3803 | // This opcode does NOT jump, it's only purpose is to detect a hot loop.
3804 |.if JIT
3805 | hotloop
3806 |.endif
3807 | // Fall through. Assumes BC_ILOOP follows.
3808 break;
3809
3810 case BC_ILOOP:
3811 | // RA = base, RC = target (loop extent)
3812 | ins_next
3813 break;
3814
3815 case BC_JLOOP:
3816 |.if JIT
3817 | // RA = base (ignored), RC = traceno
3818 | ldr CARG1, [GL, #GL_J(trace)]
3819 | mov CARG2w, #0 // Traces on ARM64 don't store the trace #, so use 0.
3820 | ldr TRACE:RC, [CARG1, RC, lsl #3]
3821 | st_vmstate CARG2w
3822 |.if PAUTH
3823 | ldr RA, TRACE:RC->mcauth
3824 |.else
3825 | ldr RA, TRACE:RC->mcode
3826 |.endif
3827 | str BASE, GL->jit_base
3828 | str L, GL->tmpbuf.L
3829 | sub sp, sp, #16 // See SPS_FIXED. Avoids sp adjust in every root trace.
3830 |.if PAUTH
3831 | braa RA, RC
3832 |.else
3833 | br RA
3834 |.endif
3835 |.endif
3836 break;
3837
3838 case BC_JMP:
3839 | // RA = base (only used by trace recorder), RC = target
3840 | add RC, PC, RC, lsl #2
3841 | sub PC, RC, #0x20000
3842 | ins_next
3843 break;
3844
3845 /* -- Function headers -------------------------------------------------- */
3846
3847 case BC_FUNCF:
3848 |.if JIT
3849 | hotcall
3850 |.endif
3851 case BC_FUNCV: /* NYI: compiled vararg functions. */
3852 | // Fall through. Assumes BC_IFUNCF/BC_IFUNCV follow.
3853 break;
3854
3855 case BC_JFUNCF:
3856 #if !LJ_HASJIT
3857 break;
3858 #endif
3859 case BC_IFUNCF:
3860 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3861 | ldr CARG1, L->maxstack
3862 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3863 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3864 | cmp RA, CARG1
3865 | bhi ->vm_growstack_l
3866 |2:
3867 | cmp NARGS8:RC, TMP1, lsl #3 // Check for missing parameters.
3868 | blo >3
3869 if (op == BC_JFUNCF) {
3870 | decode_RD RC, INS
3871 | b =>BC_JLOOP
3872 } else {
3873 | ins_next
3874 }
3875 |
3876 |3: // Clear missing parameters.
3877 | str TISNIL, [BASE, NARGS8:RC]
3878 | add NARGS8:RC, NARGS8:RC, #8
3879 | b <2
3880 break;
3881
3882 case BC_JFUNCV:
3883 #if !LJ_HASJIT
3884 break;
3885 #endif
3886 | NYI // NYI: compiled vararg functions
3887 break; /* NYI: compiled vararg functions. */
3888
3889 case BC_IFUNCV:
3890 | // BASE = new base, RA = BASE+framesize*8, CARG3 = LFUNC, RC = nargs*8
3891 | ldr CARG1, L->maxstack
3892 | movn TMP0, #~LJ_TFUNC
3893 | add TMP2, BASE, RC
3894 | add LFUNC:CARG3, CARG3, TMP0, lsl #47
3895 | add RA, RA, RC
3896 | add TMP0, RC, #16+FRAME_VARG
3897 | str LFUNC:CARG3, [TMP2], #8 // Store (tagged) copy of LFUNC.
3898 | ldr KBASE, [PC, #-4+PC2PROTO(k)]
3899 | cmp RA, CARG1
3900 | str TMP0, [TMP2], #8 // Store delta + FRAME_VARG.
3901 | bhs ->vm_growstack_l
3902 | sub RC, TMP2, #16
3903 | ldrb TMP1w, [PC, #-4+PC2PROTO(numparams)]
3904 | mov RA, BASE
3905 | mov BASE, TMP2
3906 | cbz TMP1, >2
3907 |1:
3908 | cmp RA, RC // Less args than parameters?
3909 | bhs >3
3910 | ldr TMP0, [RA]
3911 | sub TMP1, TMP1, #1
3912 | str TISNIL, [RA], #8 // Clear old fixarg slot (help the GC).
3913 | str TMP0, [TMP2], #8
3914 | cbnz TMP1, <1
3915 |2:
3916 | ins_next
3917 |
3918 |3:
3919 | sub TMP1, TMP1, #1
3920 | str TISNIL, [TMP2], #8
3921 | cbz TMP1, <2
3922 | b <3
3923 break;
3924
3925 case BC_FUNCC:
3926 case BC_FUNCCW:
3927 | // BASE = new base, RA = BASE+framesize*8, CARG3 = CFUNC, RC = nargs*8
3928 if (op == BC_FUNCC) {
3929 | ldr CARG4, CFUNC:CARG3->f
3930 } else {
3931 | ldr CARG4, GL->wrapf
3932 }
3933 | add CARG2, RA, NARGS8:RC
3934 | ldr CARG1, L->maxstack
3935 | add RC, BASE, NARGS8:RC
3936 | cmp CARG2, CARG1
3937 | stp BASE, RC, L->base
3938 if (op == BC_FUNCCW) {
3939 | ldr CARG2, CFUNC:CARG3->f
3940 }
3941 | mv_vmstate TMP0w, C
3942 | mov CARG1, L
3943 | bhi ->vm_growstack_c // Need to grow stack.
3944 | st_vmstate TMP0w
3945 | blr_auth CARG4 // (lua_State *L [, lua_CFunction f])
3946 | // Returns nresults.
3947 | ldp BASE, TMP1, L->base
3948 | str L, GL->cur_L
3949 | sbfiz RC, CRET1, #3, #32
3950 | st_vmstate ST_INTERP
3951 | ldr PC, [BASE, FRAME_PC]
3952 | sub RA, TMP1, RC // RA = L->top - nresults*8
3953 | b ->vm_returnc
3954 break;
3955
3956 /* ---------------------------------------------------------------------- */
3957
3958 default:
3959 fprintf(stderr, "Error: undefined opcode BC_%s\n", bc_names[op]);
3960 exit(2);
3961 break;
3962 }
3963 }
3964
3965 static int build_backend(BuildCtx *ctx)
3966 {
3967 int op;
3968
3969 dasm_growpc(Dst, BC__MAX);
3970
3971 build_subroutines(ctx);
3972
3973 |.code_op
3974 for (op = 0; op < BC__MAX; op++)
3975 build_ins(ctx, (BCOp)op, op);
3976
3977 return BC__MAX;
3978 }
3979
3980 /* Emit pseudo frame-info for all assembler functions. */
3981 static void emit_asm_debug(BuildCtx *ctx)
3982 {
3983 int fcofs = (int)((uint8_t *)ctx->glob[GLOB_vm_ffi_call] - ctx->code);
3984 int i;
3985 switch (ctx->mode) {
3986 case BUILD_elfasm:
3987 fprintf(ctx->fp, "\t.section .debug_frame,\"\",%%progbits\n");
3988 fprintf(ctx->fp,
3989 ".Lframe0:\n"
3990 "\t.long .LECIE0-.LSCIE0\n"
3991 ".LSCIE0:\n"
3992 "\t.long 0xffffffff\n"
3993 "\t.byte 0x1\n"
3994 "\t.string \"\"\n"
3995 "\t.uleb128 0x1\n"
3996 "\t.sleb128 -8\n"
3997 "\t.byte 30\n" /* Return address is in lr. */
3998 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
3999 "\t.align 3\n"
4000 ".LECIE0:\n\n");
4001 fprintf(ctx->fp,
4002 ".LSFDE0:\n"
4003 "\t.long .LEFDE0-.LASFDE0\n"
4004 ".LASFDE0:\n"
4005 "\t.long .Lframe0\n"
4006 "\t.quad .Lbegin\n"
4007 "\t.quad %d\n"
4008 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4009 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4010 fcofs);
4011 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4012 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4013 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4014 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4015 64+i, i+(3+(28-19+1)-8));
4016 fprintf(ctx->fp,
4017 "\t.align 3\n"
4018 ".LEFDE0:\n\n");
4019 #if LJ_HASFFI
4020 fprintf(ctx->fp,
4021 ".LSFDE1:\n"
4022 "\t.long .LEFDE1-.LASFDE1\n"
4023 ".LASFDE1:\n"
4024 "\t.long .Lframe0\n"
4025 "\t.quad lj_vm_ffi_call\n"
4026 "\t.quad %d\n"
4027 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4028 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4029 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4030 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4031 "\t.align 3\n"
4032 ".LEFDE1:\n\n", (int)ctx->codesz - fcofs);
4033 #endif
4034 #if !LJ_NO_UNWIND
4035 fprintf(ctx->fp, "\t.section .eh_frame,\"a\",%%progbits\n");
4036 fprintf(ctx->fp,
4037 ".Lframe1:\n"
4038 "\t.long .LECIE1-.LSCIE1\n"
4039 ".LSCIE1:\n"
4040 "\t.long 0\n"
4041 "\t.byte 0x1\n"
4042 "\t.string \"zPR\"\n"
4043 "\t.uleb128 0x1\n"
4044 "\t.sleb128 -8\n"
4045 "\t.byte 30\n" /* Return address is in lr. */
4046 "\t.uleb128 6\n" /* augmentation length */
4047 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4048 "\t.long lj_err_unwind_dwarf-.\n"
4049 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4050 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4051 "\t.align 3\n"
4052 ".LECIE1:\n\n");
4053 fprintf(ctx->fp,
4054 ".LSFDE2:\n"
4055 "\t.long .LEFDE2-.LASFDE2\n"
4056 ".LASFDE2:\n"
4057 "\t.long .LASFDE2-.Lframe1\n"
4058 "\t.long .Lbegin-.\n"
4059 "\t.long %d\n"
4060 "\t.uleb128 0\n" /* augmentation length */
4061 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4062 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4063 fcofs);
4064 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4065 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4066 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4067 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4068 64+i, i+(3+(28-19+1)-8));
4069 fprintf(ctx->fp,
4070 "\t.align 3\n"
4071 ".LEFDE2:\n\n");
4072 #if LJ_HASFFI
4073 fprintf(ctx->fp,
4074 ".Lframe2:\n"
4075 "\t.long .LECIE2-.LSCIE2\n"
4076 ".LSCIE2:\n"
4077 "\t.long 0\n"
4078 "\t.byte 0x1\n"
4079 "\t.string \"zR\"\n"
4080 "\t.uleb128 0x1\n"
4081 "\t.sleb128 -8\n"
4082 "\t.byte 30\n" /* Return address is in lr. */
4083 "\t.uleb128 1\n" /* augmentation length */
4084 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4085 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4086 "\t.align 3\n"
4087 ".LECIE2:\n\n");
4088 fprintf(ctx->fp,
4089 ".LSFDE3:\n"
4090 "\t.long .LEFDE3-.LASFDE3\n"
4091 ".LASFDE3:\n"
4092 "\t.long .LASFDE3-.Lframe2\n"
4093 "\t.long lj_vm_ffi_call-.\n"
4094 "\t.long %d\n"
4095 "\t.uleb128 0\n" /* augmentation length */
4096 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4097 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4098 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4099 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4100 "\t.align 3\n"
4101 ".LEFDE3:\n\n", (int)ctx->codesz - fcofs);
4102 #endif
4103 #endif
4104 break;
4105 #if !LJ_NO_UNWIND
4106 case BUILD_machasm: {
4107 #if LJ_HASFFI
4108 int fcsize = 0;
4109 #endif
4110 int j;
4111 fprintf(ctx->fp, "\t.section __TEXT,__eh_frame,coalesced,no_toc+strip_static_syms+live_support\n");
4112 fprintf(ctx->fp,
4113 "EH_frame1:\n"
4114 "\t.set L$set$x,LECIEX-LSCIEX\n"
4115 "\t.long L$set$x\n"
4116 "LSCIEX:\n"
4117 "\t.long 0\n"
4118 "\t.byte 0x1\n"
4119 "\t.ascii \"zPR\\0\"\n"
4120 "\t.uleb128 0x1\n"
4121 "\t.sleb128 -8\n"
4122 "\t.byte 30\n" /* Return address is in lr. */
4123 "\t.uleb128 6\n" /* augmentation length */
4124 "\t.byte 0x9b\n" /* indirect|pcrel|sdata4 */
4125 "\t.long _lj_err_unwind_dwarf@GOT-.\n"
4126 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4127 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4128 "\t.align 3\n"
4129 "LECIEX:\n\n");
4130 for (j = 0; j < ctx->nsym; j++) {
4131 const char *name = ctx->sym[j].name;
4132 int32_t size = ctx->sym[j+1].ofs - ctx->sym[j].ofs;
4133 if (size == 0) continue;
4134 #if LJ_HASFFI
4135 if (!strcmp(name, "_lj_vm_ffi_call")) { fcsize = size; continue; }
4136 #endif
4137 fprintf(ctx->fp,
4138 "LSFDE%d:\n"
4139 "\t.set L$set$%d,LEFDE%d-LASFDE%d\n"
4140 "\t.long L$set$%d\n"
4141 "LASFDE%d:\n"
4142 "\t.long LASFDE%d-EH_frame1\n"
4143 "\t.long %s-.\n"
4144 "\t.long %d\n"
4145 "\t.uleb128 0\n" /* augmentation length */
4146 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4147 "\t.byte 0x9d\n\t.uleb128 2\n", /* offset fp */
4148 j, j, j, j, j, j, j, name, size);
4149 for (i = 19; i <= 28; i++) /* offset x19-x28 */
4150 fprintf(ctx->fp, "\t.byte 0x%x\n\t.uleb128 %d\n", 0x80+i, i+(3-19));
4151 for (i = 8; i <= 15; i++) /* offset d8-d15 */
4152 fprintf(ctx->fp, "\t.byte 5\n\t.uleb128 0x%x\n\t.uleb128 %d\n",
4153 64+i, i+(3+(28-19+1)-8));
4154 fprintf(ctx->fp,
4155 "\t.align 3\n"
4156 "LEFDE%d:\n\n", j);
4157 }
4158 #if LJ_HASFFI
4159 if (fcsize) {
4160 fprintf(ctx->fp,
4161 "EH_frame2:\n"
4162 "\t.set L$set$y,LECIEY-LSCIEY\n"
4163 "\t.long L$set$y\n"
4164 "LSCIEY:\n"
4165 "\t.long 0\n"
4166 "\t.byte 0x1\n"
4167 "\t.ascii \"zR\\0\"\n"
4168 "\t.uleb128 0x1\n"
4169 "\t.sleb128 -8\n"
4170 "\t.byte 30\n" /* Return address is in lr. */
4171 "\t.uleb128 1\n" /* augmentation length */
4172 "\t.byte 0x1b\n" /* pcrel|sdata4 */
4173 "\t.byte 0xc\n\t.uleb128 29\n\t.uleb128 16\n" /* def_cfa fp 16 */
4174 "\t.align 3\n"
4175 "LECIEY:\n\n");
4176 fprintf(ctx->fp,
4177 "LSFDEY:\n"
4178 "\t.set L$set$yy,LEFDEY-LASFDEY\n"
4179 "\t.long L$set$yy\n"
4180 "LASFDEY:\n"
4181 "\t.long LASFDEY-EH_frame2\n"
4182 "\t.long _lj_vm_ffi_call-.\n"
4183 "\t.long %d\n"
4184 "\t.uleb128 0\n" /* augmentation length */
4185 "\t.byte 0x9e\n\t.uleb128 1\n" /* offset lr */
4186 "\t.byte 0x9d\n\t.uleb128 2\n" /* offset fp */
4187 "\t.byte 0x93\n\t.uleb128 3\n" /* offset x19 */
4188 "\t.byte 0x94\n\t.uleb128 4\n" /* offset x20 */
4189 "\t.align 3\n"
4190 "LEFDEY:\n\n", fcsize);
4191 }
4192 #endif
4193 fprintf(ctx->fp, ".subsections_via_symbols\n");
4194 }
4195 break;
4196 #endif
4197 default:
4198 break;
4199 }
4200 }
4201