Mercurial

comparison third_party/luajit/src/lj_ccallback.c @ 178:94705b5986b3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[ThirdParty] Added WRK and luajit for load testing.
author MrJuneJune <me@mrjunejune.com>
date Thu, 22 Jan 2026 20:10:30 -0800
parents
children
comparison
equal deleted inserted replaced
177:24fe8ff94056 178:94705b5986b3
1 /*
2 ** FFI C callback handling.
3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 #include "lj_obj.h"
7
8 #if LJ_HASFFI
9
10 #include "lj_gc.h"
11 #include "lj_err.h"
12 #include "lj_tab.h"
13 #include "lj_state.h"
14 #include "lj_frame.h"
15 #include "lj_ctype.h"
16 #include "lj_cconv.h"
17 #include "lj_ccall.h"
18 #include "lj_ccallback.h"
19 #include "lj_target.h"
20 #include "lj_mcode.h"
21 #include "lj_trace.h"
22 #include "lj_vm.h"
23
24 /* -- Target-specific handling of callback slots -------------------------- */
25
26 #define CALLBACK_MCODE_SIZE (LJ_PAGESIZE * LJ_NUM_CBPAGE)
27
28 #if LJ_OS_NOJIT
29
30 /* Callbacks disabled. */
31 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
32 #define CALLBACK_OFS2SLOT(ofs) (0*(ofs))
33 #define CALLBACK_MAX_SLOT 0
34
35 #elif LJ_TARGET_X86ORX64
36
37 #define CALLBACK_MCODE_HEAD (LJ_64 ? 8 : 0)
38 #define CALLBACK_MCODE_GROUP (-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5))
39
40 #define CALLBACK_SLOT2OFS(slot) \
41 (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
42
43 static MSize CALLBACK_OFS2SLOT(MSize ofs)
44 {
45 MSize group;
46 ofs -= CALLBACK_MCODE_HEAD;
47 group = ofs / (32*4 + CALLBACK_MCODE_GROUP);
48 return (ofs % (32*4 + CALLBACK_MCODE_GROUP))/4 + group*32;
49 }
50
51 #define CALLBACK_MAX_SLOT \
52 (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32)
53
54 #elif LJ_TARGET_ARM
55
56 #define CALLBACK_MCODE_HEAD 32
57
58 #elif LJ_TARGET_ARM64
59
60 #define CALLBACK_MCODE_HEAD 32
61
62 #elif LJ_TARGET_PPC
63
64 #define CALLBACK_MCODE_HEAD 24
65
66 #elif LJ_TARGET_MIPS32
67
68 #define CALLBACK_MCODE_HEAD 20
69
70 #elif LJ_TARGET_MIPS64
71
72 #define CALLBACK_MCODE_HEAD 52
73
74 #else
75
76 /* Missing support for this architecture. */
77 #define CALLBACK_SLOT2OFS(slot) (0*(slot))
78 #define CALLBACK_OFS2SLOT(ofs) (0*(ofs))
79 #define CALLBACK_MAX_SLOT 0
80
81 #endif
82
83 #ifndef CALLBACK_SLOT2OFS
84 #define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot))
85 #define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8)
86 #define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
87 #endif
88
89 /* Convert callback slot number to callback function pointer. */
90 static void *callback_slot2ptr(CTState *cts, MSize slot)
91 {
92 return (uint8_t *)cts->cb.mcode + CALLBACK_SLOT2OFS(slot);
93 }
94
95 /* Convert callback function pointer to slot number. */
96 MSize lj_ccallback_ptr2slot(CTState *cts, void *p)
97 {
98 uintptr_t ofs = (uintptr_t)((uint8_t *)p -(uint8_t *)cts->cb.mcode);
99 if (ofs < CALLBACK_MCODE_SIZE) {
100 MSize slot = CALLBACK_OFS2SLOT((MSize)ofs);
101 if (CALLBACK_SLOT2OFS(slot) == (MSize)ofs)
102 return slot;
103 }
104 return ~0u; /* Not a known callback function pointer. */
105 }
106
107 /* Initialize machine code for callback function pointers. */
108 #if LJ_OS_NOJIT
109 /* Disabled callback support. */
110 #define callback_mcode_init(g, p) (p)
111 #elif LJ_TARGET_X86ORX64
112 static void *callback_mcode_init(global_State *g, uint8_t *page)
113 {
114 uint8_t *p = page;
115 uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback;
116 MSize slot;
117 #if LJ_64
118 *(void **)p = target; p += 8;
119 #endif
120 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
121 /* mov al, slot; jmp group */
122 *p++ = XI_MOVrib | RID_EAX; *p++ = (uint8_t)slot;
123 if ((slot & 31) == 31 || slot == CALLBACK_MAX_SLOT-1) {
124 /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
125 *p++ = XI_PUSH + RID_EBP;
126 *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8);
127 #if LJ_GC64
128 *p++ = 0x48; *p++ = XI_MOVri | RID_EBP;
129 *(uint64_t *)p = (uint64_t)(g); p += 8;
130 #else
131 *p++ = XI_MOVri | RID_EBP;
132 *(int32_t *)p = i32ptr(g); p += 4;
133 #endif
134 #if LJ_64
135 /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
136 *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP;
137 *(int32_t *)p = (int32_t)(page-(p+4)); p += 4;
138 #else
139 /* jmp lj_vm_ffi_callback. */
140 *p++ = XI_JMP; *(int32_t *)p = target-(p+4); p += 4;
141 #endif
142 } else {
143 *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2);
144 }
145 }
146 return p;
147 }
148 #elif LJ_TARGET_ARM
149 static void *callback_mcode_init(global_State *g, uint32_t *page)
150 {
151 uint32_t *p = page;
152 void *target = (void *)lj_vm_ffi_callback;
153 MSize slot;
154 /* This must match with the saveregs macro in buildvm_arm.dasc. */
155 *p++ = ARMI_SUB|ARMF_D(RID_R12)|ARMF_N(RID_R12)|ARMF_M(RID_PC);
156 *p++ = ARMI_PUSH|ARMF_N(RID_SP)|RSET_RANGE(RID_R4,RID_R11+1)|RID2RSET(RID_LR);
157 *p++ = ARMI_SUB|ARMI_K12|ARMF_D(RID_R12)|ARMF_N(RID_R12)|CALLBACK_MCODE_HEAD;
158 *p++ = ARMI_STR|ARMI_LS_P|ARMI_LS_W|ARMF_D(RID_R12)|ARMF_N(RID_SP)|(CFRAME_SIZE-4*9);
159 *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_R12)|ARMF_N(RID_PC);
160 *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_PC)|ARMF_N(RID_PC);
161 *p++ = u32ptr(g);
162 *p++ = u32ptr(target);
163 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
164 *p++ = ARMI_MOV|ARMF_D(RID_R12)|ARMF_M(RID_PC);
165 *p = ARMI_B | ((page-p-2) & 0x00ffffffu);
166 p++;
167 }
168 return p;
169 }
170 #elif LJ_TARGET_ARM64
171 static void *callback_mcode_init(global_State *g, uint32_t *page)
172 {
173 uint32_t *p = page;
174 ASMFunction target = lj_vm_ffi_callback;
175 MSize slot;
176 *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X11) | A64F_S19(4));
177 *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X10) | A64F_S19(5));
178 *p++ = A64I_LE(A64I_BR_AUTH | A64F_N(RID_X11));
179 *p++ = A64I_LE(A64I_NOP);
180 ((ASMFunction *)p)[0] = target;
181 ((void **)p)[1] = g;
182 p += 4;
183 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
184 *p++ = A64I_LE(A64I_MOVZw | A64F_D(RID_X9) | A64F_U16(slot));
185 *p = A64I_LE(A64I_B | A64F_S26((page-p) & 0x03ffffffu));
186 p++;
187 }
188 return p;
189 }
190 #elif LJ_TARGET_PPC
191 static void *callback_mcode_init(global_State *g, uint32_t *page)
192 {
193 uint32_t *p = page;
194 void *target = (void *)lj_vm_ffi_callback;
195 MSize slot;
196 *p++ = PPCI_LIS | PPCF_T(RID_TMP) | (u32ptr(target) >> 16);
197 *p++ = PPCI_LIS | PPCF_T(RID_R12) | (u32ptr(g) >> 16);
198 *p++ = PPCI_ORI | PPCF_A(RID_TMP)|PPCF_T(RID_TMP) | (u32ptr(target) & 0xffff);
199 *p++ = PPCI_ORI | PPCF_A(RID_R12)|PPCF_T(RID_R12) | (u32ptr(g) & 0xffff);
200 *p++ = PPCI_MTCTR | PPCF_T(RID_TMP);
201 *p++ = PPCI_BCTR;
202 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
203 *p++ = PPCI_LI | PPCF_T(RID_R11) | slot;
204 *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2);
205 p++;
206 }
207 return p;
208 }
209 #elif LJ_TARGET_MIPS
210 static void *callback_mcode_init(global_State *g, uint32_t *page)
211 {
212 uint32_t *p = page;
213 uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback;
214 uintptr_t ug = (uintptr_t)(void *)g;
215 MSize slot;
216 #if LJ_TARGET_MIPS32
217 *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 16);
218 *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 16);
219 #else
220 *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 48);
221 *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 48);
222 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 32) & 0xffff);
223 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 32) & 0xffff);
224 *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
225 *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
226 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 16) & 0xffff);
227 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 16) & 0xffff);
228 *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
229 *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
230 #endif
231 *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | (target & 0xffff);
232 *p++ = MIPSI_JR | MIPSF_S(RID_R3);
233 *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (ug & 0xffff);
234 for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
235 *p = MIPSI_B | ((page-p-1) & 0x0000ffffu);
236 p++;
237 *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot;
238 }
239 return p;
240 }
241 #else
242 /* Missing support for this architecture. */
243 #define callback_mcode_init(g, p) (p)
244 #endif
245
246 /* -- Machine code management --------------------------------------------- */
247
248 #if LJ_TARGET_WINDOWS
249
250 #define WIN32_LEAN_AND_MEAN
251 #include <windows.h>
252
253 #elif LJ_TARGET_POSIX
254
255 #include <sys/mman.h>
256 #ifndef MAP_ANONYMOUS
257 #define MAP_ANONYMOUS MAP_ANON
258 #endif
259 #ifdef PROT_MPROTECT
260 #define CCPROT_CREATE (PROT_MPROTECT(PROT_EXEC))
261 #else
262 #define CCPROT_CREATE 0
263 #endif
264
265 #endif
266
267 /* Allocate and initialize area for callback function pointers. */
268 static void callback_mcode_new(CTState *cts)
269 {
270 size_t sz = (size_t)CALLBACK_MCODE_SIZE;
271 void *p, *pe;
272 if (CALLBACK_MAX_SLOT == 0)
273 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
274 #if LJ_TARGET_WINDOWS
275 p = LJ_WIN_VALLOC(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
276 if (!p)
277 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
278 #elif LJ_TARGET_POSIX
279 p = mmap(NULL, sz, (PROT_READ|PROT_WRITE|CCPROT_CREATE), MAP_PRIVATE|MAP_ANONYMOUS,
280 -1, 0);
281 if (p == MAP_FAILED)
282 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
283 #else
284 /* Fallback allocator. Fails if memory is not executable by default. */
285 p = lj_mem_new(cts->L, sz);
286 #endif
287 cts->cb.mcode = p;
288 pe = callback_mcode_init(cts->g, p);
289 UNUSED(pe);
290 lj_assertCTS((size_t)((char *)pe - (char *)p) <= sz,
291 "miscalculated CALLBACK_MAX_SLOT");
292 lj_mcode_sync(p, (char *)p + sz);
293 #if LJ_TARGET_WINDOWS
294 {
295 DWORD oprot;
296 LJ_WIN_VPROTECT(p, sz, PAGE_EXECUTE_READ, &oprot);
297 }
298 #elif LJ_TARGET_POSIX
299 mprotect(p, sz, (PROT_READ|PROT_EXEC));
300 #endif
301 }
302
303 /* Free area for callback function pointers. */
304 void lj_ccallback_mcode_free(CTState *cts)
305 {
306 size_t sz = (size_t)CALLBACK_MCODE_SIZE;
307 void *p = cts->cb.mcode;
308 if (p == NULL) return;
309 #if LJ_TARGET_WINDOWS
310 VirtualFree(p, 0, MEM_RELEASE);
311 UNUSED(sz);
312 #elif LJ_TARGET_POSIX
313 munmap(p, sz);
314 #else
315 lj_mem_free(cts->g, p, sz);
316 #endif
317 }
318
319 /* -- C callback entry ---------------------------------------------------- */
320
321 /* Target-specific handling of register arguments. Similar to lj_ccall.c. */
322 #if LJ_TARGET_X86
323
324 #define CALLBACK_HANDLE_REGARG \
325 if (!isfp) { /* Only non-FP values may be passed in registers. */ \
326 if (n > 1) { /* Anything > 32 bit is passed on the stack. */ \
327 if (!LJ_ABI_WIN) ngpr = maxgpr; /* Prevent reordering. */ \
328 } else if (ngpr + 1 <= maxgpr) { \
329 sp = &cts->cb.gpr[ngpr]; \
330 ngpr += n; \
331 goto done; \
332 } \
333 }
334
335 #elif LJ_TARGET_X64 && LJ_ABI_WIN
336
337 /* Windows/x64 argument registers are strictly positional (use ngpr). */
338 #define CALLBACK_HANDLE_REGARG \
339 if (isfp) { \
340 if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \
341 } else { \
342 if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \
343 }
344
345 #elif LJ_TARGET_X64
346
347 #define CALLBACK_HANDLE_REGARG \
348 if (isfp) { \
349 if (nfpr + n <= CCALL_NARG_FPR) { \
350 sp = &cts->cb.fpr[nfpr]; \
351 nfpr += n; \
352 goto done; \
353 } \
354 } else { \
355 if (ngpr + n <= maxgpr) { \
356 sp = &cts->cb.gpr[ngpr]; \
357 ngpr += n; \
358 goto done; \
359 } \
360 }
361
362 #elif LJ_TARGET_ARM
363
364 #if LJ_ABI_SOFTFP
365
366 #define CALLBACK_HANDLE_REGARG_FP1 UNUSED(isfp);
367 #define CALLBACK_HANDLE_REGARG_FP2
368
369 #else
370
371 #define CALLBACK_HANDLE_REGARG_FP1 \
372 if (isfp) { \
373 if (n == 1) { \
374 if (fprodd) { \
375 sp = &cts->cb.fpr[fprodd-1]; \
376 fprodd = 0; \
377 goto done; \
378 } else if (nfpr + 1 <= CCALL_NARG_FPR) { \
379 sp = &cts->cb.fpr[nfpr++]; \
380 fprodd = nfpr; \
381 goto done; \
382 } \
383 } else { \
384 if (nfpr + 1 <= CCALL_NARG_FPR) { \
385 sp = &cts->cb.fpr[nfpr++]; \
386 goto done; \
387 } \
388 } \
389 fprodd = 0; /* No reordering after the first FP value is on stack. */ \
390 } else {
391
392 #define CALLBACK_HANDLE_REGARG_FP2 }
393
394 #endif
395
396 #define CALLBACK_HANDLE_REGARG \
397 CALLBACK_HANDLE_REGARG_FP1 \
398 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
399 if (ngpr + n <= maxgpr) { \
400 sp = &cts->cb.gpr[ngpr]; \
401 ngpr += n; \
402 goto done; \
403 } CALLBACK_HANDLE_REGARG_FP2
404
405 #elif LJ_TARGET_ARM64
406
407 #define CALLBACK_HANDLE_REGARG \
408 if (isfp) { \
409 if (nfpr + n <= CCALL_NARG_FPR) { \
410 sp = &cts->cb.fpr[nfpr]; \
411 nfpr += n; \
412 goto done; \
413 } else { \
414 nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \
415 } \
416 } else { \
417 if (!LJ_TARGET_OSX && n > 1) \
418 ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
419 if (ngpr + n <= maxgpr) { \
420 sp = &cts->cb.gpr[ngpr]; \
421 ngpr += n; \
422 goto done; \
423 } else { \
424 ngpr = CCALL_NARG_GPR; /* Prevent reordering. */ \
425 } \
426 }
427
428 #elif LJ_TARGET_PPC
429
430 #define CALLBACK_HANDLE_GPR \
431 if (n > 1) { \
432 lj_assertCTS(((LJ_ABI_SOFTFP && ctype_isnum(cta->info)) || /* double. */ \
433 ctype_isinteger(cta->info)) && n == 2, /* int64_t. */ \
434 "bad GPR type"); \
435 ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \
436 } \
437 if (ngpr + n <= maxgpr) { \
438 sp = &cts->cb.gpr[ngpr]; \
439 ngpr += n; \
440 goto done; \
441 }
442
443 #if LJ_ABI_SOFTFP
444 #define CALLBACK_HANDLE_REGARG \
445 CALLBACK_HANDLE_GPR \
446 UNUSED(isfp);
447 #else
448 #define CALLBACK_HANDLE_REGARG \
449 if (isfp) { \
450 if (nfpr + 1 <= CCALL_NARG_FPR) { \
451 sp = &cts->cb.fpr[nfpr++]; \
452 cta = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \
453 goto done; \
454 } \
455 } else { /* Try to pass argument in GPRs. */ \
456 CALLBACK_HANDLE_GPR \
457 }
458 #endif
459
460 #if !LJ_ABI_SOFTFP
461 #define CALLBACK_HANDLE_RET \
462 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
463 *(double *)dp = *(float *)dp; /* FPRs always hold doubles. */
464 #endif
465
466 #elif LJ_TARGET_MIPS32
467
468 #define CALLBACK_HANDLE_GPR \
469 if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \
470 if (ngpr + n <= maxgpr) { \
471 sp = &cts->cb.gpr[ngpr]; \
472 ngpr += n; \
473 goto done; \
474 }
475
476 #if !LJ_ABI_SOFTFP /* MIPS32 hard-float */
477 #define CALLBACK_HANDLE_REGARG \
478 if (isfp && nfpr < CCALL_NARG_FPR) { /* Try to pass argument in FPRs. */ \
479 sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \
480 nfpr++; ngpr += n; \
481 goto done; \
482 } else { /* Try to pass argument in GPRs. */ \
483 nfpr = CCALL_NARG_FPR; \
484 CALLBACK_HANDLE_GPR \
485 }
486 #else /* MIPS32 soft-float */
487 #define CALLBACK_HANDLE_REGARG \
488 CALLBACK_HANDLE_GPR \
489 UNUSED(isfp);
490 #endif
491
492 #define CALLBACK_HANDLE_RET \
493 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
494 ((float *)dp)[1] = *(float *)dp;
495
496 #elif LJ_TARGET_MIPS64
497
498 #if !LJ_ABI_SOFTFP /* MIPS64 hard-float */
499 #define CALLBACK_HANDLE_REGARG \
500 if (ngpr + n <= maxgpr) { \
501 sp = isfp ? (void*) &cts->cb.fpr[ngpr] : (void*) &cts->cb.gpr[ngpr]; \
502 ngpr += n; \
503 goto done; \
504 }
505 #else /* MIPS64 soft-float */
506 #define CALLBACK_HANDLE_REGARG \
507 if (ngpr + n <= maxgpr) { \
508 UNUSED(isfp); \
509 sp = (void*) &cts->cb.gpr[ngpr]; \
510 ngpr += n; \
511 goto done; \
512 }
513 #endif
514
515 #define CALLBACK_HANDLE_RET \
516 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
517 ((float *)dp)[1] = *(float *)dp;
518
519 #else
520 #error "Missing calling convention definitions for this architecture"
521 #endif
522
523 /* Convert and push callback arguments to Lua stack. */
524 static void callback_conv_args(CTState *cts, lua_State *L)
525 {
526 TValue *o = L->top;
527 intptr_t *stack = cts->cb.stack;
528 MSize slot = cts->cb.slot;
529 CTypeID id = 0, rid, fid;
530 int gcsteps = 0;
531 CType *ct;
532 GCfunc *fn;
533 int fntp;
534 MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR;
535 #if CCALL_NARG_FPR
536 MSize nfpr = 0;
537 #if LJ_TARGET_ARM
538 MSize fprodd = 0;
539 #endif
540 #endif
541
542 if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) {
543 ct = ctype_get(cts, id);
544 rid = ctype_cid(ct->info); /* Return type. x86: +(spadj<<16). */
545 fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot));
546 fntp = LJ_TFUNC;
547 } else { /* Must set up frame first, before throwing the error. */
548 ct = NULL;
549 rid = 0;
550 fn = (GCfunc *)L;
551 fntp = LJ_TTHREAD;
552 }
553 /* Continuation returns from callback. */
554 if (LJ_FR2) {
555 (o++)->u64 = LJ_CONT_FFI_CALLBACK;
556 (o++)->u64 = rid;
557 } else {
558 o->u32.lo = LJ_CONT_FFI_CALLBACK;
559 o->u32.hi = rid;
560 o++;
561 }
562 setframe_gc(o, obj2gco(fn), fntp);
563 if (LJ_FR2) o++;
564 setframe_ftsz(o, ((char *)(o+1) - (char *)L->base) + FRAME_CONT);
565 L->top = L->base = ++o;
566 if (!ct)
567 lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK);
568 if (isluafunc(fn))
569 setcframe_pc(L->cframe, proto_bc(funcproto(fn))+1);
570 lj_state_checkstack(L, LUA_MINSTACK); /* May throw. */
571 o = L->base; /* Might have been reallocated. */
572
573 #if LJ_TARGET_X86
574 /* x86 has several different calling conventions. */
575 switch (ctype_cconv(ct->info)) {
576 case CTCC_FASTCALL: maxgpr = 2; break;
577 case CTCC_THISCALL: maxgpr = 1; break;
578 default: maxgpr = 0; break;
579 }
580 #endif
581
582 fid = ct->sib;
583 while (fid) {
584 CType *ctf = ctype_get(cts, fid);
585 if (!ctype_isattrib(ctf->info)) {
586 CType *cta;
587 void *sp;
588 CTSize sz;
589 int isfp;
590 MSize n;
591 lj_assertCTS(ctype_isfield(ctf->info), "field expected");
592 cta = ctype_rawchild(cts, ctf);
593 isfp = ctype_isfp(cta->info);
594 sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
595 n = sz / CTSIZE_PTR; /* Number of GPRs or stack slots needed. */
596
597 CALLBACK_HANDLE_REGARG /* Handle register arguments. */
598
599 /* Otherwise pass argument on stack. */
600 if (CCALL_ALIGN_STACKARG && LJ_32 && sz == 8)
601 nsp = (nsp + 1) & ~1u; /* Align 64 bit argument on stack. */
602 sp = &stack[nsp];
603 nsp += n;
604
605 done:
606 if (LJ_BE && cta->size < CTSIZE_PTR
607 #if LJ_TARGET_MIPS64
608 && !(isfp && nsp)
609 #endif
610 )
611 sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size);
612 gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp);
613 }
614 fid = ctf->sib;
615 }
616 L->top = o;
617 #if LJ_TARGET_X86
618 /* Store stack adjustment for returns from non-cdecl callbacks. */
619 if (ctype_cconv(ct->info) != CTCC_CDECL) {
620 #if LJ_FR2
621 (L->base-3)->u64 |= (nsp << (16+2));
622 #else
623 (L->base-2)->u32.hi |= (nsp << (16+2));
624 #endif
625 }
626 #endif
627 while (gcsteps-- > 0)
628 lj_gc_check(L);
629 }
630
631 /* Convert Lua object to callback result. */
632 static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
633 {
634 #if LJ_FR2
635 CType *ctr = ctype_raw(cts, (uint16_t)(L->base-3)->u64);
636 #else
637 CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi);
638 #endif
639 #if LJ_TARGET_X86
640 cts->cb.gpr[2] = 0;
641 #endif
642 if (!ctype_isvoid(ctr->info)) {
643 uint8_t *dp = (uint8_t *)&cts->cb.gpr[0];
644 #if CCALL_NUM_FPR
645 if (ctype_isfp(ctr->info))
646 dp = (uint8_t *)&cts->cb.fpr[0];
647 #endif
648 #if LJ_TARGET_ARM64 && LJ_BE
649 if (ctype_isfp(ctr->info) && ctr->size == sizeof(float))
650 dp = (uint8_t *)&cts->cb.fpr[0].f[1];
651 #endif
652 lj_cconv_ct_tv(cts, ctr, dp, o, 0);
653 #ifdef CALLBACK_HANDLE_RET
654 CALLBACK_HANDLE_RET
655 #endif
656 /* Extend returned integers to (at least) 32 bits. */
657 if (ctype_isinteger_or_bool(ctr->info) && ctr->size < 4) {
658 if (ctr->info & CTF_UNSIGNED)
659 *(uint32_t *)dp = ctr->size == 1 ? (uint32_t)*(uint8_t *)dp :
660 (uint32_t)*(uint16_t *)dp;
661 else
662 *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp :
663 (int32_t)*(int16_t *)dp;
664 }
665 #if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
666 /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */
667 if (ctr->size <= 4 &&
668 (LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info)))
669 *(int64_t *)dp = (int64_t)*(int32_t *)dp;
670 #endif
671 #if LJ_TARGET_X86
672 if (ctype_isfp(ctr->info))
673 cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2;
674 #endif
675 }
676 }
677
678 /* Enter callback. */
679 lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
680 {
681 lua_State *L = cts->L;
682 global_State *g = cts->g;
683 lj_assertG(L != NULL, "uninitialized cts->L in callback");
684 if (tvref(g->jit_base)) {
685 setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
686 if (g->panic) g->panic(L);
687 exit(EXIT_FAILURE);
688 }
689 lj_trace_abort(g); /* Never record across callback. */
690 /* Setup C frame. */
691 cframe_prev(cf) = L->cframe;
692 setcframe_L(cf, L);
693 cframe_errfunc(cf) = -1;
694 cframe_nres(cf) = 0;
695 L->cframe = cf;
696 callback_conv_args(cts, L);
697 return L; /* Now call the function on this stack. */
698 }
699
700 /* Leave callback. */
701 void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o)
702 {
703 lua_State *L = cts->L;
704 GCfunc *fn;
705 TValue *obase = L->base;
706 L->base = L->top; /* Keep continuation frame for throwing errors. */
707 if (o >= L->base) {
708 /* PC of RET* is lost. Point to last line for result conv. errors. */
709 fn = curr_func(L);
710 if (isluafunc(fn)) {
711 GCproto *pt = funcproto(fn);
712 setcframe_pc(L->cframe, proto_bc(pt)+pt->sizebc+1);
713 }
714 }
715 callback_conv_result(cts, L, o);
716 /* Finally drop C frame and continuation frame. */
717 L->top -= 2+2*LJ_FR2;
718 L->base = obase;
719 L->cframe = cframe_prev(L->cframe);
720 cts->cb.slot = 0; /* Blacklist C function that called the callback. */
721 }
722
723 /* -- C callback management ----------------------------------------------- */
724
725 /* Get an unused slot in the callback slot table. */
726 static MSize callback_slot_new(CTState *cts, CType *ct)
727 {
728 CTypeID id = ctype_typeid(cts, ct);
729 CTypeID1 *cbid = cts->cb.cbid;
730 MSize top;
731 for (top = cts->cb.topid; top < cts->cb.sizeid; top++)
732 if (LJ_LIKELY(cbid[top] == 0))
733 goto found;
734 #if CALLBACK_MAX_SLOT
735 if (top >= CALLBACK_MAX_SLOT)
736 #endif
737 lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
738 if (!cts->cb.mcode)
739 callback_mcode_new(cts);
740 lj_mem_growvec(cts->L, cbid, cts->cb.sizeid, CALLBACK_MAX_SLOT, CTypeID1);
741 cts->cb.cbid = cbid;
742 memset(cbid+top, 0, (cts->cb.sizeid-top)*sizeof(CTypeID1));
743 found:
744 cbid[top] = id;
745 cts->cb.topid = top+1;
746 return top;
747 }
748
749 /* Check for function pointer and supported argument/result types. */
750 static CType *callback_checkfunc(CTState *cts, CType *ct)
751 {
752 int narg = 0;
753 if (!ctype_isptr(ct->info) || (LJ_64 && ct->size != CTSIZE_PTR))
754 return NULL;
755 ct = ctype_rawchild(cts, ct);
756 if (ctype_isfunc(ct->info)) {
757 CType *ctr = ctype_rawchild(cts, ct);
758 CTypeID fid = ct->sib;
759 if (!(ctype_isvoid(ctr->info) || ctype_isenum(ctr->info) ||
760 ctype_isptr(ctr->info) || (ctype_isnum(ctr->info) && ctr->size <= 8)))
761 return NULL;
762 if ((ct->info & CTF_VARARG))
763 return NULL;
764 while (fid) {
765 CType *ctf = ctype_get(cts, fid);
766 if (!ctype_isattrib(ctf->info)) {
767 CType *cta;
768 lj_assertCTS(ctype_isfield(ctf->info), "field expected");
769 cta = ctype_rawchild(cts, ctf);
770 if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) ||
771 (ctype_isnum(cta->info) && cta->size <= 8)) ||
772 ++narg >= LUA_MINSTACK-3)
773 return NULL;
774 }
775 fid = ctf->sib;
776 }
777 return ct;
778 }
779 return NULL;
780 }
781
782 /* Create a new callback and return the callback function pointer. */
783 void *lj_ccallback_new(CTState *cts, CType *ct, GCfunc *fn)
784 {
785 ct = callback_checkfunc(cts, ct);
786 if (ct) {
787 MSize slot = callback_slot_new(cts, ct);
788 GCtab *t = cts->miscmap;
789 setfuncV(cts->L, lj_tab_setint(cts->L, t, (int32_t)slot), fn);
790 lj_gc_anybarriert(cts->L, t);
791 return callback_slot2ptr(cts, slot);
792 }
793 return NULL; /* Bad conversion. */
794 }
795
796 #endif