Mercurial

comparison third_party/luajit/src/lj_crecord.c @ 186:8cf4ec5e2191 hg-web

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fixed merge conflict.
author MrJuneJune <me@mrjunejune.com>
date Fri, 23 Jan 2026 22:38:59 -0800
parents 94705b5986b3
children
comparison
equal deleted inserted replaced
176:fed99fc04e12 186:8cf4ec5e2191
1 /*
2 ** Trace recorder for C data operations.
3 ** Copyright (C) 2005-2023 Mike Pall. See Copyright Notice in luajit.h
4 */
5
6 #define lj_ffrecord_c
7 #define LUA_CORE
8
9 #include "lj_obj.h"
10
11 #if LJ_HASJIT && LJ_HASFFI
12
13 #include "lj_err.h"
14 #include "lj_tab.h"
15 #include "lj_frame.h"
16 #include "lj_ctype.h"
17 #include "lj_cdata.h"
18 #include "lj_cparse.h"
19 #include "lj_cconv.h"
20 #include "lj_carith.h"
21 #include "lj_clib.h"
22 #include "lj_ccall.h"
23 #include "lj_ff.h"
24 #include "lj_ir.h"
25 #include "lj_jit.h"
26 #include "lj_ircall.h"
27 #include "lj_iropt.h"
28 #include "lj_trace.h"
29 #include "lj_record.h"
30 #include "lj_ffrecord.h"
31 #include "lj_snap.h"
32 #include "lj_crecord.h"
33 #include "lj_dispatch.h"
34 #include "lj_strfmt.h"
35
36 /* Some local macros to save typing. Undef'd at the end. */
37 #define IR(ref) (&J->cur.ir[(ref)])
38
39 /* Pass IR on to next optimization in chain (FOLD). */
40 #define emitir(ot, a, b) (lj_ir_set(J, (ot), (a), (b)), lj_opt_fold(J))
41
42 #define emitconv(a, dt, st, flags) \
43 emitir(IRT(IR_CONV, (dt)), (a), (st)|((dt) << 5)|(flags))
44
45 /* -- C type checks ------------------------------------------------------- */
46
47 static GCcdata *argv2cdata(jit_State *J, TRef tr, cTValue *o)
48 {
49 GCcdata *cd;
50 TRef trtypeid;
51 if (!tref_iscdata(tr))
52 lj_trace_err(J, LJ_TRERR_BADTYPE);
53 cd = cdataV(o);
54 /* Specialize to the CTypeID. */
55 trtypeid = emitir(IRT(IR_FLOAD, IRT_U16), tr, IRFL_CDATA_CTYPEID);
56 emitir(IRTG(IR_EQ, IRT_INT), trtypeid, lj_ir_kint(J, (int32_t)cd->ctypeid));
57 return cd;
58 }
59
60 /* Specialize to the CTypeID held by a cdata constructor. */
61 static CTypeID crec_constructor(jit_State *J, GCcdata *cd, TRef tr)
62 {
63 CTypeID id;
64 lj_assertJ(tref_iscdata(tr) && cd->ctypeid == CTID_CTYPEID,
65 "expected CTypeID cdata");
66 id = *(CTypeID *)cdataptr(cd);
67 tr = emitir(IRT(IR_FLOAD, IRT_INT), tr, IRFL_CDATA_INT);
68 emitir(IRTG(IR_EQ, IRT_INT), tr, lj_ir_kint(J, (int32_t)id));
69 return id;
70 }
71
72 static CTypeID argv2ctype(jit_State *J, TRef tr, cTValue *o)
73 {
74 if (tref_isstr(tr)) {
75 GCstr *s = strV(o);
76 CPState cp;
77 CTypeID oldtop;
78 /* Specialize to the string containing the C type declaration. */
79 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, s));
80 cp.L = J->L;
81 cp.cts = ctype_cts(J->L);
82 oldtop = cp.cts->top;
83 cp.srcname = strdata(s);
84 cp.p = strdata(s);
85 cp.param = NULL;
86 cp.mode = CPARSE_MODE_ABSTRACT|CPARSE_MODE_NOIMPLICIT;
87 if (lj_cparse(&cp) || cp.cts->top > oldtop) /* Avoid new struct defs. */
88 lj_trace_err(J, LJ_TRERR_BADTYPE);
89 return cp.val.id;
90 } else {
91 GCcdata *cd = argv2cdata(J, tr, o);
92 return cd->ctypeid == CTID_CTYPEID ? crec_constructor(J, cd, tr) :
93 cd->ctypeid;
94 }
95 }
96
97 /* Convert CType to IRType (if possible). */
98 static IRType crec_ct2irt(CTState *cts, CType *ct)
99 {
100 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
101 if (LJ_LIKELY(ctype_isnum(ct->info))) {
102 if ((ct->info & CTF_FP)) {
103 if (ct->size == sizeof(double))
104 return IRT_NUM;
105 else if (ct->size == sizeof(float))
106 return IRT_FLOAT;
107 } else {
108 uint32_t b = lj_fls(ct->size);
109 if (b <= 3)
110 return IRT_I8 + 2*b + ((ct->info & CTF_UNSIGNED) ? 1 : 0);
111 }
112 } else if (ctype_isptr(ct->info)) {
113 return (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
114 } else if (ctype_iscomplex(ct->info)) {
115 if (ct->size == 2*sizeof(double))
116 return IRT_NUM;
117 else if (ct->size == 2*sizeof(float))
118 return IRT_FLOAT;
119 }
120 return IRT_CDATA;
121 }
122
123 /* -- Optimized memory fill and copy -------------------------------------- */
124
125 /* Maximum length and unroll of inlined copy/fill. */
126 #define CREC_COPY_MAXUNROLL 16
127 #define CREC_COPY_MAXLEN 128
128
129 #define CREC_FILL_MAXUNROLL 16
130
131 /* Number of windowed registers used for optimized memory copy. */
132 #if LJ_TARGET_X86
133 #define CREC_COPY_REGWIN 2
134 #elif LJ_TARGET_PPC || LJ_TARGET_MIPS
135 #define CREC_COPY_REGWIN 8
136 #else
137 #define CREC_COPY_REGWIN 4
138 #endif
139
140 /* List of memory offsets for copy/fill. */
141 typedef struct CRecMemList {
142 CTSize ofs; /* Offset in bytes. */
143 IRType tp; /* Type of load/store. */
144 TRef trofs; /* TRef of interned offset. */
145 TRef trval; /* TRef of load value. */
146 } CRecMemList;
147
148 /* Generate copy list for element-wise struct copy. */
149 static MSize crec_copy_struct(CRecMemList *ml, CTState *cts, CType *ct)
150 {
151 CTypeID fid = ct->sib;
152 MSize mlp = 0;
153 while (fid) {
154 CType *df = ctype_get(cts, fid);
155 fid = df->sib;
156 if (ctype_isfield(df->info)) {
157 CType *cct;
158 IRType tp;
159 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
160 cct = ctype_rawchild(cts, df); /* Field type. */
161 tp = crec_ct2irt(cts, cct);
162 if (tp == IRT_CDATA) return 0; /* NYI: aggregates. */
163 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
164 ml[mlp].ofs = df->size;
165 ml[mlp].tp = tp;
166 mlp++;
167 if (ctype_iscomplex(cct->info)) {
168 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
169 ml[mlp].ofs = df->size + (cct->size >> 1);
170 ml[mlp].tp = tp;
171 mlp++;
172 }
173 } else if (!ctype_isconstval(df->info)) {
174 /* NYI: bitfields and sub-structures. */
175 return 0;
176 }
177 }
178 return mlp;
179 }
180
181 /* Generate unrolled copy list, from highest to lowest step size/alignment. */
182 static MSize crec_copy_unroll(CRecMemList *ml, CTSize len, CTSize step,
183 IRType tp)
184 {
185 CTSize ofs = 0;
186 MSize mlp = 0;
187 if (tp == IRT_CDATA) tp = IRT_U8 + 2*lj_fls(step);
188 do {
189 while (ofs + step <= len) {
190 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
191 ml[mlp].ofs = ofs;
192 ml[mlp].tp = tp;
193 mlp++;
194 ofs += step;
195 }
196 step >>= 1;
197 tp -= 2;
198 } while (ofs < len);
199 return mlp;
200 }
201
202 /*
203 ** Emit copy list with windowed loads/stores.
204 ** LJ_TARGET_UNALIGNED: may emit unaligned loads/stores (not marked as such).
205 */
206 static void crec_copy_emit(jit_State *J, CRecMemList *ml, MSize mlp,
207 TRef trdst, TRef trsrc)
208 {
209 MSize i, j, rwin = 0;
210 for (i = 0, j = 0; i < mlp; ) {
211 TRef trofs = lj_ir_kintp(J, ml[i].ofs);
212 TRef trsptr = emitir(IRT(IR_ADD, IRT_PTR), trsrc, trofs);
213 ml[i].trval = emitir(IRT(IR_XLOAD, ml[i].tp), trsptr, 0);
214 ml[i].trofs = trofs;
215 i++;
216 rwin += (LJ_SOFTFP32 && ml[i].tp == IRT_NUM) ? 2 : 1;
217 if (rwin >= CREC_COPY_REGWIN || i >= mlp) { /* Flush buffered stores. */
218 rwin = 0;
219 for ( ; j < i; j++) {
220 TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, ml[j].trofs);
221 emitir(IRT(IR_XSTORE, ml[j].tp), trdptr, ml[j].trval);
222 }
223 }
224 }
225 }
226
227 /* Optimized memory copy. */
228 static void crec_copy(jit_State *J, TRef trdst, TRef trsrc, TRef trlen,
229 CType *ct)
230 {
231 if (tref_isk(trlen)) { /* Length must be constant. */
232 CRecMemList ml[CREC_COPY_MAXUNROLL];
233 MSize mlp = 0;
234 CTSize step = 1, len = (CTSize)IR(tref_ref(trlen))->i;
235 IRType tp = IRT_CDATA;
236 int needxbar = 0;
237 if (len == 0) return; /* Shortcut. */
238 if (len > CREC_COPY_MAXLEN) goto fallback;
239 if (ct) {
240 CTState *cts = ctype_ctsG(J2G(J));
241 lj_assertJ(ctype_isarray(ct->info) || ctype_isstruct(ct->info),
242 "copy of non-aggregate");
243 if (ctype_isarray(ct->info)) {
244 CType *cct = ctype_rawchild(cts, ct);
245 tp = crec_ct2irt(cts, cct);
246 if (tp == IRT_CDATA) goto rawcopy;
247 step = lj_ir_type_size[tp];
248 lj_assertJ((len & (step-1)) == 0, "copy of fractional size");
249 } else if ((ct->info & CTF_UNION)) {
250 step = (1u << ctype_align(ct->info));
251 goto rawcopy;
252 } else {
253 mlp = crec_copy_struct(ml, cts, ct);
254 goto emitcopy;
255 }
256 } else {
257 rawcopy:
258 needxbar = 1;
259 if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
260 step = CTSIZE_PTR;
261 }
262 mlp = crec_copy_unroll(ml, len, step, tp);
263 emitcopy:
264 if (mlp) {
265 crec_copy_emit(J, ml, mlp, trdst, trsrc);
266 if (needxbar)
267 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
268 return;
269 }
270 }
271 fallback:
272 /* Call memcpy. Always needs a barrier to disable alias analysis. */
273 lj_ir_call(J, IRCALL_memcpy, trdst, trsrc, trlen);
274 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
275 }
276
277 /* Generate unrolled fill list, from highest to lowest step size/alignment. */
278 static MSize crec_fill_unroll(CRecMemList *ml, CTSize len, CTSize step)
279 {
280 CTSize ofs = 0;
281 MSize mlp = 0;
282 IRType tp = IRT_U8 + 2*lj_fls(step);
283 do {
284 while (ofs + step <= len) {
285 if (mlp >= CREC_COPY_MAXUNROLL) return 0;
286 ml[mlp].ofs = ofs;
287 ml[mlp].tp = tp;
288 mlp++;
289 ofs += step;
290 }
291 step >>= 1;
292 tp -= 2;
293 } while (ofs < len);
294 return mlp;
295 }
296
297 /*
298 ** Emit stores for fill list.
299 ** LJ_TARGET_UNALIGNED: may emit unaligned stores (not marked as such).
300 */
301 static void crec_fill_emit(jit_State *J, CRecMemList *ml, MSize mlp,
302 TRef trdst, TRef trfill)
303 {
304 MSize i;
305 for (i = 0; i < mlp; i++) {
306 TRef trofs = lj_ir_kintp(J, ml[i].ofs);
307 TRef trdptr = emitir(IRT(IR_ADD, IRT_PTR), trdst, trofs);
308 emitir(IRT(IR_XSTORE, ml[i].tp), trdptr, trfill);
309 }
310 }
311
312 /* Optimized memory fill. */
313 static void crec_fill(jit_State *J, TRef trdst, TRef trlen, TRef trfill,
314 CTSize step)
315 {
316 if (tref_isk(trlen)) { /* Length must be constant. */
317 CRecMemList ml[CREC_FILL_MAXUNROLL];
318 MSize mlp;
319 CTSize len = (CTSize)IR(tref_ref(trlen))->i;
320 if (len == 0) return; /* Shortcut. */
321 if (LJ_TARGET_UNALIGNED || step >= CTSIZE_PTR)
322 step = CTSIZE_PTR;
323 if (step * CREC_FILL_MAXUNROLL < len) goto fallback;
324 mlp = crec_fill_unroll(ml, len, step);
325 if (!mlp) goto fallback;
326 if (tref_isk(trfill) || ml[0].tp != IRT_U8)
327 trfill = emitconv(trfill, IRT_INT, IRT_U8, 0);
328 if (ml[0].tp != IRT_U8) { /* Scatter U8 to U16/U32/U64. */
329 if (CTSIZE_PTR == 8 && ml[0].tp == IRT_U64) {
330 if (tref_isk(trfill)) /* Pointless on x64 with zero-extended regs. */
331 trfill = emitconv(trfill, IRT_U64, IRT_U32, 0);
332 trfill = emitir(IRT(IR_MUL, IRT_U64), trfill,
333 lj_ir_kint64(J, U64x(01010101,01010101)));
334 } else {
335 trfill = emitir(IRTI(IR_MUL), trfill,
336 lj_ir_kint(J, ml[0].tp == IRT_U16 ? 0x0101 : 0x01010101));
337 }
338 }
339 crec_fill_emit(J, ml, mlp, trdst, trfill);
340 } else {
341 fallback:
342 /* Call memset. Always needs a barrier to disable alias analysis. */
343 lj_ir_call(J, IRCALL_memset, trdst, trfill, trlen); /* Note: arg order! */
344 }
345 emitir(IRT(IR_XBAR, IRT_NIL), 0, 0);
346 }
347
348 /* -- Convert C type to C type -------------------------------------------- */
349
350 /*
351 ** This code mirrors the code in lj_cconv.c. It performs the same steps
352 ** for the trace recorder that lj_cconv.c does for the interpreter.
353 **
354 ** One major difference is that we can get away with much fewer checks
355 ** here. E.g. checks for casts, constness or correct types can often be
356 ** omitted, even if they might fail. The interpreter subsequently throws
357 ** an error, which aborts the trace.
358 **
359 ** All operations are specialized to their C types, so the on-trace
360 ** outcome must be the same as the outcome in the interpreter. If the
361 ** interpreter doesn't throw an error, then the trace is correct, too.
362 ** Care must be taken not to generate invalid (temporary) IR or to
363 ** trigger asserts.
364 */
365
366 /* Determine whether a passed number or cdata number is non-zero. */
367 static int crec_isnonzero(CType *s, void *p)
368 {
369 if (p == (void *)0)
370 return 0;
371 if (p == (void *)1)
372 return 1;
373 if ((s->info & CTF_FP)) {
374 if (s->size == sizeof(float))
375 return (*(float *)p != 0);
376 else
377 return (*(double *)p != 0);
378 } else {
379 if (s->size == 1)
380 return (*(uint8_t *)p != 0);
381 else if (s->size == 2)
382 return (*(uint16_t *)p != 0);
383 else if (s->size == 4)
384 return (*(uint32_t *)p != 0);
385 else
386 return (*(uint64_t *)p != 0);
387 }
388 }
389
390 static TRef crec_ct_ct(jit_State *J, CType *d, CType *s, TRef dp, TRef sp,
391 void *svisnz)
392 {
393 IRType dt = crec_ct2irt(ctype_ctsG(J2G(J)), d);
394 IRType st = crec_ct2irt(ctype_ctsG(J2G(J)), s);
395 CTSize dsize = d->size, ssize = s->size;
396 CTInfo dinfo = d->info, sinfo = s->info;
397
398 if (ctype_type(dinfo) > CT_MAYCONVERT || ctype_type(sinfo) > CT_MAYCONVERT)
399 goto err_conv;
400
401 /*
402 ** Note: Unlike lj_cconv_ct_ct(), sp holds the _value_ of pointers and
403 ** numbers up to 8 bytes. Otherwise sp holds a pointer.
404 */
405
406 switch (cconv_idx2(dinfo, sinfo)) {
407 /* Destination is a bool. */
408 case CCX(B, B):
409 goto xstore; /* Source operand is already normalized. */
410 case CCX(B, I):
411 case CCX(B, F):
412 if (st != IRT_CDATA) {
413 /* Specialize to the result of a comparison against 0. */
414 TRef zero = (st == IRT_NUM || st == IRT_FLOAT) ? lj_ir_knum(J, 0) :
415 (st == IRT_I64 || st == IRT_U64) ? lj_ir_kint64(J, 0) :
416 lj_ir_kint(J, 0);
417 int isnz = crec_isnonzero(s, svisnz);
418 emitir(IRTG(isnz ? IR_NE : IR_EQ, st), sp, zero);
419 sp = lj_ir_kint(J, isnz);
420 goto xstore;
421 }
422 goto err_nyi;
423
424 /* Destination is an integer. */
425 case CCX(I, B):
426 case CCX(I, I):
427 conv_I_I:
428 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
429 /* Extend 32 to 64 bit integer. */
430 if (dsize == 8 && ssize < 8 && !(LJ_64 && (sinfo & CTF_UNSIGNED)))
431 sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st,
432 (sinfo & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
433 else if (dsize < 8 && ssize == 8) /* Truncate from 64 bit integer. */
434 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, 0);
435 else if (st == IRT_INT)
436 sp = lj_opt_narrow_toint(J, sp);
437 xstore:
438 if (dt == IRT_I64 || dt == IRT_U64) lj_needsplit(J);
439 if (dp == 0) return sp;
440 emitir(IRT(IR_XSTORE, dt), dp, sp);
441 break;
442 case CCX(I, C):
443 sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
444 /* fallthrough */
445 case CCX(I, F):
446 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
447 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, st, IRCONV_ANY);
448 goto xstore;
449 case CCX(I, P):
450 case CCX(I, A):
451 sinfo = CTINFO(CT_NUM, CTF_UNSIGNED);
452 ssize = CTSIZE_PTR;
453 st = IRT_UINTP;
454 if (((dsize ^ ssize) & 8) == 0) { /* Must insert no-op type conversion. */
455 sp = emitconv(sp, dsize < 4 ? IRT_INT : dt, IRT_PTR, 0);
456 goto xstore;
457 }
458 goto conv_I_I;
459
460 /* Destination is a floating-point number. */
461 case CCX(F, B):
462 case CCX(F, I):
463 conv_F_I:
464 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
465 sp = emitconv(sp, dt, ssize < 4 ? IRT_INT : st, 0);
466 goto xstore;
467 case CCX(F, C):
468 sp = emitir(IRT(IR_XLOAD, st), sp, 0); /* Load re. */
469 /* fallthrough */
470 case CCX(F, F):
471 conv_F_F:
472 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
473 if (dt != st) sp = emitconv(sp, dt, st, 0);
474 goto xstore;
475
476 /* Destination is a complex number. */
477 case CCX(C, I):
478 case CCX(C, F):
479 { /* Clear im. */
480 TRef ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
481 emitir(IRT(IR_XSTORE, dt), ptr, lj_ir_knum(J, 0));
482 }
483 /* Convert to re. */
484 if ((sinfo & CTF_FP)) goto conv_F_F; else goto conv_F_I;
485
486 case CCX(C, C):
487 if (dt == IRT_CDATA || st == IRT_CDATA) goto err_nyi;
488 {
489 TRef re, im, ptr;
490 re = emitir(IRT(IR_XLOAD, st), sp, 0);
491 ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, (ssize >> 1)));
492 im = emitir(IRT(IR_XLOAD, st), ptr, 0);
493 if (dt != st) {
494 re = emitconv(re, dt, st, 0);
495 im = emitconv(im, dt, st, 0);
496 }
497 emitir(IRT(IR_XSTORE, dt), dp, re);
498 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, (dsize >> 1)));
499 emitir(IRT(IR_XSTORE, dt), ptr, im);
500 }
501 break;
502
503 /* Destination is a vector. */
504 case CCX(V, I):
505 case CCX(V, F):
506 case CCX(V, C):
507 case CCX(V, V):
508 goto err_nyi;
509
510 /* Destination is a pointer. */
511 case CCX(P, P):
512 case CCX(P, A):
513 case CCX(P, S):
514 /* There are only 32 bit pointers/addresses on 32 bit machines.
515 ** Also ok on x64, since all 32 bit ops clear the upper part of the reg.
516 */
517 goto xstore;
518 case CCX(P, I):
519 if (st == IRT_CDATA) goto err_nyi;
520 if (!LJ_64 && ssize == 8) /* Truncate from 64 bit integer. */
521 sp = emitconv(sp, IRT_U32, st, 0);
522 goto xstore;
523 case CCX(P, F):
524 if (st == IRT_CDATA) goto err_nyi;
525 /* The signed conversion is cheaper. x64 really has 47 bit pointers. */
526 sp = emitconv(sp, (LJ_64 && dsize == 8) ? IRT_I64 : IRT_U32,
527 st, IRCONV_ANY);
528 goto xstore;
529
530 /* Destination is an array. */
531 case CCX(A, A):
532 /* Destination is a struct/union. */
533 case CCX(S, S):
534 if (dp == 0) goto err_conv;
535 crec_copy(J, dp, sp, lj_ir_kint(J, dsize), d);
536 break;
537
538 default:
539 err_conv:
540 err_nyi:
541 lj_trace_err(J, LJ_TRERR_NYICONV);
542 break;
543 }
544 return 0;
545 }
546
547 /* -- Convert C type to TValue (load) ------------------------------------- */
548
549 static TRef crec_tv_ct(jit_State *J, CType *s, CTypeID sid, TRef sp)
550 {
551 CTState *cts = ctype_ctsG(J2G(J));
552 IRType t = crec_ct2irt(cts, s);
553 CTInfo sinfo = s->info;
554 if (ctype_isnum(sinfo)) {
555 TRef tr;
556 if (t == IRT_CDATA)
557 goto err_nyi; /* NYI: copyval of >64 bit integers. */
558 tr = emitir(IRT(IR_XLOAD, t), sp, 0);
559 if (t == IRT_FLOAT || t == IRT_U32) { /* Keep uint32_t/float as numbers. */
560 return emitconv(tr, IRT_NUM, t, 0);
561 } else if (t == IRT_I64 || t == IRT_U64) { /* Box 64 bit integer. */
562 sp = tr;
563 lj_needsplit(J);
564 } else if ((sinfo & CTF_BOOL)) {
565 /* Assume not equal to zero. Fixup and emit pending guard later. */
566 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
567 J->postproc = LJ_POST_FIXGUARD;
568 return TREF_TRUE;
569 } else {
570 return tr;
571 }
572 } else if (ctype_isptr(sinfo) || ctype_isenum(sinfo)) {
573 sp = emitir(IRT(IR_XLOAD, t), sp, 0); /* Box pointers and enums. */
574 } else if (ctype_isrefarray(sinfo) || ctype_isstruct(sinfo)) {
575 cts->L = J->L;
576 sid = lj_ctype_intern(cts, CTINFO_REF(sid), CTSIZE_PTR); /* Create ref. */
577 } else if (ctype_iscomplex(sinfo)) { /* Unbox/box complex. */
578 ptrdiff_t esz = (ptrdiff_t)(s->size >> 1);
579 TRef ptr, tr1, tr2, dp;
580 dp = emitir(IRTG(IR_CNEW, IRT_CDATA), lj_ir_kint(J, sid), TREF_NIL);
581 tr1 = emitir(IRT(IR_XLOAD, t), sp, 0);
582 ptr = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, esz));
583 tr2 = emitir(IRT(IR_XLOAD, t), ptr, 0);
584 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)));
585 emitir(IRT(IR_XSTORE, t), ptr, tr1);
586 ptr = emitir(IRT(IR_ADD, IRT_PTR), dp, lj_ir_kintp(J, sizeof(GCcdata)+esz));
587 emitir(IRT(IR_XSTORE, t), ptr, tr2);
588 return dp;
589 } else {
590 /* NYI: copyval of vectors. */
591 err_nyi:
592 lj_trace_err(J, LJ_TRERR_NYICONV);
593 }
594 /* Box pointer, ref, enum or 64 bit integer. */
595 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, sid), sp);
596 }
597
598 /* -- Convert TValue to C type (store) ------------------------------------ */
599
600 static TRef crec_ct_tv(jit_State *J, CType *d, TRef dp, TRef sp, cTValue *sval)
601 {
602 CTState *cts = ctype_ctsG(J2G(J));
603 CTypeID sid = CTID_P_VOID;
604 void *svisnz = 0;
605 CType *s;
606 if (LJ_LIKELY(tref_isinteger(sp))) {
607 sid = CTID_INT32;
608 svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
609 } else if (tref_isnum(sp)) {
610 sid = CTID_DOUBLE;
611 svisnz = (void *)(intptr_t)(tvisint(sval)?(intV(sval)!=0):!tviszero(sval));
612 } else if (tref_isbool(sp)) {
613 sp = lj_ir_kint(J, tref_istrue(sp) ? 1 : 0);
614 sid = CTID_BOOL;
615 } else if (tref_isnil(sp)) {
616 sp = lj_ir_kptr(J, NULL);
617 } else if (tref_isudata(sp)) {
618 GCudata *ud = udataV(sval);
619 if (ud->udtype == UDTYPE_IO_FILE || ud->udtype == UDTYPE_BUFFER) {
620 TRef tr = emitir(IRT(IR_FLOAD, IRT_U8), sp, IRFL_UDATA_UDTYPE);
621 emitir(IRTGI(IR_EQ), tr, lj_ir_kint(J, ud->udtype));
622 sp = emitir(IRT(IR_FLOAD, IRT_PTR), sp,
623 ud->udtype == UDTYPE_IO_FILE ? IRFL_UDATA_FILE :
624 IRFL_SBUF_R);
625 } else {
626 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCudata)));
627 }
628 } else if (tref_isstr(sp)) {
629 if (ctype_isenum(d->info)) { /* Match string against enum constant. */
630 GCstr *str = strV(sval);
631 CTSize ofs;
632 CType *cct = lj_ctype_getfield(cts, d, str, &ofs);
633 /* Specialize to the name of the enum constant. */
634 emitir(IRTG(IR_EQ, IRT_STR), sp, lj_ir_kstr(J, str));
635 if (cct && ctype_isconstval(cct->info)) {
636 lj_assertJ(ctype_child(cts, cct)->size == 4,
637 "only 32 bit const supported"); /* NYI */
638 svisnz = (void *)(intptr_t)(ofs != 0);
639 sp = lj_ir_kint(J, (int32_t)ofs);
640 sid = ctype_cid(cct->info);
641 } /* else: interpreter will throw. */
642 } else if (ctype_isrefarray(d->info)) { /* Copy string to array. */
643 lj_trace_err(J, LJ_TRERR_BADTYPE); /* NYI */
644 } else { /* Otherwise pass the string data as a const char[]. */
645 /* Don't use STRREF. It folds with SNEW, which loses the trailing NUL. */
646 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCstr)));
647 sid = CTID_A_CCHAR;
648 }
649 } else if (tref_islightud(sp)) {
650 #if LJ_64
651 lj_trace_err(J, LJ_TRERR_NYICONV);
652 #endif
653 } else { /* NYI: tref_istab(sp). */
654 IRType t;
655 sid = argv2cdata(J, sp, sval)->ctypeid;
656 s = ctype_raw(cts, sid);
657 svisnz = cdataptr(cdataV(sval));
658 if (ctype_isfunc(s->info)) {
659 sid = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|sid), CTSIZE_PTR);
660 s = ctype_get(cts, sid);
661 t = IRT_PTR;
662 } else {
663 t = crec_ct2irt(cts, s);
664 }
665 if (ctype_isptr(s->info)) {
666 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_PTR);
667 if (ctype_isref(s->info)) {
668 svisnz = *(void **)svisnz;
669 s = ctype_rawchild(cts, s);
670 if (ctype_isenum(s->info)) s = ctype_child(cts, s);
671 t = crec_ct2irt(cts, s);
672 } else {
673 goto doconv;
674 }
675 } else if (t == IRT_I64 || t == IRT_U64) {
676 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT64);
677 lj_needsplit(J);
678 goto doconv;
679 } else if (t == IRT_INT || t == IRT_U32) {
680 if (ctype_isenum(s->info)) s = ctype_child(cts, s);
681 sp = emitir(IRT(IR_FLOAD, t), sp, IRFL_CDATA_INT);
682 goto doconv;
683 } else {
684 sp = emitir(IRT(IR_ADD, IRT_PTR), sp, lj_ir_kintp(J, sizeof(GCcdata)));
685 }
686 if (ctype_isnum(s->info) && t != IRT_CDATA)
687 sp = emitir(IRT(IR_XLOAD, t), sp, 0); /* Load number value. */
688 goto doconv;
689 }
690 s = ctype_get(cts, sid);
691 doconv:
692 if (ctype_isenum(d->info)) d = ctype_child(cts, d);
693 return crec_ct_ct(J, d, s, dp, sp, svisnz);
694 }
695
696 /* -- C data metamethods -------------------------------------------------- */
697
698 /* This would be rather difficult in FOLD, so do it here:
699 ** (base+k)+(idx*sz)+ofs ==> (base+idx*sz)+(ofs+k)
700 ** (base+(idx+k)*sz)+ofs ==> (base+idx*sz)+(ofs+k*sz)
701 */
702 static TRef crec_reassoc_ofs(jit_State *J, TRef tr, ptrdiff_t *ofsp, MSize sz)
703 {
704 IRIns *ir = IR(tref_ref(tr));
705 if (LJ_LIKELY(J->flags & JIT_F_OPT_FOLD) && irref_isk(ir->op2) &&
706 (ir->o == IR_ADD || ir->o == IR_ADDOV || ir->o == IR_SUBOV)) {
707 IRIns *irk = IR(ir->op2);
708 ptrdiff_t k;
709 if (LJ_64 && irk->o == IR_KINT64)
710 k = (ptrdiff_t)ir_kint64(irk)->u64 * sz;
711 else
712 k = (ptrdiff_t)irk->i * sz;
713 if (ir->o == IR_SUBOV) *ofsp -= k; else *ofsp += k;
714 tr = ir->op1; /* Not a TRef, but the caller doesn't care. */
715 }
716 return tr;
717 }
718
719 /* Tailcall to function. */
720 static void crec_tailcall(jit_State *J, RecordFFData *rd, cTValue *tv)
721 {
722 TRef kfunc = lj_ir_kfunc(J, funcV(tv));
723 #if LJ_FR2
724 J->base[-2] = kfunc;
725 J->base[-1] = TREF_FRAME;
726 #else
727 J->base[-1] = kfunc | TREF_FRAME;
728 #endif
729 rd->nres = -1; /* Pending tailcall. */
730 }
731
732 /* Record ctype __index/__newindex metamethods. */
733 static void crec_index_meta(jit_State *J, CTState *cts, CType *ct,
734 RecordFFData *rd)
735 {
736 CTypeID id = ctype_typeid(cts, ct);
737 cTValue *tv = lj_ctype_meta(cts, id, rd->data ? MM_newindex : MM_index);
738 if (!tv)
739 lj_trace_err(J, LJ_TRERR_BADTYPE);
740 if (tvisfunc(tv)) {
741 crec_tailcall(J, rd, tv);
742 } else if (rd->data == 0 && tvistab(tv) && tref_isstr(J->base[1])) {
743 /* Specialize to result of __index lookup. */
744 cTValue *o = lj_tab_get(J->L, tabV(tv), &rd->argv[1]);
745 J->base[0] = lj_record_constify(J, o);
746 if (!J->base[0])
747 lj_trace_err(J, LJ_TRERR_BADTYPE);
748 /* Always specialize to the key. */
749 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
750 } else {
751 /* NYI: resolving of non-function metamethods. */
752 /* NYI: non-string keys for __index table. */
753 /* NYI: stores to __newindex table. */
754 lj_trace_err(J, LJ_TRERR_BADTYPE);
755 }
756 }
757
758 /* Record bitfield load/store. */
759 static void crec_index_bf(jit_State *J, RecordFFData *rd, TRef ptr, CTInfo info)
760 {
761 IRType t = IRT_I8 + 2*lj_fls(ctype_bitcsz(info)) + ((info&CTF_UNSIGNED)?1:0);
762 TRef tr = emitir(IRT(IR_XLOAD, t), ptr, 0);
763 CTSize pos = ctype_bitpos(info), bsz = ctype_bitbsz(info), shift = 32 - bsz;
764 lj_assertJ(t <= IRT_U32, "only 32 bit bitfields supported"); /* NYI */
765 if (rd->data == 0) { /* __index metamethod. */
766 if ((info & CTF_BOOL)) {
767 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << pos))));
768 /* Assume not equal to zero. Fixup and emit pending guard later. */
769 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
770 J->postproc = LJ_POST_FIXGUARD;
771 tr = TREF_TRUE;
772 } else if (!(info & CTF_UNSIGNED)) {
773 tr = emitir(IRTI(IR_BSHL), tr, lj_ir_kint(J, shift - pos));
774 tr = emitir(IRTI(IR_BSAR), tr, lj_ir_kint(J, shift));
775 } else {
776 lj_assertJ(bsz < 32, "unexpected full bitfield index");
777 tr = emitir(IRTI(IR_BSHR), tr, lj_ir_kint(J, pos));
778 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << bsz)-1)));
779 /* We can omit the U32 to NUM conversion, since bsz < 32. */
780 }
781 J->base[0] = tr;
782 } else { /* __newindex metamethod. */
783 CTState *cts = ctype_ctsG(J2G(J));
784 CType *ct = ctype_get(cts,
785 (info & CTF_BOOL) ? CTID_BOOL :
786 (info & CTF_UNSIGNED) ? CTID_UINT32 : CTID_INT32);
787 int32_t mask = (int32_t)(((1u << bsz)-1) << pos);
788 TRef sp = crec_ct_tv(J, ct, 0, J->base[2], &rd->argv[2]);
789 sp = emitir(IRTI(IR_BSHL), sp, lj_ir_kint(J, pos));
790 /* Use of the target type avoids forwarding conversions. */
791 sp = emitir(IRT(IR_BAND, t), sp, lj_ir_kint(J, mask));
792 tr = emitir(IRT(IR_BAND, t), tr, lj_ir_kint(J, (int32_t)~mask));
793 tr = emitir(IRT(IR_BOR, t), tr, sp);
794 emitir(IRT(IR_XSTORE, t), ptr, tr);
795 rd->nres = 0;
796 J->needsnap = 1;
797 }
798 }
799
800 void LJ_FASTCALL recff_cdata_index(jit_State *J, RecordFFData *rd)
801 {
802 TRef idx, ptr = J->base[0];
803 ptrdiff_t ofs = sizeof(GCcdata);
804 GCcdata *cd = argv2cdata(J, ptr, &rd->argv[0]);
805 CTState *cts = ctype_ctsG(J2G(J));
806 CType *ct = ctype_raw(cts, cd->ctypeid);
807 CTypeID sid = 0;
808
809 /* Resolve pointer or reference for cdata object. */
810 if (ctype_isptr(ct->info)) {
811 IRType t = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
812 if (ctype_isref(ct->info)) ct = ctype_rawchild(cts, ct);
813 ptr = emitir(IRT(IR_FLOAD, t), ptr, IRFL_CDATA_PTR);
814 ofs = 0;
815 ptr = crec_reassoc_ofs(J, ptr, &ofs, 1);
816 }
817
818 again:
819 idx = J->base[1];
820 if (tref_isnumber(idx)) {
821 idx = lj_opt_narrow_cindex(J, idx);
822 if (ctype_ispointer(ct->info)) {
823 CTSize sz;
824 integer_key:
825 if ((ct->info & CTF_COMPLEX))
826 idx = emitir(IRT(IR_BAND, IRT_INTP), idx, lj_ir_kintp(J, 1));
827 sz = lj_ctype_size(cts, (sid = ctype_cid(ct->info)));
828 idx = crec_reassoc_ofs(J, idx, &ofs, sz);
829 #if LJ_TARGET_ARM || LJ_TARGET_PPC
830 /* Hoist base add to allow fusion of index/shift into operands. */
831 if (LJ_LIKELY(J->flags & JIT_F_OPT_LOOP) && ofs
832 #if LJ_TARGET_ARM
833 && (sz == 1 || sz == 4)
834 #endif
835 ) {
836 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
837 ofs = 0;
838 }
839 #endif
840 idx = emitir(IRT(IR_MUL, IRT_INTP), idx, lj_ir_kintp(J, sz));
841 ptr = emitir(IRT(IR_ADD, IRT_PTR), idx, ptr);
842 }
843 } else if (tref_iscdata(idx)) {
844 GCcdata *cdk = cdataV(&rd->argv[1]);
845 CType *ctk = ctype_raw(cts, cdk->ctypeid);
846 IRType t = crec_ct2irt(cts, ctk);
847 if (ctype_ispointer(ct->info) && t >= IRT_I8 && t <= IRT_U64) {
848 if (ctk->size == 8) {
849 idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT64);
850 } else if (ctk->size == 4) {
851 idx = emitir(IRT(IR_FLOAD, t), idx, IRFL_CDATA_INT);
852 } else {
853 idx = emitir(IRT(IR_ADD, IRT_PTR), idx,
854 lj_ir_kintp(J, sizeof(GCcdata)));
855 idx = emitir(IRT(IR_XLOAD, t), idx, 0);
856 }
857 if (LJ_64 && ctk->size < sizeof(intptr_t) && !(ctk->info & CTF_UNSIGNED))
858 idx = emitconv(idx, IRT_INTP, IRT_INT, IRCONV_SEXT);
859 if (!LJ_64 && ctk->size > sizeof(intptr_t)) {
860 idx = emitconv(idx, IRT_INTP, t, 0);
861 lj_needsplit(J);
862 }
863 goto integer_key;
864 }
865 } else if (tref_isstr(idx)) {
866 GCstr *name = strV(&rd->argv[1]);
867 if (cd && cd->ctypeid == CTID_CTYPEID)
868 ct = ctype_raw(cts, crec_constructor(J, cd, ptr));
869 if (ctype_isstruct(ct->info)) {
870 CTSize fofs;
871 CType *fct;
872 fct = lj_ctype_getfield(cts, ct, name, &fofs);
873 if (fct) {
874 ofs += (ptrdiff_t)fofs;
875 /* Always specialize to the field name. */
876 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
877 if (ctype_isconstval(fct->info)) {
878 if (fct->size >= 0x80000000u &&
879 (ctype_child(cts, fct)->info & CTF_UNSIGNED)) {
880 J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)fct->size);
881 return;
882 }
883 J->base[0] = lj_ir_kint(J, (int32_t)fct->size);
884 return; /* Interpreter will throw for newindex. */
885 } else if (ctype_isbitfield(fct->info)) {
886 if (ofs)
887 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
888 crec_index_bf(J, rd, ptr, fct->info);
889 return;
890 } else {
891 lj_assertJ(ctype_isfield(fct->info), "field expected");
892 sid = ctype_cid(fct->info);
893 }
894 }
895 } else if (ctype_iscomplex(ct->info)) {
896 if (name->len == 2 &&
897 ((strdata(name)[0] == 'r' && strdata(name)[1] == 'e') ||
898 (strdata(name)[0] == 'i' && strdata(name)[1] == 'm'))) {
899 /* Always specialize to the field name. */
900 emitir(IRTG(IR_EQ, IRT_STR), idx, lj_ir_kstr(J, name));
901 if (strdata(name)[0] == 'i') ofs += (ct->size >> 1);
902 sid = ctype_cid(ct->info);
903 }
904 }
905 }
906 if (!sid) {
907 if (ctype_isptr(ct->info)) { /* Automatically perform '->'. */
908 CType *cct = ctype_rawchild(cts, ct);
909 if (ctype_isstruct(cct->info)) {
910 ct = cct;
911 cd = NULL;
912 if (tref_isstr(idx)) goto again;
913 }
914 }
915 crec_index_meta(J, cts, ct, rd);
916 return;
917 }
918
919 if (ofs)
920 ptr = emitir(IRT(IR_ADD, IRT_PTR), ptr, lj_ir_kintp(J, ofs));
921
922 /* Resolve reference for field. */
923 ct = ctype_get(cts, sid);
924 if (ctype_isref(ct->info)) {
925 ptr = emitir(IRT(IR_XLOAD, IRT_PTR), ptr, 0);
926 sid = ctype_cid(ct->info);
927 ct = ctype_get(cts, sid);
928 }
929
930 while (ctype_isattrib(ct->info))
931 ct = ctype_child(cts, ct); /* Skip attributes. */
932
933 if (rd->data == 0) { /* __index metamethod. */
934 J->base[0] = crec_tv_ct(J, ct, sid, ptr);
935 } else { /* __newindex metamethod. */
936 rd->nres = 0;
937 J->needsnap = 1;
938 crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
939 }
940 }
941
942 /* Record setting a finalizer. */
943 static void crec_finalizer(jit_State *J, TRef trcd, TRef trfin, cTValue *fin)
944 {
945 if (tvisgcv(fin)) {
946 if (!trfin) trfin = lj_ir_kptr(J, gcval(fin));
947 } else if (tvisnil(fin)) {
948 trfin = lj_ir_kptr(J, NULL);
949 } else {
950 lj_trace_err(J, LJ_TRERR_BADTYPE);
951 }
952 lj_ir_call(J, IRCALL_lj_cdata_setfin, trcd,
953 trfin, lj_ir_kint(J, (int32_t)itype(fin)));
954 J->needsnap = 1;
955 }
956
957 /* Record cdata allocation. */
958 static void crec_alloc(jit_State *J, RecordFFData *rd, CTypeID id)
959 {
960 CTState *cts = ctype_ctsG(J2G(J));
961 CTSize sz;
962 CTInfo info = lj_ctype_info(cts, id, &sz);
963 CType *d = ctype_raw(cts, id);
964 TRef trcd, trid = lj_ir_kint(J, id);
965 cTValue *fin;
966 /* Use special instruction to box pointer or 32/64 bit integer. */
967 if (ctype_isptr(info) || (ctype_isinteger(info) && (sz == 4 || sz == 8))) {
968 TRef sp = J->base[1] ? crec_ct_tv(J, d, 0, J->base[1], &rd->argv[1]) :
969 ctype_isptr(info) ? lj_ir_kptr(J, NULL) :
970 sz == 4 ? lj_ir_kint(J, 0) :
971 (lj_needsplit(J), lj_ir_kint64(J, 0));
972 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, sp);
973 return;
974 } else {
975 TRef trsz = TREF_NIL;
976 if ((info & CTF_VLA)) { /* Calculate VLA/VLS size at runtime. */
977 CTSize sz0, sz1;
978 if (!J->base[1] || J->base[2])
979 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init VLA/VLS. */
980 trsz = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0,
981 J->base[1], &rd->argv[1]);
982 sz0 = lj_ctype_vlsize(cts, d, 0);
983 sz1 = lj_ctype_vlsize(cts, d, 1);
984 trsz = emitir(IRTGI(IR_MULOV), trsz, lj_ir_kint(J, (int32_t)(sz1-sz0)));
985 trsz = emitir(IRTGI(IR_ADDOV), trsz, lj_ir_kint(J, (int32_t)sz0));
986 J->base[1] = 0; /* Simplify logic below. */
987 } else if (ctype_align(info) > CT_MEMALIGN) {
988 trsz = lj_ir_kint(J, sz);
989 }
990 trcd = emitir(IRTG(IR_CNEW, IRT_CDATA), trid, trsz);
991 if (sz > 128 || (info & CTF_VLA)) {
992 TRef dp;
993 CTSize align;
994 special: /* Only handle bulk zero-fill for large/VLA/VLS types. */
995 if (J->base[1])
996 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init large/VLA/VLS types. */
997 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
998 if (trsz == TREF_NIL) trsz = lj_ir_kint(J, sz);
999 align = ctype_align(info);
1000 if (align < CT_MEMALIGN) align = CT_MEMALIGN;
1001 crec_fill(J, dp, trsz, lj_ir_kint(J, 0), (1u << align));
1002 } else if (J->base[1] && !J->base[2] &&
1003 !lj_cconv_multi_init(cts, d, &rd->argv[1])) {
1004 goto single_init;
1005 } else if (ctype_isarray(d->info)) {
1006 CType *dc = ctype_rawchild(cts, d); /* Array element type. */
1007 CTSize ofs, esize = dc->size;
1008 TRef sp = 0;
1009 TValue tv;
1010 TValue *sval = &tv;
1011 MSize i;
1012 tv.u64 = 0;
1013 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info)) ||
1014 esize * CREC_FILL_MAXUNROLL < sz)
1015 goto special;
1016 for (i = 1, ofs = 0; ofs < sz; ofs += esize) {
1017 TRef dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1018 lj_ir_kintp(J, ofs + sizeof(GCcdata)));
1019 if (J->base[i]) {
1020 sp = J->base[i];
1021 sval = &rd->argv[i];
1022 i++;
1023 } else if (i != 2) {
1024 sp = ctype_isnum(dc->info) ? lj_ir_kint(J, 0) : TREF_NIL;
1025 }
1026 crec_ct_tv(J, dc, dp, sp, sval);
1027 }
1028 } else if (ctype_isstruct(d->info)) {
1029 CTypeID fid;
1030 MSize i = 1;
1031 if (!J->base[1]) { /* Handle zero-fill of struct-of-NYI. */
1032 fid = d->sib;
1033 while (fid) {
1034 CType *df = ctype_get(cts, fid);
1035 fid = df->sib;
1036 if (ctype_isfield(df->info)) {
1037 CType *dc;
1038 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
1039 dc = ctype_rawchild(cts, df); /* Field type. */
1040 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
1041 ctype_isenum(dc->info)))
1042 goto special;
1043 } else if (!ctype_isconstval(df->info)) {
1044 goto special;
1045 }
1046 }
1047 }
1048 fid = d->sib;
1049 while (fid) {
1050 CType *df = ctype_get(cts, fid);
1051 fid = df->sib;
1052 if (ctype_isfield(df->info)) {
1053 CType *dc;
1054 TRef sp, dp;
1055 TValue tv;
1056 TValue *sval = &tv;
1057 setintV(&tv, 0);
1058 if (!gcref(df->name)) continue; /* Ignore unnamed fields. */
1059 dc = ctype_rawchild(cts, df); /* Field type. */
1060 if (!(ctype_isnum(dc->info) || ctype_isptr(dc->info) ||
1061 ctype_isenum(dc->info)))
1062 lj_trace_err(J, LJ_TRERR_NYICONV); /* NYI: init aggregates. */
1063 if (J->base[i]) {
1064 sp = J->base[i];
1065 sval = &rd->argv[i];
1066 i++;
1067 } else {
1068 sp = ctype_isptr(dc->info) ? TREF_NIL : lj_ir_kint(J, 0);
1069 }
1070 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd,
1071 lj_ir_kintp(J, df->size + sizeof(GCcdata)));
1072 crec_ct_tv(J, dc, dp, sp, sval);
1073 if ((d->info & CTF_UNION)) {
1074 if (d->size != dc->size) /* NYI: partial init of union. */
1075 lj_trace_err(J, LJ_TRERR_NYICONV);
1076 break;
1077 }
1078 } else if (!ctype_isconstval(df->info)) {
1079 /* NYI: init bitfields and sub-structures. */
1080 lj_trace_err(J, LJ_TRERR_NYICONV);
1081 }
1082 }
1083 } else {
1084 TRef dp;
1085 single_init:
1086 dp = emitir(IRT(IR_ADD, IRT_PTR), trcd, lj_ir_kintp(J, sizeof(GCcdata)));
1087 if (J->base[1]) {
1088 crec_ct_tv(J, d, dp, J->base[1], &rd->argv[1]);
1089 } else {
1090 TValue tv;
1091 tv.u64 = 0;
1092 crec_ct_tv(J, d, dp, lj_ir_kint(J, 0), &tv);
1093 }
1094 }
1095 }
1096 J->base[0] = trcd;
1097 /* Handle __gc metamethod. */
1098 fin = lj_ctype_meta(cts, id, MM_gc);
1099 if (fin)
1100 crec_finalizer(J, trcd, 0, fin);
1101 }
1102
1103 /* Record argument conversions. */
1104 static TRef crec_call_args(jit_State *J, RecordFFData *rd,
1105 CTState *cts, CType *ct)
1106 {
1107 TRef args[CCI_NARGS_MAX];
1108 CTypeID fid;
1109 MSize i, n;
1110 TRef tr, *base;
1111 cTValue *o;
1112 #if LJ_TARGET_X86
1113 #if LJ_ABI_WIN
1114 TRef *arg0 = NULL, *arg1 = NULL;
1115 #endif
1116 int ngpr = 0;
1117 if (ctype_cconv(ct->info) == CTCC_THISCALL)
1118 ngpr = 1;
1119 else if (ctype_cconv(ct->info) == CTCC_FASTCALL)
1120 ngpr = 2;
1121 #endif
1122
1123 /* Skip initial attributes. */
1124 fid = ct->sib;
1125 while (fid) {
1126 CType *ctf = ctype_get(cts, fid);
1127 if (!ctype_isattrib(ctf->info)) break;
1128 fid = ctf->sib;
1129 }
1130 args[0] = TREF_NIL;
1131 for (n = 0, base = J->base+1, o = rd->argv+1; *base; n++, base++, o++) {
1132 CTypeID did;
1133 CType *d;
1134
1135 if (n >= CCI_NARGS_MAX)
1136 lj_trace_err(J, LJ_TRERR_NYICALL);
1137
1138 if (fid) { /* Get argument type from field. */
1139 CType *ctf = ctype_get(cts, fid);
1140 fid = ctf->sib;
1141 lj_assertJ(ctype_isfield(ctf->info), "field expected");
1142 did = ctype_cid(ctf->info);
1143 } else {
1144 if (!(ct->info & CTF_VARARG))
1145 lj_trace_err(J, LJ_TRERR_NYICALL); /* Too many arguments. */
1146 did = lj_ccall_ctid_vararg(cts, o); /* Infer vararg type. */
1147 }
1148 d = ctype_raw(cts, did);
1149 if (!(ctype_isnum(d->info) || ctype_isptr(d->info) ||
1150 ctype_isenum(d->info)))
1151 lj_trace_err(J, LJ_TRERR_NYICALL);
1152 tr = crec_ct_tv(J, d, 0, *base, o);
1153 if (ctype_isinteger_or_bool(d->info)) {
1154 if (d->size < 4) {
1155 if ((d->info & CTF_UNSIGNED))
1156 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_U8 : IRT_U16, 0);
1157 else
1158 tr = emitconv(tr, IRT_INT, d->size==1 ? IRT_I8 : IRT_I16,IRCONV_SEXT);
1159 }
1160 } else if (LJ_SOFTFP32 && ctype_isfp(d->info) && d->size > 4) {
1161 lj_needsplit(J);
1162 }
1163 #if LJ_TARGET_X86
1164 /* 64 bit args must not end up in registers for fastcall/thiscall. */
1165 #if LJ_ABI_WIN
1166 if (!ctype_isfp(d->info)) {
1167 /* Sigh, the Windows/x86 ABI allows reordering across 64 bit args. */
1168 if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1169 if (ngpr) {
1170 arg0 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1171 if (ngpr) {
1172 arg1 = &args[n]; args[n++] = TREF_NIL; ngpr--;
1173 }
1174 }
1175 } else {
1176 if (arg0) { *arg0 = tr; arg0 = NULL; n--; continue; }
1177 if (arg1) { *arg1 = tr; arg1 = NULL; n--; continue; }
1178 if (ngpr) ngpr--;
1179 }
1180 }
1181 #else
1182 if (!ctype_isfp(d->info) && ngpr) {
1183 if (tref_typerange(tr, IRT_I64, IRT_U64)) {
1184 /* No reordering for other x86 ABIs. Simply add alignment args. */
1185 do { args[n++] = TREF_NIL; } while (--ngpr);
1186 } else {
1187 ngpr--;
1188 }
1189 }
1190 #endif
1191 #endif
1192 args[n] = tr;
1193 }
1194 tr = args[0];
1195 for (i = 1; i < n; i++)
1196 tr = emitir(IRT(IR_CARG, IRT_NIL), tr, args[i]);
1197 return tr;
1198 }
1199
1200 /* Create a snapshot for the caller, simulating a 'false' return value. */
1201 static void crec_snap_caller(jit_State *J)
1202 {
1203 lua_State *L = J->L;
1204 TValue *base = L->base, *top = L->top;
1205 const BCIns *pc = J->pc;
1206 TRef ftr = J->base[-1-LJ_FR2];
1207 ptrdiff_t delta;
1208 if (!frame_islua(base-1) || J->framedepth <= 0)
1209 lj_trace_err(J, LJ_TRERR_NYICALL);
1210 J->pc = frame_pc(base-1); delta = 1+LJ_FR2+bc_a(J->pc[-1]);
1211 L->top = base; L->base = base - delta;
1212 J->base[-1-LJ_FR2] = TREF_FALSE;
1213 J->base -= delta; J->baseslot -= (BCReg)delta;
1214 J->maxslot = (BCReg)delta-LJ_FR2; J->framedepth--;
1215 lj_snap_add(J);
1216 L->base = base; L->top = top;
1217 J->framedepth++; J->maxslot = 1;
1218 J->base += delta; J->baseslot += (BCReg)delta;
1219 J->base[-1-LJ_FR2] = ftr; J->pc = pc;
1220 }
1221
1222 /* Record function call. */
1223 static int crec_call(jit_State *J, RecordFFData *rd, GCcdata *cd)
1224 {
1225 CTState *cts = ctype_ctsG(J2G(J));
1226 CType *ct = ctype_raw(cts, cd->ctypeid);
1227 IRType tp = IRT_PTR;
1228 if (ctype_isptr(ct->info)) {
1229 tp = (LJ_64 && ct->size == 8) ? IRT_P64 : IRT_P32;
1230 ct = ctype_rawchild(cts, ct);
1231 }
1232 if (ctype_isfunc(ct->info)) {
1233 TRef func = emitir(IRT(IR_FLOAD, tp), J->base[0], IRFL_CDATA_PTR);
1234 CType *ctr = ctype_rawchild(cts, ct);
1235 IRType t = crec_ct2irt(cts, ctr);
1236 TRef tr;
1237 TValue tv;
1238 /* Check for blacklisted C functions that might call a callback. */
1239 tv.u64 = ((uintptr_t)cdata_getptr(cdataptr(cd), (LJ_64 && tp == IRT_P64) ? 8 : 4) >> 2) | U64x(800000000, 00000000);
1240 if (tvistrue(lj_tab_get(J->L, cts->miscmap, &tv)))
1241 lj_trace_err(J, LJ_TRERR_BLACKL);
1242 if (ctype_isvoid(ctr->info)) {
1243 t = IRT_NIL;
1244 rd->nres = 0;
1245 } else if (!(ctype_isnum(ctr->info) || ctype_isptr(ctr->info) ||
1246 ctype_isenum(ctr->info)) || t == IRT_CDATA) {
1247 lj_trace_err(J, LJ_TRERR_NYICALL);
1248 }
1249 if ((ct->info & CTF_VARARG)
1250 #if LJ_TARGET_X86
1251 || ctype_cconv(ct->info) != CTCC_CDECL
1252 #endif
1253 )
1254 func = emitir(IRT(IR_CARG, IRT_NIL), func,
1255 lj_ir_kint(J, ctype_typeid(cts, ct)));
1256 tr = emitir(IRT(IR_CALLXS, t), crec_call_args(J, rd, cts, ct), func);
1257 if (ctype_isbool(ctr->info)) {
1258 if (frame_islua(J->L->base-1) && bc_b(frame_pc(J->L->base-1)[-1]) == 1) {
1259 /* Don't check result if ignored. */
1260 tr = TREF_NIL;
1261 } else {
1262 crec_snap_caller(J);
1263 #if LJ_TARGET_X86ORX64
1264 /* Note: only the x86/x64 backend supports U8 and only for EQ(tr, 0). */
1265 lj_ir_set(J, IRTG(IR_NE, IRT_U8), tr, lj_ir_kint(J, 0));
1266 #else
1267 lj_ir_set(J, IRTGI(IR_NE), tr, lj_ir_kint(J, 0));
1268 #endif
1269 J->postproc = LJ_POST_FIXGUARDSNAP;
1270 tr = TREF_TRUE;
1271 }
1272 } else if (t == IRT_PTR || (LJ_64 && t == IRT_P32) ||
1273 t == IRT_I64 || t == IRT_U64 || ctype_isenum(ctr->info)) {
1274 TRef trid = lj_ir_kint(J, ctype_cid(ct->info));
1275 tr = emitir(IRTG(IR_CNEWI, IRT_CDATA), trid, tr);
1276 if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1277 } else if (t == IRT_FLOAT || t == IRT_U32) {
1278 tr = emitconv(tr, IRT_NUM, t, 0);
1279 } else if (t == IRT_I8 || t == IRT_I16) {
1280 tr = emitconv(tr, IRT_INT, t, IRCONV_SEXT);
1281 } else if (t == IRT_U8 || t == IRT_U16) {
1282 tr = emitconv(tr, IRT_INT, t, 0);
1283 }
1284 J->base[0] = tr;
1285 J->needsnap = 1;
1286 return 1;
1287 }
1288 return 0;
1289 }
1290
1291 void LJ_FASTCALL recff_cdata_call(jit_State *J, RecordFFData *rd)
1292 {
1293 CTState *cts = ctype_ctsG(J2G(J));
1294 GCcdata *cd = argv2cdata(J, J->base[0], &rd->argv[0]);
1295 CTypeID id = cd->ctypeid;
1296 CType *ct;
1297 cTValue *tv;
1298 MMS mm = MM_call;
1299 if (id == CTID_CTYPEID) {
1300 id = crec_constructor(J, cd, J->base[0]);
1301 mm = MM_new;
1302 } else if (crec_call(J, rd, cd)) {
1303 return;
1304 }
1305 /* Record ctype __call/__new metamethod. */
1306 ct = ctype_raw(cts, id);
1307 tv = lj_ctype_meta(cts, ctype_isptr(ct->info) ? ctype_cid(ct->info) : id, mm);
1308 if (tv) {
1309 if (tvisfunc(tv)) {
1310 crec_tailcall(J, rd, tv);
1311 return;
1312 }
1313 } else if (mm == MM_new) {
1314 crec_alloc(J, rd, id);
1315 return;
1316 }
1317 /* No metamethod or NYI: non-function metamethods. */
1318 lj_trace_err(J, LJ_TRERR_BADTYPE);
1319 }
1320
1321 static TRef crec_arith_int64(jit_State *J, TRef *sp, CType **s, MMS mm)
1322 {
1323 if (sp[0] && sp[1] && ctype_isnum(s[0]->info) && ctype_isnum(s[1]->info)) {
1324 IRType dt;
1325 CTypeID id;
1326 TRef tr;
1327 MSize i;
1328 IROp op;
1329 lj_needsplit(J);
1330 if (((s[0]->info & CTF_UNSIGNED) && s[0]->size == 8) ||
1331 ((s[1]->info & CTF_UNSIGNED) && s[1]->size == 8)) {
1332 dt = IRT_U64; id = CTID_UINT64;
1333 } else {
1334 dt = IRT_I64; id = CTID_INT64;
1335 if (mm < MM_add &&
1336 !((s[0]->info | s[1]->info) & CTF_FP) &&
1337 s[0]->size == 4 && s[1]->size == 4) { /* Try to narrow comparison. */
1338 if (!((s[0]->info ^ s[1]->info) & CTF_UNSIGNED) ||
1339 (tref_isk(sp[1]) && IR(tref_ref(sp[1]))->i >= 0)) {
1340 dt = (s[0]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1341 goto comp;
1342 } else if (tref_isk(sp[0]) && IR(tref_ref(sp[0]))->i >= 0) {
1343 dt = (s[1]->info & CTF_UNSIGNED) ? IRT_U32 : IRT_INT;
1344 goto comp;
1345 }
1346 }
1347 }
1348 for (i = 0; i < 2; i++) {
1349 IRType st = tref_type(sp[i]);
1350 if (st == IRT_NUM || st == IRT_FLOAT)
1351 sp[i] = emitconv(sp[i], dt, st, IRCONV_ANY);
1352 else if (!(st == IRT_I64 || st == IRT_U64))
1353 sp[i] = emitconv(sp[i], dt, IRT_INT,
1354 (s[i]->info & CTF_UNSIGNED) ? 0 : IRCONV_SEXT);
1355 }
1356 if (mm < MM_add) {
1357 comp:
1358 /* Assume true comparison. Fixup and emit pending guard later. */
1359 if (mm == MM_eq) {
1360 op = IR_EQ;
1361 } else {
1362 op = mm == MM_lt ? IR_LT : IR_LE;
1363 if (dt == IRT_U32 || dt == IRT_U64)
1364 op += (IR_ULT-IR_LT);
1365 }
1366 lj_ir_set(J, IRTG(op, dt), sp[0], sp[1]);
1367 J->postproc = LJ_POST_FIXGUARD;
1368 return TREF_TRUE;
1369 } else {
1370 tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, dt), sp[0], sp[1]);
1371 }
1372 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1373 }
1374 return 0;
1375 }
1376
1377 static TRef crec_arith_ptr(jit_State *J, TRef *sp, CType **s, MMS mm)
1378 {
1379 CTState *cts = ctype_ctsG(J2G(J));
1380 CType *ctp = s[0];
1381 if (!(sp[0] && sp[1])) return 0;
1382 if (ctype_isptr(ctp->info) || ctype_isrefarray(ctp->info)) {
1383 if ((mm == MM_sub || mm == MM_eq || mm == MM_lt || mm == MM_le) &&
1384 (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1385 if (mm == MM_sub) { /* Pointer difference. */
1386 TRef tr;
1387 CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1388 if (sz == 0 || (sz & (sz-1)) != 0)
1389 return 0; /* NYI: integer division. */
1390 tr = emitir(IRT(IR_SUB, IRT_INTP), sp[0], sp[1]);
1391 tr = emitir(IRT(IR_BSAR, IRT_INTP), tr, lj_ir_kint(J, lj_fls(sz)));
1392 #if LJ_64
1393 tr = emitconv(tr, IRT_NUM, IRT_INTP, 0);
1394 #endif
1395 return tr;
1396 } else { /* Pointer comparison (unsigned). */
1397 /* Assume true comparison. Fixup and emit pending guard later. */
1398 IROp op = mm == MM_eq ? IR_EQ : mm == MM_lt ? IR_ULT : IR_ULE;
1399 lj_ir_set(J, IRTG(op, IRT_PTR), sp[0], sp[1]);
1400 J->postproc = LJ_POST_FIXGUARD;
1401 return TREF_TRUE;
1402 }
1403 }
1404 if (!((mm == MM_add || mm == MM_sub) && ctype_isnum(s[1]->info)))
1405 return 0;
1406 } else if (mm == MM_add && ctype_isnum(ctp->info) &&
1407 (ctype_isptr(s[1]->info) || ctype_isrefarray(s[1]->info))) {
1408 TRef tr = sp[0]; sp[0] = sp[1]; sp[1] = tr; /* Swap pointer and index. */
1409 ctp = s[1];
1410 } else {
1411 return 0;
1412 }
1413 {
1414 TRef tr = sp[1];
1415 IRType t = tref_type(tr);
1416 CTSize sz = lj_ctype_size(cts, ctype_cid(ctp->info));
1417 CTypeID id;
1418 #if LJ_64
1419 if (t == IRT_NUM || t == IRT_FLOAT)
1420 tr = emitconv(tr, IRT_INTP, t, IRCONV_ANY);
1421 else if (!(t == IRT_I64 || t == IRT_U64))
1422 tr = emitconv(tr, IRT_INTP, IRT_INT,
1423 ((t - IRT_I8) & 1) ? 0 : IRCONV_SEXT);
1424 #else
1425 if (!tref_typerange(sp[1], IRT_I8, IRT_U32)) {
1426 tr = emitconv(tr, IRT_INTP, t,
1427 (t == IRT_NUM || t == IRT_FLOAT) ? IRCONV_ANY : 0);
1428 }
1429 #endif
1430 tr = emitir(IRT(IR_MUL, IRT_INTP), tr, lj_ir_kintp(J, sz));
1431 tr = emitir(IRT(mm+(int)IR_ADD-(int)MM_add, IRT_PTR), sp[0], tr);
1432 id = lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|ctype_cid(ctp->info)),
1433 CTSIZE_PTR);
1434 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1435 }
1436 }
1437
1438 /* Record ctype arithmetic metamethods. */
1439 static TRef crec_arith_meta(jit_State *J, TRef *sp, CType **s, CTState *cts,
1440 RecordFFData *rd)
1441 {
1442 cTValue *tv = NULL;
1443 if (J->base[0]) {
1444 if (tviscdata(&rd->argv[0])) {
1445 CTypeID id = argv2cdata(J, J->base[0], &rd->argv[0])->ctypeid;
1446 CType *ct = ctype_raw(cts, id);
1447 if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1448 tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1449 }
1450 if (!tv && J->base[1] && tviscdata(&rd->argv[1])) {
1451 CTypeID id = argv2cdata(J, J->base[1], &rd->argv[1])->ctypeid;
1452 CType *ct = ctype_raw(cts, id);
1453 if (ctype_isptr(ct->info)) id = ctype_cid(ct->info);
1454 tv = lj_ctype_meta(cts, id, (MMS)rd->data);
1455 }
1456 }
1457 if (tv) {
1458 if (tvisfunc(tv)) {
1459 crec_tailcall(J, rd, tv);
1460 return 0;
1461 } /* NYI: non-function metamethods. */
1462 } else if ((MMS)rd->data == MM_eq) { /* Fallback cdata pointer comparison. */
1463 if (sp[0] && sp[1] && ctype_isnum(s[0]->info) == ctype_isnum(s[1]->info)) {
1464 /* Assume true comparison. Fixup and emit pending guard later. */
1465 lj_ir_set(J, IRTG(IR_EQ, IRT_PTR), sp[0], sp[1]);
1466 J->postproc = LJ_POST_FIXGUARD;
1467 return TREF_TRUE;
1468 } else {
1469 return TREF_FALSE;
1470 }
1471 }
1472 lj_trace_err(J, LJ_TRERR_BADTYPE);
1473 return 0;
1474 }
1475
1476 void LJ_FASTCALL recff_cdata_arith(jit_State *J, RecordFFData *rd)
1477 {
1478 CTState *cts = ctype_ctsG(J2G(J));
1479 TRef sp[2];
1480 CType *s[2];
1481 MSize i;
1482 for (i = 0; i < 2; i++) {
1483 TRef tr = J->base[i];
1484 CType *ct = ctype_get(cts, CTID_DOUBLE);
1485 if (!tr) {
1486 lj_trace_err(J, LJ_TRERR_BADTYPE);
1487 } else if (tref_iscdata(tr)) {
1488 CTypeID id = argv2cdata(J, tr, &rd->argv[i])->ctypeid;
1489 IRType t;
1490 ct = ctype_raw(cts, id);
1491 t = crec_ct2irt(cts, ct);
1492 if (ctype_isptr(ct->info)) { /* Resolve pointer or reference. */
1493 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_PTR);
1494 if (ctype_isref(ct->info)) {
1495 ct = ctype_rawchild(cts, ct);
1496 t = crec_ct2irt(cts, ct);
1497 }
1498 } else if (t == IRT_I64 || t == IRT_U64) {
1499 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT64);
1500 lj_needsplit(J);
1501 goto ok;
1502 } else if (t == IRT_INT || t == IRT_U32) {
1503 tr = emitir(IRT(IR_FLOAD, t), tr, IRFL_CDATA_INT);
1504 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1505 goto ok;
1506 } else if (ctype_isfunc(ct->info)) {
1507 CTypeID id0 = i ? ctype_typeid(cts, s[0]) : 0;
1508 tr = emitir(IRT(IR_FLOAD, IRT_PTR), tr, IRFL_CDATA_PTR);
1509 ct = ctype_get(cts,
1510 lj_ctype_intern(cts, CTINFO(CT_PTR, CTALIGN_PTR|id), CTSIZE_PTR));
1511 if (i) {
1512 s[0] = ctype_get(cts, id0); /* cts->tab may have been reallocated. */
1513 }
1514 goto ok;
1515 } else {
1516 tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCcdata)));
1517 }
1518 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1519 if (ctype_isnum(ct->info)) {
1520 if (t == IRT_CDATA) {
1521 tr = 0;
1522 } else {
1523 if (t == IRT_I64 || t == IRT_U64) lj_needsplit(J);
1524 tr = emitir(IRT(IR_XLOAD, t), tr, 0);
1525 }
1526 }
1527 } else if (tref_isnil(tr)) {
1528 tr = lj_ir_kptr(J, NULL);
1529 ct = ctype_get(cts, CTID_P_VOID);
1530 } else if (tref_isinteger(tr)) {
1531 ct = ctype_get(cts, CTID_INT32);
1532 } else if (tref_isstr(tr)) {
1533 TRef tr2 = J->base[1-i];
1534 CTypeID id = argv2cdata(J, tr2, &rd->argv[1-i])->ctypeid;
1535 ct = ctype_raw(cts, id);
1536 if (ctype_isenum(ct->info)) { /* Match string against enum constant. */
1537 GCstr *str = strV(&rd->argv[i]);
1538 CTSize ofs;
1539 CType *cct = lj_ctype_getfield(cts, ct, str, &ofs);
1540 if (cct && ctype_isconstval(cct->info)) {
1541 /* Specialize to the name of the enum constant. */
1542 emitir(IRTG(IR_EQ, IRT_STR), tr, lj_ir_kstr(J, str));
1543 ct = ctype_child(cts, cct);
1544 tr = lj_ir_kint(J, (int32_t)ofs);
1545 } else { /* Interpreter will throw or return false. */
1546 ct = ctype_get(cts, CTID_P_VOID);
1547 }
1548 } else if (ctype_isptr(ct->info)) {
1549 tr = emitir(IRT(IR_ADD, IRT_PTR), tr, lj_ir_kintp(J, sizeof(GCstr)));
1550 } else {
1551 ct = ctype_get(cts, CTID_P_VOID);
1552 }
1553 } else if (!tref_isnum(tr)) {
1554 tr = 0;
1555 ct = ctype_get(cts, CTID_P_VOID);
1556 }
1557 ok:
1558 s[i] = ct;
1559 sp[i] = tr;
1560 }
1561 {
1562 TRef tr;
1563 MMS mm = (MMS)rd->data;
1564 if ((mm == MM_len || mm == MM_concat ||
1565 (!(tr = crec_arith_int64(J, sp, s, mm)) &&
1566 !(tr = crec_arith_ptr(J, sp, s, mm)))) &&
1567 !(tr = crec_arith_meta(J, sp, s, cts, rd)))
1568 return;
1569 J->base[0] = tr;
1570 /* Fixup cdata comparisons, too. Avoids some cdata escapes. */
1571 if (J->postproc == LJ_POST_FIXGUARD && frame_iscont(J->L->base-1) &&
1572 !irt_isguard(J->guardemit)) {
1573 const BCIns *pc = frame_contpc(J->L->base-1) - 1;
1574 if (bc_op(*pc) <= BC_ISNEP) {
1575 J2G(J)->tmptv.u64 = (uint64_t)(uintptr_t)pc;
1576 J->postproc = LJ_POST_FIXCOMP;
1577 }
1578 }
1579 }
1580 }
1581
1582 /* -- C library namespace metamethods ------------------------------------- */
1583
1584 void LJ_FASTCALL recff_clib_index(jit_State *J, RecordFFData *rd)
1585 {
1586 CTState *cts = ctype_ctsG(J2G(J));
1587 if (tref_isudata(J->base[0]) && tref_isstr(J->base[1]) &&
1588 udataV(&rd->argv[0])->udtype == UDTYPE_FFI_CLIB) {
1589 CLibrary *cl = (CLibrary *)uddata(udataV(&rd->argv[0]));
1590 GCstr *name = strV(&rd->argv[1]);
1591 CType *ct;
1592 CTypeID id = lj_ctype_getname(cts, &ct, name, CLNS_INDEX);
1593 cTValue *tv = lj_tab_getstr(cl->cache, name);
1594 rd->nres = rd->data;
1595 if (id && tv && !tvisnil(tv)) {
1596 /* Specialize to the symbol name and make the result a constant. */
1597 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, name));
1598 if (ctype_isconstval(ct->info)) {
1599 if (ct->size >= 0x80000000u &&
1600 (ctype_child(cts, ct)->info & CTF_UNSIGNED))
1601 J->base[0] = lj_ir_knum(J, (lua_Number)(uint32_t)ct->size);
1602 else
1603 J->base[0] = lj_ir_kint(J, (int32_t)ct->size);
1604 } else if (ctype_isextern(ct->info)) {
1605 CTypeID sid = ctype_cid(ct->info);
1606 void *sp = *(void **)cdataptr(cdataV(tv));
1607 TRef ptr;
1608 ct = ctype_raw(cts, sid);
1609 if (LJ_64 && !checkptr32(sp))
1610 ptr = lj_ir_kintp(J, (uintptr_t)sp);
1611 else
1612 ptr = lj_ir_kptr(J, sp);
1613 if (rd->data) {
1614 J->base[0] = crec_tv_ct(J, ct, sid, ptr);
1615 } else {
1616 J->needsnap = 1;
1617 crec_ct_tv(J, ct, ptr, J->base[2], &rd->argv[2]);
1618 }
1619 } else {
1620 J->base[0] = lj_ir_kgc(J, obj2gco(cdataV(tv)), IRT_CDATA);
1621 }
1622 } else {
1623 lj_trace_err(J, LJ_TRERR_NOCACHE);
1624 }
1625 } /* else: interpreter will throw. */
1626 }
1627
1628 /* -- FFI library functions ----------------------------------------------- */
1629
1630 static TRef crec_toint(jit_State *J, CTState *cts, TRef sp, TValue *sval)
1631 {
1632 return crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, sp, sval);
1633 }
1634
1635 void LJ_FASTCALL recff_ffi_new(jit_State *J, RecordFFData *rd)
1636 {
1637 crec_alloc(J, rd, argv2ctype(J, J->base[0], &rd->argv[0]));
1638 }
1639
1640 void LJ_FASTCALL recff_ffi_errno(jit_State *J, RecordFFData *rd)
1641 {
1642 UNUSED(rd);
1643 if (J->base[0])
1644 lj_trace_err(J, LJ_TRERR_NYICALL);
1645 J->base[0] = lj_ir_call(J, IRCALL_lj_vm_errno);
1646 }
1647
1648 void LJ_FASTCALL recff_ffi_string(jit_State *J, RecordFFData *rd)
1649 {
1650 CTState *cts = ctype_ctsG(J2G(J));
1651 TRef tr = J->base[0];
1652 if (tr) {
1653 TRef trlen = J->base[1];
1654 if (!tref_isnil(trlen)) {
1655 trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1656 tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, &rd->argv[0]);
1657 } else {
1658 tr = crec_ct_tv(J, ctype_get(cts, CTID_P_CCHAR), 0, tr, &rd->argv[0]);
1659 trlen = lj_ir_call(J, IRCALL_strlen, tr);
1660 }
1661 J->base[0] = emitir(IRT(IR_XSNEW, IRT_STR), tr, trlen);
1662 } /* else: interpreter will throw. */
1663 }
1664
1665 void LJ_FASTCALL recff_ffi_copy(jit_State *J, RecordFFData *rd)
1666 {
1667 CTState *cts = ctype_ctsG(J2G(J));
1668 TRef trdst = J->base[0], trsrc = J->base[1], trlen = J->base[2];
1669 if (trdst && trsrc && (trlen || tref_isstr(trsrc))) {
1670 trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1671 trsrc = crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, trsrc, &rd->argv[1]);
1672 if (trlen) {
1673 trlen = crec_toint(J, cts, trlen, &rd->argv[2]);
1674 } else {
1675 trlen = emitir(IRTI(IR_FLOAD), J->base[1], IRFL_STR_LEN);
1676 trlen = emitir(IRTI(IR_ADD), trlen, lj_ir_kint(J, 1));
1677 }
1678 rd->nres = 0;
1679 crec_copy(J, trdst, trsrc, trlen, NULL);
1680 } /* else: interpreter will throw. */
1681 }
1682
1683 void LJ_FASTCALL recff_ffi_fill(jit_State *J, RecordFFData *rd)
1684 {
1685 CTState *cts = ctype_ctsG(J2G(J));
1686 TRef trdst = J->base[0], trlen = J->base[1], trfill = J->base[2];
1687 if (trdst && trlen) {
1688 CTSize step = 1;
1689 if (tviscdata(&rd->argv[0])) { /* Get alignment of original destination. */
1690 CTSize sz;
1691 CType *ct = ctype_raw(cts, cdataV(&rd->argv[0])->ctypeid);
1692 if (ctype_isptr(ct->info))
1693 ct = ctype_rawchild(cts, ct);
1694 step = (1u<<ctype_align(lj_ctype_info(cts, ctype_typeid(cts, ct), &sz)));
1695 }
1696 trdst = crec_ct_tv(J, ctype_get(cts, CTID_P_VOID), 0, trdst, &rd->argv[0]);
1697 trlen = crec_toint(J, cts, trlen, &rd->argv[1]);
1698 if (trfill)
1699 trfill = crec_toint(J, cts, trfill, &rd->argv[2]);
1700 else
1701 trfill = lj_ir_kint(J, 0);
1702 rd->nres = 0;
1703 crec_fill(J, trdst, trlen, trfill, step);
1704 } /* else: interpreter will throw. */
1705 }
1706
1707 void LJ_FASTCALL recff_ffi_typeof(jit_State *J, RecordFFData *rd)
1708 {
1709 if (tref_iscdata(J->base[0])) {
1710 TRef trid = lj_ir_kint(J, argv2ctype(J, J->base[0], &rd->argv[0]));
1711 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA),
1712 lj_ir_kint(J, CTID_CTYPEID), trid);
1713 } else {
1714 setfuncV(J->L, &J->errinfo, J->fn);
1715 lj_trace_err_info(J, LJ_TRERR_NYIFFU);
1716 }
1717 }
1718
1719 void LJ_FASTCALL recff_ffi_istype(jit_State *J, RecordFFData *rd)
1720 {
1721 argv2ctype(J, J->base[0], &rd->argv[0]);
1722 if (tref_iscdata(J->base[1])) {
1723 argv2ctype(J, J->base[1], &rd->argv[1]);
1724 J->postproc = LJ_POST_FIXBOOL;
1725 J->base[0] = TREF_TRUE;
1726 } else {
1727 J->base[0] = TREF_FALSE;
1728 }
1729 }
1730
1731 void LJ_FASTCALL recff_ffi_abi(jit_State *J, RecordFFData *rd)
1732 {
1733 if (tref_isstr(J->base[0])) {
1734 /* Specialize to the ABI string to make the boolean result a constant. */
1735 emitir(IRTG(IR_EQ, IRT_STR), J->base[0], lj_ir_kstr(J, strV(&rd->argv[0])));
1736 J->postproc = LJ_POST_FIXBOOL;
1737 J->base[0] = TREF_TRUE;
1738 } else {
1739 lj_trace_err(J, LJ_TRERR_BADTYPE);
1740 }
1741 }
1742
1743 /* Record ffi.sizeof(), ffi.alignof(), ffi.offsetof(). */
1744 void LJ_FASTCALL recff_ffi_xof(jit_State *J, RecordFFData *rd)
1745 {
1746 CTypeID id = argv2ctype(J, J->base[0], &rd->argv[0]);
1747 if (rd->data == FF_ffi_sizeof) {
1748 CType *ct = lj_ctype_rawref(ctype_ctsG(J2G(J)), id);
1749 if (ctype_isvltype(ct->info))
1750 lj_trace_err(J, LJ_TRERR_BADTYPE);
1751 } else if (rd->data == FF_ffi_offsetof) { /* Specialize to the field name. */
1752 if (!tref_isstr(J->base[1]))
1753 lj_trace_err(J, LJ_TRERR_BADTYPE);
1754 emitir(IRTG(IR_EQ, IRT_STR), J->base[1], lj_ir_kstr(J, strV(&rd->argv[1])));
1755 rd->nres = 3; /* Just in case. */
1756 }
1757 J->postproc = LJ_POST_FIXCONST;
1758 J->base[0] = J->base[1] = J->base[2] = TREF_NIL;
1759 }
1760
1761 void LJ_FASTCALL recff_ffi_gc(jit_State *J, RecordFFData *rd)
1762 {
1763 argv2cdata(J, J->base[0], &rd->argv[0]);
1764 if (!J->base[1])
1765 lj_trace_err(J, LJ_TRERR_BADTYPE);
1766 crec_finalizer(J, J->base[0], J->base[1], &rd->argv[1]);
1767 }
1768
1769 /* -- 64 bit bit.* library functions -------------------------------------- */
1770
1771 /* Determine bit operation type from argument type. */
1772 static CTypeID crec_bit64_type(CTState *cts, cTValue *tv)
1773 {
1774 if (tviscdata(tv)) {
1775 CType *ct = lj_ctype_rawref(cts, cdataV(tv)->ctypeid);
1776 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1777 if ((ct->info & (CTMASK_NUM|CTF_BOOL|CTF_FP|CTF_UNSIGNED)) ==
1778 CTINFO(CT_NUM, CTF_UNSIGNED) && ct->size == 8)
1779 return CTID_UINT64; /* Use uint64_t, since it has the highest rank. */
1780 return CTID_INT64; /* Otherwise use int64_t. */
1781 }
1782 return 0; /* Use regular 32 bit ops. */
1783 }
1784
1785 void LJ_FASTCALL recff_bit64_tobit(jit_State *J, RecordFFData *rd)
1786 {
1787 CTState *cts = ctype_ctsG(J2G(J));
1788 TRef tr = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1789 J->base[0], &rd->argv[0]);
1790 if (!tref_isinteger(tr))
1791 tr = emitconv(tr, IRT_INT, tref_type(tr), 0);
1792 J->base[0] = tr;
1793 }
1794
1795 int LJ_FASTCALL recff_bit64_unary(jit_State *J, RecordFFData *rd)
1796 {
1797 CTState *cts = ctype_ctsG(J2G(J));
1798 CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1799 if (id) {
1800 TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1801 tr = emitir(IRT(rd->data, id-CTID_INT64+IRT_I64), tr, 0);
1802 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1803 return 1;
1804 }
1805 return 0;
1806 }
1807
1808 int LJ_FASTCALL recff_bit64_nary(jit_State *J, RecordFFData *rd)
1809 {
1810 CTState *cts = ctype_ctsG(J2G(J));
1811 CTypeID id = 0;
1812 MSize i;
1813 for (i = 0; J->base[i] != 0; i++) {
1814 CTypeID aid = crec_bit64_type(cts, &rd->argv[i]);
1815 if (id < aid) id = aid; /* Determine highest type rank of all arguments. */
1816 }
1817 if (id) {
1818 CType *ct = ctype_get(cts, id);
1819 uint32_t ot = IRT(rd->data, id-CTID_INT64+IRT_I64);
1820 TRef tr = crec_ct_tv(J, ct, 0, J->base[0], &rd->argv[0]);
1821 for (i = 1; J->base[i] != 0; i++) {
1822 TRef tr2 = crec_ct_tv(J, ct, 0, J->base[i], &rd->argv[i]);
1823 tr = emitir(ot, tr, tr2);
1824 }
1825 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1826 return 1;
1827 }
1828 return 0;
1829 }
1830
1831 int LJ_FASTCALL recff_bit64_shift(jit_State *J, RecordFFData *rd)
1832 {
1833 CTState *cts = ctype_ctsG(J2G(J));
1834 CTypeID id;
1835 TRef tsh = 0;
1836 if (J->base[0] && tref_iscdata(J->base[1])) {
1837 tsh = crec_ct_tv(J, ctype_get(cts, CTID_INT64), 0,
1838 J->base[1], &rd->argv[1]);
1839 if (!tref_isinteger(tsh))
1840 tsh = emitconv(tsh, IRT_INT, tref_type(tsh), 0);
1841 J->base[1] = tsh;
1842 }
1843 id = crec_bit64_type(cts, &rd->argv[0]);
1844 if (id) {
1845 TRef tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1846 uint32_t op = rd->data;
1847 if (!tsh) tsh = lj_opt_narrow_tobit(J, J->base[1]);
1848 if (!(op < IR_BROL ? LJ_TARGET_MASKSHIFT : LJ_TARGET_MASKROT) &&
1849 !tref_isk(tsh))
1850 tsh = emitir(IRTI(IR_BAND), tsh, lj_ir_kint(J, 63));
1851 #ifdef LJ_TARGET_UNIFYROT
1852 if (op == (LJ_TARGET_UNIFYROT == 1 ? IR_BROR : IR_BROL)) {
1853 op = LJ_TARGET_UNIFYROT == 1 ? IR_BROL : IR_BROR;
1854 tsh = emitir(IRTI(IR_NEG), tsh, tsh);
1855 }
1856 #endif
1857 tr = emitir(IRT(op, id-CTID_INT64+IRT_I64), tr, tsh);
1858 J->base[0] = emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, id), tr);
1859 return 1;
1860 }
1861 return 0;
1862 }
1863
1864 TRef recff_bit64_tohex(jit_State *J, RecordFFData *rd, TRef hdr)
1865 {
1866 CTState *cts = ctype_ctsG(J2G(J));
1867 CTypeID id = crec_bit64_type(cts, &rd->argv[0]);
1868 TRef tr, trsf = J->base[1];
1869 SFormat sf = (STRFMT_UINT|STRFMT_T_HEX);
1870 int32_t n;
1871 if (trsf) {
1872 CTypeID id2 = 0;
1873 n = (int32_t)lj_carith_check64(J->L, 2, &id2);
1874 if (id2)
1875 trsf = crec_ct_tv(J, ctype_get(cts, CTID_INT32), 0, trsf, &rd->argv[1]);
1876 else
1877 trsf = lj_opt_narrow_tobit(J, trsf);
1878 emitir(IRTGI(IR_EQ), trsf, lj_ir_kint(J, n)); /* Specialize to n. */
1879 } else {
1880 n = id ? 16 : 8;
1881 }
1882 if (n < 0) { n = (int32_t)(~n+1u); sf |= STRFMT_F_UPPER; }
1883 if ((uint32_t)n > 254) n = 254;
1884 sf |= ((SFormat)((n+1)&255) << STRFMT_SH_PREC);
1885 if (id) {
1886 tr = crec_ct_tv(J, ctype_get(cts, id), 0, J->base[0], &rd->argv[0]);
1887 if (n < 16)
1888 tr = emitir(IRT(IR_BAND, IRT_U64), tr,
1889 lj_ir_kint64(J, ((uint64_t)1 << 4*n)-1));
1890 } else {
1891 tr = lj_opt_narrow_tobit(J, J->base[0]);
1892 if (n < 8)
1893 tr = emitir(IRTI(IR_BAND), tr, lj_ir_kint(J, (int32_t)((1u << 4*n)-1)));
1894 tr = emitconv(tr, IRT_U64, IRT_INT, 0); /* No sign-extension. */
1895 lj_needsplit(J);
1896 }
1897 return lj_ir_call(J, IRCALL_lj_strfmt_putfxint, hdr, lj_ir_kint(J, sf), tr);
1898 }
1899
1900 /* -- Miscellaneous library functions ------------------------------------- */
1901
1902 void LJ_FASTCALL lj_crecord_tonumber(jit_State *J, RecordFFData *rd)
1903 {
1904 CTState *cts = ctype_ctsG(J2G(J));
1905 CType *d, *ct = lj_ctype_rawref(cts, cdataV(&rd->argv[0])->ctypeid);
1906 if (ctype_isenum(ct->info)) ct = ctype_child(cts, ct);
1907 if (ctype_isnum(ct->info) || ctype_iscomplex(ct->info)) {
1908 if (ctype_isinteger_or_bool(ct->info) && ct->size <= 4 &&
1909 !(ct->size == 4 && (ct->info & CTF_UNSIGNED)))
1910 d = ctype_get(cts, CTID_INT32);
1911 else
1912 d = ctype_get(cts, CTID_DOUBLE);
1913 J->base[0] = crec_ct_tv(J, d, 0, J->base[0], &rd->argv[0]);
1914 } else {
1915 /* Specialize to the ctype that couldn't be converted. */
1916 argv2cdata(J, J->base[0], &rd->argv[0]);
1917 J->base[0] = TREF_NIL;
1918 }
1919 }
1920
1921 TRef lj_crecord_loadiu64(jit_State *J, TRef tr, cTValue *o)
1922 {
1923 CTypeID id = argv2cdata(J, tr, o)->ctypeid;
1924 if (!(id == CTID_INT64 || id == CTID_UINT64))
1925 lj_trace_err(J, LJ_TRERR_BADTYPE);
1926 lj_needsplit(J);
1927 return emitir(IRT(IR_FLOAD, id == CTID_INT64 ? IRT_I64 : IRT_U64), tr,
1928 IRFL_CDATA_INT64);
1929 }
1930
1931 #if LJ_HASBUFFER
1932 TRef lj_crecord_topcvoid(jit_State *J, TRef tr, cTValue *o)
1933 {
1934 CTState *cts = ctype_ctsG(J2G(J));
1935 if (!tref_iscdata(tr)) lj_trace_err(J, LJ_TRERR_BADTYPE);
1936 return crec_ct_tv(J, ctype_get(cts, CTID_P_CVOID), 0, tr, o);
1937 }
1938
1939 TRef lj_crecord_topuint8(jit_State *J, TRef tr)
1940 {
1941 return emitir(IRTG(IR_CNEWI, IRT_CDATA), lj_ir_kint(J, CTID_P_UINT8), tr);
1942 }
1943 #endif
1944
1945 #undef IR
1946 #undef emitir
1947 #undef emitconv
1948
1949 #endif