Mercurial

comparison third_party/luajit/dynasm/dasm_x86.h @ 178:94705b5986b3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
[ThirdParty] Added WRK and luajit for load testing.
author MrJuneJune <me@mrjunejune.com>
date Thu, 22 Jan 2026 20:10:30 -0800
parents
children
comparison
equal deleted inserted replaced
177:24fe8ff94056 178:94705b5986b3
1 /*
2 ** DynASM x86 encoding engine.
3 ** Copyright (C) 2005-2023 Mike Pall. All rights reserved.
4 ** Released under the MIT license. See dynasm.lua for full copyright notice.
5 */
6
7 #include <stddef.h>
8 #include <stdarg.h>
9 #include <string.h>
10 #include <stdlib.h>
11
12 #define DASM_ARCH "x86"
13
14 #ifndef DASM_EXTERN
15 #define DASM_EXTERN(a,b,c,d) 0
16 #endif
17
18 /* Action definitions. DASM_STOP must be 255. */
19 enum {
20 DASM_DISP = 233,
21 DASM_IMM_S, DASM_IMM_B, DASM_IMM_W, DASM_IMM_D, DASM_IMM_WB, DASM_IMM_DB,
22 DASM_VREG, DASM_SPACE, DASM_SETLABEL, DASM_REL_A, DASM_REL_LG, DASM_REL_PC,
23 DASM_IMM_LG, DASM_IMM_PC, DASM_LABEL_LG, DASM_LABEL_PC, DASM_ALIGN,
24 DASM_EXTERN, DASM_ESC, DASM_MARK, DASM_SECTION, DASM_STOP
25 };
26
27 /* Maximum number of section buffer positions for a single dasm_put() call. */
28 #define DASM_MAXSECPOS 25
29
30 /* DynASM encoder status codes. Action list offset or number are or'ed in. */
31 #define DASM_S_OK 0x00000000
32 #define DASM_S_NOMEM 0x01000000
33 #define DASM_S_PHASE 0x02000000
34 #define DASM_S_MATCH_SEC 0x03000000
35 #define DASM_S_RANGE_I 0x11000000
36 #define DASM_S_RANGE_SEC 0x12000000
37 #define DASM_S_RANGE_LG 0x13000000
38 #define DASM_S_RANGE_PC 0x14000000
39 #define DASM_S_RANGE_VREG 0x15000000
40 #define DASM_S_UNDEF_L 0x21000000
41 #define DASM_S_UNDEF_PC 0x22000000
42
43 /* Macros to convert positions (8 bit section + 24 bit index). */
44 #define DASM_POS2IDX(pos) ((pos)&0x00ffffff)
45 #define DASM_POS2BIAS(pos) ((pos)&0xff000000)
46 #define DASM_SEC2POS(sec) ((sec)<<24)
47 #define DASM_POS2SEC(pos) ((pos)>>24)
48 #define DASM_POS2PTR(D, pos) (D->sections[DASM_POS2SEC(pos)].rbuf + (pos))
49
50 /* Action list type. */
51 typedef const unsigned char *dasm_ActList;
52
53 /* Per-section structure. */
54 typedef struct dasm_Section {
55 int *rbuf; /* Biased buffer pointer (negative section bias). */
56 int *buf; /* True buffer pointer. */
57 size_t bsize; /* Buffer size in bytes. */
58 int pos; /* Biased buffer position. */
59 int epos; /* End of biased buffer position - max single put. */
60 int ofs; /* Byte offset into section. */
61 } dasm_Section;
62
63 /* Core structure holding the DynASM encoding state. */
64 struct dasm_State {
65 size_t psize; /* Allocated size of this structure. */
66 dasm_ActList actionlist; /* Current actionlist pointer. */
67 int *lglabels; /* Local/global chain/pos ptrs. */
68 size_t lgsize;
69 int *pclabels; /* PC label chains/pos ptrs. */
70 size_t pcsize;
71 void **globals; /* Array of globals. */
72 dasm_Section *section; /* Pointer to active section. */
73 size_t codesize; /* Total size of all code sections. */
74 int maxsection; /* 0 <= sectionidx < maxsection. */
75 int status; /* Status code. */
76 dasm_Section sections[1]; /* All sections. Alloc-extended. */
77 };
78
79 /* The size of the core structure depends on the max. number of sections. */
80 #define DASM_PSZ(ms) (sizeof(dasm_State)+(ms-1)*sizeof(dasm_Section))
81
82
83 /* Initialize DynASM state. */
84 void dasm_init(Dst_DECL, int maxsection)
85 {
86 dasm_State *D;
87 size_t psz = 0;
88 Dst_REF = NULL;
89 DASM_M_GROW(Dst, struct dasm_State, Dst_REF, psz, DASM_PSZ(maxsection));
90 D = Dst_REF;
91 D->psize = psz;
92 D->lglabels = NULL;
93 D->lgsize = 0;
94 D->pclabels = NULL;
95 D->pcsize = 0;
96 D->globals = NULL;
97 D->maxsection = maxsection;
98 memset((void *)D->sections, 0, maxsection * sizeof(dasm_Section));
99 }
100
101 /* Free DynASM state. */
102 void dasm_free(Dst_DECL)
103 {
104 dasm_State *D = Dst_REF;
105 int i;
106 for (i = 0; i < D->maxsection; i++)
107 if (D->sections[i].buf)
108 DASM_M_FREE(Dst, D->sections[i].buf, D->sections[i].bsize);
109 if (D->pclabels) DASM_M_FREE(Dst, D->pclabels, D->pcsize);
110 if (D->lglabels) DASM_M_FREE(Dst, D->lglabels, D->lgsize);
111 DASM_M_FREE(Dst, D, D->psize);
112 }
113
114 /* Setup global label array. Must be called before dasm_setup(). */
115 void dasm_setupglobal(Dst_DECL, void **gl, unsigned int maxgl)
116 {
117 dasm_State *D = Dst_REF;
118 D->globals = gl;
119 DASM_M_GROW(Dst, int, D->lglabels, D->lgsize, (10+maxgl)*sizeof(int));
120 }
121
122 /* Grow PC label array. Can be called after dasm_setup(), too. */
123 void dasm_growpc(Dst_DECL, unsigned int maxpc)
124 {
125 dasm_State *D = Dst_REF;
126 size_t osz = D->pcsize;
127 DASM_M_GROW(Dst, int, D->pclabels, D->pcsize, maxpc*sizeof(int));
128 memset((void *)(((unsigned char *)D->pclabels)+osz), 0, D->pcsize-osz);
129 }
130
131 /* Setup encoder. */
132 void dasm_setup(Dst_DECL, const void *actionlist)
133 {
134 dasm_State *D = Dst_REF;
135 int i;
136 D->actionlist = (dasm_ActList)actionlist;
137 D->status = DASM_S_OK;
138 D->section = &D->sections[0];
139 memset((void *)D->lglabels, 0, D->lgsize);
140 if (D->pclabels) memset((void *)D->pclabels, 0, D->pcsize);
141 for (i = 0; i < D->maxsection; i++) {
142 D->sections[i].pos = DASM_SEC2POS(i);
143 D->sections[i].rbuf = D->sections[i].buf - D->sections[i].pos;
144 D->sections[i].ofs = 0;
145 }
146 }
147
148
149 #ifdef DASM_CHECKS
150 #define CK(x, st) \
151 do { if (!(x)) { \
152 D->status = DASM_S_##st|(int)(p-D->actionlist-1); return; } } while (0)
153 #define CKPL(kind, st) \
154 do { if ((size_t)((char *)pl-(char *)D->kind##labels) >= D->kind##size) { \
155 D->status=DASM_S_RANGE_##st|(int)(p-D->actionlist-1); return; } } while (0)
156 #else
157 #define CK(x, st) ((void)0)
158 #define CKPL(kind, st) ((void)0)
159 #endif
160
161 /* Pass 1: Store actions and args, link branches/labels, estimate offsets. */
162 void dasm_put(Dst_DECL, int start, ...)
163 {
164 va_list ap;
165 dasm_State *D = Dst_REF;
166 dasm_ActList p = D->actionlist + start;
167 dasm_Section *sec = D->section;
168 int pos = sec->pos, ofs = sec->ofs, mrm = -1;
169 int *b;
170
171 if (pos >= sec->epos) {
172 DASM_M_GROW(Dst, int, sec->buf, sec->bsize,
173 sec->bsize + 2*DASM_MAXSECPOS*sizeof(int));
174 sec->rbuf = sec->buf - DASM_POS2BIAS(pos);
175 sec->epos = (int)sec->bsize/sizeof(int) - DASM_MAXSECPOS+DASM_POS2BIAS(pos);
176 }
177
178 b = sec->rbuf;
179 b[pos++] = start;
180
181 va_start(ap, start);
182 while (1) {
183 int action = *p++;
184 if (action < DASM_DISP) {
185 ofs++;
186 } else if (action <= DASM_REL_A) {
187 int n = va_arg(ap, int);
188 b[pos++] = n;
189 switch (action) {
190 case DASM_DISP:
191 if (n == 0) { if (mrm < 0) mrm = p[-2]; if ((mrm&7) != 5) break; }
192 /* fallthrough */
193 case DASM_IMM_DB: if (((n+128)&-256) == 0) goto ob; /* fallthrough */
194 case DASM_REL_A: /* Assumes ptrdiff_t is int. !x64 */
195 case DASM_IMM_D: ofs += 4; break;
196 case DASM_IMM_S: CK(((n+128)&-256) == 0, RANGE_I); goto ob;
197 case DASM_IMM_B: CK((n&-256) == 0, RANGE_I); ob: ofs++; break;
198 case DASM_IMM_WB: if (((n+128)&-256) == 0) goto ob; /* fallthrough */
199 case DASM_IMM_W: CK((n&-65536) == 0, RANGE_I); ofs += 2; break;
200 case DASM_SPACE: p++; ofs += n; break;
201 case DASM_SETLABEL: b[pos-2] = -0x40000000; break; /* Neg. label ofs. */
202 case DASM_VREG: CK((n&-16) == 0 && (n != 4 || (*p>>5) != 2), RANGE_VREG);
203 if (*p < 0x40 && p[1] == DASM_DISP) mrm = n;
204 if (*p < 0x20 && (n&7) == 4) ofs++;
205 switch ((*p++ >> 3) & 3) {
206 case 3: n |= b[pos-3]; /* fallthrough */
207 case 2: n |= b[pos-2]; /* fallthrough */
208 case 1: if (n <= 7) { b[pos-1] |= 0x10; ofs--; }
209 }
210 continue;
211 }
212 mrm = -1;
213 } else {
214 int *pl, n;
215 switch (action) {
216 case DASM_REL_LG:
217 case DASM_IMM_LG:
218 n = *p++; pl = D->lglabels + n;
219 /* Bkwd rel or global. */
220 if (n <= 246) { CK(n>=10||*pl<0, RANGE_LG); CKPL(lg, LG); goto putrel; }
221 pl -= 246; n = *pl;
222 if (n < 0) n = 0; /* Start new chain for fwd rel if label exists. */
223 goto linkrel;
224 case DASM_REL_PC:
225 case DASM_IMM_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC);
226 putrel:
227 n = *pl;
228 if (n < 0) { /* Label exists. Get label pos and store it. */
229 b[pos] = -n;
230 } else {
231 linkrel:
232 b[pos] = n; /* Else link to rel chain, anchored at label. */
233 *pl = pos;
234 }
235 pos++;
236 ofs += 4; /* Maximum offset needed. */
237 if (action == DASM_REL_LG || action == DASM_REL_PC) {
238 b[pos++] = ofs; /* Store pass1 offset estimate. */
239 } else if (sizeof(ptrdiff_t) == 8) {
240 ofs += 4;
241 }
242 break;
243 case DASM_LABEL_LG: pl = D->lglabels + *p++; CKPL(lg, LG); goto putlabel;
244 case DASM_LABEL_PC: pl = D->pclabels + va_arg(ap, int); CKPL(pc, PC);
245 putlabel:
246 n = *pl; /* n > 0: Collapse rel chain and replace with label pos. */
247 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = pos; }
248 *pl = -pos; /* Label exists now. */
249 b[pos++] = ofs; /* Store pass1 offset estimate. */
250 break;
251 case DASM_ALIGN:
252 ofs += *p++; /* Maximum alignment needed (arg is 2**n-1). */
253 b[pos++] = ofs; /* Store pass1 offset estimate. */
254 break;
255 case DASM_EXTERN: p += 2; ofs += 4; break;
256 case DASM_ESC: p++; ofs++; break;
257 case DASM_MARK: mrm = p[-2]; break;
258 case DASM_SECTION:
259 n = *p; CK(n < D->maxsection, RANGE_SEC); D->section = &D->sections[n];
260 case DASM_STOP: goto stop;
261 }
262 }
263 }
264 stop:
265 va_end(ap);
266 sec->pos = pos;
267 sec->ofs = ofs;
268 }
269 #undef CK
270
271 /* Pass 2: Link sections, shrink branches/aligns, fix label offsets. */
272 int dasm_link(Dst_DECL, size_t *szp)
273 {
274 dasm_State *D = Dst_REF;
275 int secnum;
276 int ofs = 0;
277
278 #ifdef DASM_CHECKS
279 *szp = 0;
280 if (D->status != DASM_S_OK) return D->status;
281 {
282 int pc;
283 for (pc = 0; pc*sizeof(int) < D->pcsize; pc++)
284 if (D->pclabels[pc] > 0) return DASM_S_UNDEF_PC|pc;
285 }
286 #endif
287
288 { /* Handle globals not defined in this translation unit. */
289 int idx;
290 for (idx = 10; idx*sizeof(int) < D->lgsize; idx++) {
291 int n = D->lglabels[idx];
292 /* Undefined label: Collapse rel chain and replace with marker (< 0). */
293 while (n > 0) { int *pb = DASM_POS2PTR(D, n); n = *pb; *pb = -idx; }
294 }
295 }
296
297 /* Combine all code sections. No support for data sections (yet). */
298 for (secnum = 0; secnum < D->maxsection; secnum++) {
299 dasm_Section *sec = D->sections + secnum;
300 int *b = sec->rbuf;
301 int pos = DASM_SEC2POS(secnum);
302 int lastpos = sec->pos;
303
304 while (pos != lastpos) {
305 dasm_ActList p = D->actionlist + b[pos++];
306 int op = 0;
307 while (1) {
308 int action = *p++;
309 switch (action) {
310 case DASM_REL_LG: p++;
311 /* fallthrough */
312 case DASM_REL_PC: {
313 int shrink = op == 0xe9 ? 3 : ((op&0xf0) == 0x80 ? 4 : 0);
314 if (shrink) { /* Shrinkable branch opcode? */
315 int lofs, lpos = b[pos];
316 if (lpos < 0) goto noshrink; /* Ext global? */
317 lofs = *DASM_POS2PTR(D, lpos);
318 if (lpos > pos) { /* Fwd label: add cumulative section offsets. */
319 int i;
320 for (i = secnum; i < DASM_POS2SEC(lpos); i++)
321 lofs += D->sections[i].ofs;
322 } else {
323 lofs -= ofs; /* Bkwd label: unfix offset. */
324 }
325 lofs -= b[pos+1]; /* Short branch ok? */
326 if (lofs >= -128-shrink && lofs <= 127) ofs -= shrink; /* Yes. */
327 else { noshrink: shrink = 0; } /* No, cannot shrink op. */
328 }
329 b[pos+1] = shrink;
330 pos += 2;
331 break;
332 }
333 /* fallthrough */
334 case DASM_SPACE: case DASM_IMM_LG: case DASM_VREG: p++;
335 /* fallthrough */
336 case DASM_DISP: case DASM_IMM_S: case DASM_IMM_B: case DASM_IMM_W:
337 case DASM_IMM_D: case DASM_IMM_WB: case DASM_IMM_DB:
338 case DASM_SETLABEL: case DASM_REL_A: case DASM_IMM_PC: pos++; break;
339 case DASM_LABEL_LG: p++;
340 /* fallthrough */
341 case DASM_LABEL_PC: b[pos++] += ofs; break; /* Fix label offset. */
342 case DASM_ALIGN: ofs -= (b[pos++]+ofs)&*p++; break; /* Adjust ofs. */
343 case DASM_EXTERN: p += 2; break;
344 case DASM_ESC: op = *p++; break;
345 case DASM_MARK: break;
346 case DASM_SECTION: case DASM_STOP: goto stop;
347 default: op = action; break;
348 }
349 }
350 stop: (void)0;
351 }
352 ofs += sec->ofs; /* Next section starts right after current section. */
353 }
354
355 D->codesize = ofs; /* Total size of all code sections */
356 *szp = ofs;
357 return DASM_S_OK;
358 }
359
360 #define dasmb(x) *cp++ = (unsigned char)(x)
361 #ifndef DASM_ALIGNED_WRITES
362 #define dasmw(x) \
363 do { *((unsigned short *)cp) = (unsigned short)(x); cp+=2; } while (0)
364 #define dasmd(x) \
365 do { *((unsigned int *)cp) = (unsigned int)(x); cp+=4; } while (0)
366 #define dasmq(x) \
367 do { *((unsigned long long *)cp) = (unsigned long long)(x); cp+=8; } while (0)
368 #else
369 #define dasmw(x) do { dasmb(x); dasmb((x)>>8); } while (0)
370 #define dasmd(x) do { dasmw(x); dasmw((x)>>16); } while (0)
371 #define dasmq(x) do { dasmd(x); dasmd((x)>>32); } while (0)
372 #endif
373 static unsigned char *dasma_(unsigned char *cp, ptrdiff_t x)
374 {
375 if (sizeof(ptrdiff_t) == 8)
376 dasmq((unsigned long long)x);
377 else
378 dasmd((unsigned int)x);
379 return cp;
380 }
381 #define dasma(x) (cp = dasma_(cp, (x)))
382
383 /* Pass 3: Encode sections. */
384 int dasm_encode(Dst_DECL, void *buffer)
385 {
386 dasm_State *D = Dst_REF;
387 unsigned char *base = (unsigned char *)buffer;
388 unsigned char *cp = base;
389 int secnum;
390
391 /* Encode all code sections. No support for data sections (yet). */
392 for (secnum = 0; secnum < D->maxsection; secnum++) {
393 dasm_Section *sec = D->sections + secnum;
394 int *b = sec->buf;
395 int *endb = sec->rbuf + sec->pos;
396
397 while (b != endb) {
398 dasm_ActList p = D->actionlist + *b++;
399 unsigned char *mark = NULL;
400 while (1) {
401 int action = *p++;
402 int n = (action >= DASM_DISP && action <= DASM_ALIGN) ? *b++ : 0;
403 switch (action) {
404 case DASM_DISP: if (!mark) mark = cp; {
405 unsigned char *mm = mark;
406 if (*p != DASM_IMM_DB && *p != DASM_IMM_WB) mark = NULL;
407 if (n == 0) { int mrm = mm[-1]&7; if (mrm == 4) mrm = mm[0]&7;
408 if (mrm != 5) { mm[-1] -= 0x80; break; } }
409 if (((n+128) & -256) != 0) goto wd; else mm[-1] -= 0x40;
410 }
411 /* fallthrough */
412 case DASM_IMM_S: case DASM_IMM_B: wb: dasmb(n); break;
413 case DASM_IMM_DB: if (((n+128)&-256) == 0) {
414 db: if (!mark) mark = cp; mark[-2] += 2; mark = NULL; goto wb;
415 } else mark = NULL;
416 /* fallthrough */
417 case DASM_IMM_D: wd: dasmd(n); break;
418 case DASM_IMM_WB: if (((n+128)&-256) == 0) goto db; else mark = NULL;
419 /* fallthrough */
420 case DASM_IMM_W: dasmw(n); break;
421 case DASM_VREG: {
422 int t = *p++;
423 unsigned char *ex = cp - (t&7);
424 if ((n & 8) && t < 0xa0) {
425 if (*ex & 0x80) ex[1] ^= 0x20 << (t>>6); else *ex ^= 1 << (t>>6);
426 n &= 7;
427 } else if (n & 0x10) {
428 if (*ex & 0x80) {
429 *ex = 0xc5; ex[1] = (ex[1] & 0x80) | ex[2]; ex += 2;
430 }
431 while (++ex < cp) ex[-1] = *ex;
432 if (mark) mark--;
433 cp--;
434 n &= 7;
435 }
436 if (t >= 0xc0) n <<= 4;
437 else if (t >= 0x40) n <<= 3;
438 else if (n == 4 && t < 0x20) { cp[-1] ^= n; *cp++ = 0x20; }
439 cp[-1] ^= n;
440 break;
441 }
442 case DASM_REL_LG: p++; if (n >= 0) goto rel_pc;
443 b++; n = (int)(ptrdiff_t)D->globals[-n-10];
444 /* fallthrough */
445 case DASM_REL_A: rel_a:
446 n -= (unsigned int)(ptrdiff_t)(cp+4); goto wd; /* !x64 */
447 case DASM_REL_PC: rel_pc: {
448 int shrink = *b++;
449 int *pb = DASM_POS2PTR(D, n); if (*pb < 0) { n = pb[1]; goto rel_a; }
450 n = *pb - ((int)(cp-base) + 4-shrink);
451 if (shrink == 0) goto wd;
452 if (shrink == 4) { cp--; cp[-1] = *cp-0x10; } else cp[-1] = 0xeb;
453 goto wb;
454 }
455 case DASM_IMM_LG:
456 p++;
457 if (n < 0) { dasma((ptrdiff_t)D->globals[-n-10]); break; }
458 /* fallthrough */
459 case DASM_IMM_PC: {
460 int *pb = DASM_POS2PTR(D, n);
461 dasma(*pb < 0 ? (ptrdiff_t)pb[1] : (*pb + (ptrdiff_t)base));
462 break;
463 }
464 case DASM_LABEL_LG: {
465 int idx = *p++;
466 if (idx >= 10)
467 D->globals[idx-10] = (void *)(base + (*p == DASM_SETLABEL ? *b : n));
468 break;
469 }
470 case DASM_LABEL_PC: case DASM_SETLABEL: break;
471 case DASM_SPACE: { int fill = *p++; while (n--) *cp++ = fill; break; }
472 case DASM_ALIGN:
473 n = *p++;
474 while (((cp-base) & n)) *cp++ = 0x90; /* nop */
475 break;
476 case DASM_EXTERN: n = DASM_EXTERN(Dst, cp, p[1], *p); p += 2; goto wd;
477 case DASM_MARK: mark = cp; break;
478 case DASM_ESC: action = *p++;
479 /* fallthrough */
480 default: *cp++ = action; break;
481 case DASM_SECTION: case DASM_STOP: goto stop;
482 }
483 }
484 stop: (void)0;
485 }
486 }
487
488 if (base + D->codesize != cp) /* Check for phase errors. */
489 return DASM_S_PHASE;
490 return DASM_S_OK;
491 }
492
493 /* Get PC label offset. */
494 int dasm_getpclabel(Dst_DECL, unsigned int pc)
495 {
496 dasm_State *D = Dst_REF;
497 if (pc*sizeof(int) < D->pcsize) {
498 int pos = D->pclabels[pc];
499 if (pos < 0) return *DASM_POS2PTR(D, -pos);
500 if (pos > 0) return -1; /* Undefined. */
501 }
502 return -2; /* Unused or out of range. */
503 }
504
505 #ifdef DASM_CHECKS
506 /* Optional sanity checker to call between isolated encoding steps. */
507 int dasm_checkstep(Dst_DECL, int secmatch)
508 {
509 dasm_State *D = Dst_REF;
510 if (D->status == DASM_S_OK) {
511 int i;
512 for (i = 1; i <= 9; i++) {
513 if (D->lglabels[i] > 0) { D->status = DASM_S_UNDEF_L|i; break; }
514 D->lglabels[i] = 0;
515 }
516 }
517 if (D->status == DASM_S_OK && secmatch >= 0 &&
518 D->section != &D->sections[secmatch])
519 D->status = DASM_S_MATCH_SEC|(int)(D->section-D->sections);
520 return D->status;
521 }
522 #endif
523