xref: /illumos-kvm-cmd/target-i386/ops_sse.h (revision 68396ea9)
1 /*
2  *  MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4/PNI support
3  *
4  *  Copyright (c) 2005 Fabrice Bellard
5  *  Copyright (c) 2008 Intel Corporation  <andrew.zaborowski@intel.com>
6  *
7  * This library is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2 of the License, or (at your option) any later version.
11  *
12  * This library is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19  */
20 #if SHIFT == 0
21 #define Reg MMXReg
22 #define XMM_ONLY(...)
23 #define B(n) MMX_B(n)
24 #define W(n) MMX_W(n)
25 #define L(n) MMX_L(n)
26 #define Q(n) q
27 #define SUFFIX _mmx
28 #else
29 #define Reg XMMReg
30 #define XMM_ONLY(...) __VA_ARGS__
31 #define B(n) XMM_B(n)
32 #define W(n) XMM_W(n)
33 #define L(n) XMM_L(n)
34 #define Q(n) XMM_Q(n)
35 #define SUFFIX _xmm
36 #endif
37 
glue(helper_psrlw,SUFFIX)38 void glue(helper_psrlw, SUFFIX)(Reg *d, Reg *s)
39 {
40     int shift;
41 
42     if (s->Q(0) > 15) {
43         d->Q(0) = 0;
44 #if SHIFT == 1
45         d->Q(1) = 0;
46 #endif
47     } else {
48         shift = s->B(0);
49         d->W(0) >>= shift;
50         d->W(1) >>= shift;
51         d->W(2) >>= shift;
52         d->W(3) >>= shift;
53 #if SHIFT == 1
54         d->W(4) >>= shift;
55         d->W(5) >>= shift;
56         d->W(6) >>= shift;
57         d->W(7) >>= shift;
58 #endif
59     }
60 }
61 
glue(helper_psraw,SUFFIX)62 void glue(helper_psraw, SUFFIX)(Reg *d, Reg *s)
63 {
64     int shift;
65 
66     if (s->Q(0) > 15) {
67         shift = 15;
68     } else {
69         shift = s->B(0);
70     }
71     d->W(0) = (int16_t)d->W(0) >> shift;
72     d->W(1) = (int16_t)d->W(1) >> shift;
73     d->W(2) = (int16_t)d->W(2) >> shift;
74     d->W(3) = (int16_t)d->W(3) >> shift;
75 #if SHIFT == 1
76     d->W(4) = (int16_t)d->W(4) >> shift;
77     d->W(5) = (int16_t)d->W(5) >> shift;
78     d->W(6) = (int16_t)d->W(6) >> shift;
79     d->W(7) = (int16_t)d->W(7) >> shift;
80 #endif
81 }
82 
glue(helper_psllw,SUFFIX)83 void glue(helper_psllw, SUFFIX)(Reg *d, Reg *s)
84 {
85     int shift;
86 
87     if (s->Q(0) > 15) {
88         d->Q(0) = 0;
89 #if SHIFT == 1
90         d->Q(1) = 0;
91 #endif
92     } else {
93         shift = s->B(0);
94         d->W(0) <<= shift;
95         d->W(1) <<= shift;
96         d->W(2) <<= shift;
97         d->W(3) <<= shift;
98 #if SHIFT == 1
99         d->W(4) <<= shift;
100         d->W(5) <<= shift;
101         d->W(6) <<= shift;
102         d->W(7) <<= shift;
103 #endif
104     }
105 }
106 
glue(helper_psrld,SUFFIX)107 void glue(helper_psrld, SUFFIX)(Reg *d, Reg *s)
108 {
109     int shift;
110 
111     if (s->Q(0) > 31) {
112         d->Q(0) = 0;
113 #if SHIFT == 1
114         d->Q(1) = 0;
115 #endif
116     } else {
117         shift = s->B(0);
118         d->L(0) >>= shift;
119         d->L(1) >>= shift;
120 #if SHIFT == 1
121         d->L(2) >>= shift;
122         d->L(3) >>= shift;
123 #endif
124     }
125 }
126 
glue(helper_psrad,SUFFIX)127 void glue(helper_psrad, SUFFIX)(Reg *d, Reg *s)
128 {
129     int shift;
130 
131     if (s->Q(0) > 31) {
132         shift = 31;
133     } else {
134         shift = s->B(0);
135     }
136     d->L(0) = (int32_t)d->L(0) >> shift;
137     d->L(1) = (int32_t)d->L(1) >> shift;
138 #if SHIFT == 1
139     d->L(2) = (int32_t)d->L(2) >> shift;
140     d->L(3) = (int32_t)d->L(3) >> shift;
141 #endif
142 }
143 
glue(helper_pslld,SUFFIX)144 void glue(helper_pslld, SUFFIX)(Reg *d, Reg *s)
145 {
146     int shift;
147 
148     if (s->Q(0) > 31) {
149         d->Q(0) = 0;
150 #if SHIFT == 1
151         d->Q(1) = 0;
152 #endif
153     } else {
154         shift = s->B(0);
155         d->L(0) <<= shift;
156         d->L(1) <<= shift;
157 #if SHIFT == 1
158         d->L(2) <<= shift;
159         d->L(3) <<= shift;
160 #endif
161     }
162 }
163 
glue(helper_psrlq,SUFFIX)164 void glue(helper_psrlq, SUFFIX)(Reg *d, Reg *s)
165 {
166     int shift;
167 
168     if (s->Q(0) > 63) {
169         d->Q(0) = 0;
170 #if SHIFT == 1
171         d->Q(1) = 0;
172 #endif
173     } else {
174         shift = s->B(0);
175         d->Q(0) >>= shift;
176 #if SHIFT == 1
177         d->Q(1) >>= shift;
178 #endif
179     }
180 }
181 
glue(helper_psllq,SUFFIX)182 void glue(helper_psllq, SUFFIX)(Reg *d, Reg *s)
183 {
184     int shift;
185 
186     if (s->Q(0) > 63) {
187         d->Q(0) = 0;
188 #if SHIFT == 1
189         d->Q(1) = 0;
190 #endif
191     } else {
192         shift = s->B(0);
193         d->Q(0) <<= shift;
194 #if SHIFT == 1
195         d->Q(1) <<= shift;
196 #endif
197     }
198 }
199 
200 #if SHIFT == 1
glue(helper_psrldq,SUFFIX)201 void glue(helper_psrldq, SUFFIX)(Reg *d, Reg *s)
202 {
203     int shift, i;
204 
205     shift = s->L(0);
206     if (shift > 16)
207         shift = 16;
208     for(i = 0; i < 16 - shift; i++)
209         d->B(i) = d->B(i + shift);
210     for(i = 16 - shift; i < 16; i++)
211         d->B(i) = 0;
212 }
213 
glue(helper_pslldq,SUFFIX)214 void glue(helper_pslldq, SUFFIX)(Reg *d, Reg *s)
215 {
216     int shift, i;
217 
218     shift = s->L(0);
219     if (shift > 16)
220         shift = 16;
221     for(i = 15; i >= shift; i--)
222         d->B(i) = d->B(i - shift);
223     for(i = 0; i < shift; i++)
224         d->B(i) = 0;
225 }
226 #endif
227 
228 #define SSE_HELPER_B(name, F)\
229 void glue(name, SUFFIX) (Reg *d, Reg *s)\
230 {\
231     d->B(0) = F(d->B(0), s->B(0));\
232     d->B(1) = F(d->B(1), s->B(1));\
233     d->B(2) = F(d->B(2), s->B(2));\
234     d->B(3) = F(d->B(3), s->B(3));\
235     d->B(4) = F(d->B(4), s->B(4));\
236     d->B(5) = F(d->B(5), s->B(5));\
237     d->B(6) = F(d->B(6), s->B(6));\
238     d->B(7) = F(d->B(7), s->B(7));\
239     XMM_ONLY(\
240     d->B(8) = F(d->B(8), s->B(8));\
241     d->B(9) = F(d->B(9), s->B(9));\
242     d->B(10) = F(d->B(10), s->B(10));\
243     d->B(11) = F(d->B(11), s->B(11));\
244     d->B(12) = F(d->B(12), s->B(12));\
245     d->B(13) = F(d->B(13), s->B(13));\
246     d->B(14) = F(d->B(14), s->B(14));\
247     d->B(15) = F(d->B(15), s->B(15));\
248     )\
249 }
250 
251 #define SSE_HELPER_W(name, F)\
252 void glue(name, SUFFIX) (Reg *d, Reg *s)\
253 {\
254     d->W(0) = F(d->W(0), s->W(0));\
255     d->W(1) = F(d->W(1), s->W(1));\
256     d->W(2) = F(d->W(2), s->W(2));\
257     d->W(3) = F(d->W(3), s->W(3));\
258     XMM_ONLY(\
259     d->W(4) = F(d->W(4), s->W(4));\
260     d->W(5) = F(d->W(5), s->W(5));\
261     d->W(6) = F(d->W(6), s->W(6));\
262     d->W(7) = F(d->W(7), s->W(7));\
263     )\
264 }
265 
266 #define SSE_HELPER_L(name, F)\
267 void glue(name, SUFFIX) (Reg *d, Reg *s)\
268 {\
269     d->L(0) = F(d->L(0), s->L(0));\
270     d->L(1) = F(d->L(1), s->L(1));\
271     XMM_ONLY(\
272     d->L(2) = F(d->L(2), s->L(2));\
273     d->L(3) = F(d->L(3), s->L(3));\
274     )\
275 }
276 
277 #define SSE_HELPER_Q(name, F)\
278 void glue(name, SUFFIX) (Reg *d, Reg *s)\
279 {\
280     d->Q(0) = F(d->Q(0), s->Q(0));\
281     XMM_ONLY(\
282     d->Q(1) = F(d->Q(1), s->Q(1));\
283     )\
284 }
285 
286 #if SHIFT == 0
satub(int x)287 static inline int satub(int x)
288 {
289     if (x < 0)
290         return 0;
291     else if (x > 255)
292         return 255;
293     else
294         return x;
295 }
296 
satuw(int x)297 static inline int satuw(int x)
298 {
299     if (x < 0)
300         return 0;
301     else if (x > 65535)
302         return 65535;
303     else
304         return x;
305 }
306 
satsb(int x)307 static inline int satsb(int x)
308 {
309     if (x < -128)
310         return -128;
311     else if (x > 127)
312         return 127;
313     else
314         return x;
315 }
316 
satsw(int x)317 static inline int satsw(int x)
318 {
319     if (x < -32768)
320         return -32768;
321     else if (x > 32767)
322         return 32767;
323     else
324         return x;
325 }
326 
327 #define FADD(a, b) ((a) + (b))
328 #define FADDUB(a, b) satub((a) + (b))
329 #define FADDUW(a, b) satuw((a) + (b))
330 #define FADDSB(a, b) satsb((int8_t)(a) + (int8_t)(b))
331 #define FADDSW(a, b) satsw((int16_t)(a) + (int16_t)(b))
332 
333 #define FSUB(a, b) ((a) - (b))
334 #define FSUBUB(a, b) satub((a) - (b))
335 #define FSUBUW(a, b) satuw((a) - (b))
336 #define FSUBSB(a, b) satsb((int8_t)(a) - (int8_t)(b))
337 #define FSUBSW(a, b) satsw((int16_t)(a) - (int16_t)(b))
338 #define FMINUB(a, b) ((a) < (b)) ? (a) : (b)
339 #define FMINSW(a, b) ((int16_t)(a) < (int16_t)(b)) ? (a) : (b)
340 #define FMAXUB(a, b) ((a) > (b)) ? (a) : (b)
341 #define FMAXSW(a, b) ((int16_t)(a) > (int16_t)(b)) ? (a) : (b)
342 
343 #define FAND(a, b) (a) & (b)
344 #define FANDN(a, b) ((~(a)) & (b))
345 #define FOR(a, b) (a) | (b)
346 #define FXOR(a, b) (a) ^ (b)
347 
348 #define FCMPGTB(a, b) (int8_t)(a) > (int8_t)(b) ? -1 : 0
349 #define FCMPGTW(a, b) (int16_t)(a) > (int16_t)(b) ? -1 : 0
350 #define FCMPGTL(a, b) (int32_t)(a) > (int32_t)(b) ? -1 : 0
351 #define FCMPEQ(a, b) (a) == (b) ? -1 : 0
352 
353 #define FMULLW(a, b) (a) * (b)
354 #define FMULHRW(a, b) ((int16_t)(a) * (int16_t)(b) + 0x8000) >> 16
355 #define FMULHUW(a, b) (a) * (b) >> 16
356 #define FMULHW(a, b) (int16_t)(a) * (int16_t)(b) >> 16
357 
358 #define FAVG(a, b) ((a) + (b) + 1) >> 1
359 #endif
360 
SSE_HELPER_B(helper_paddb,FADD)361 SSE_HELPER_B(helper_paddb, FADD)
362 SSE_HELPER_W(helper_paddw, FADD)
363 SSE_HELPER_L(helper_paddl, FADD)
364 SSE_HELPER_Q(helper_paddq, FADD)
365 
366 SSE_HELPER_B(helper_psubb, FSUB)
367 SSE_HELPER_W(helper_psubw, FSUB)
368 SSE_HELPER_L(helper_psubl, FSUB)
369 SSE_HELPER_Q(helper_psubq, FSUB)
370 
371 SSE_HELPER_B(helper_paddusb, FADDUB)
372 SSE_HELPER_B(helper_paddsb, FADDSB)
373 SSE_HELPER_B(helper_psubusb, FSUBUB)
374 SSE_HELPER_B(helper_psubsb, FSUBSB)
375 
376 SSE_HELPER_W(helper_paddusw, FADDUW)
377 SSE_HELPER_W(helper_paddsw, FADDSW)
378 SSE_HELPER_W(helper_psubusw, FSUBUW)
379 SSE_HELPER_W(helper_psubsw, FSUBSW)
380 
381 SSE_HELPER_B(helper_pminub, FMINUB)
382 SSE_HELPER_B(helper_pmaxub, FMAXUB)
383 
384 SSE_HELPER_W(helper_pminsw, FMINSW)
385 SSE_HELPER_W(helper_pmaxsw, FMAXSW)
386 
387 SSE_HELPER_Q(helper_pand, FAND)
388 SSE_HELPER_Q(helper_pandn, FANDN)
389 SSE_HELPER_Q(helper_por, FOR)
390 SSE_HELPER_Q(helper_pxor, FXOR)
391 
392 SSE_HELPER_B(helper_pcmpgtb, FCMPGTB)
393 SSE_HELPER_W(helper_pcmpgtw, FCMPGTW)
394 SSE_HELPER_L(helper_pcmpgtl, FCMPGTL)
395 
396 SSE_HELPER_B(helper_pcmpeqb, FCMPEQ)
397 SSE_HELPER_W(helper_pcmpeqw, FCMPEQ)
398 SSE_HELPER_L(helper_pcmpeql, FCMPEQ)
399 
400 SSE_HELPER_W(helper_pmullw, FMULLW)
401 #if SHIFT == 0
402 SSE_HELPER_W(helper_pmulhrw, FMULHRW)
403 #endif
404 SSE_HELPER_W(helper_pmulhuw, FMULHUW)
405 SSE_HELPER_W(helper_pmulhw, FMULHW)
406 
407 SSE_HELPER_B(helper_pavgb, FAVG)
408 SSE_HELPER_W(helper_pavgw, FAVG)
409 
410 void glue(helper_pmuludq, SUFFIX) (Reg *d, Reg *s)
411 {
412     d->Q(0) = (uint64_t)s->L(0) * (uint64_t)d->L(0);
413 #if SHIFT == 1
414     d->Q(1) = (uint64_t)s->L(2) * (uint64_t)d->L(2);
415 #endif
416 }
417 
glue(helper_pmaddwd,SUFFIX)418 void glue(helper_pmaddwd, SUFFIX) (Reg *d, Reg *s)
419 {
420     int i;
421 
422     for(i = 0; i < (2 << SHIFT); i++) {
423         d->L(i) = (int16_t)s->W(2*i) * (int16_t)d->W(2*i) +
424             (int16_t)s->W(2*i+1) * (int16_t)d->W(2*i+1);
425     }
426 }
427 
428 #if SHIFT == 0
abs1(int a)429 static inline int abs1(int a)
430 {
431     if (a < 0)
432         return -a;
433     else
434         return a;
435 }
436 #endif
glue(helper_psadbw,SUFFIX)437 void glue(helper_psadbw, SUFFIX) (Reg *d, Reg *s)
438 {
439     unsigned int val;
440 
441     val = 0;
442     val += abs1(d->B(0) - s->B(0));
443     val += abs1(d->B(1) - s->B(1));
444     val += abs1(d->B(2) - s->B(2));
445     val += abs1(d->B(3) - s->B(3));
446     val += abs1(d->B(4) - s->B(4));
447     val += abs1(d->B(5) - s->B(5));
448     val += abs1(d->B(6) - s->B(6));
449     val += abs1(d->B(7) - s->B(7));
450     d->Q(0) = val;
451 #if SHIFT == 1
452     val = 0;
453     val += abs1(d->B(8) - s->B(8));
454     val += abs1(d->B(9) - s->B(9));
455     val += abs1(d->B(10) - s->B(10));
456     val += abs1(d->B(11) - s->B(11));
457     val += abs1(d->B(12) - s->B(12));
458     val += abs1(d->B(13) - s->B(13));
459     val += abs1(d->B(14) - s->B(14));
460     val += abs1(d->B(15) - s->B(15));
461     d->Q(1) = val;
462 #endif
463 }
464 
glue(helper_maskmov,SUFFIX)465 void glue(helper_maskmov, SUFFIX) (Reg *d, Reg *s, target_ulong a0)
466 {
467     int i;
468     for(i = 0; i < (8 << SHIFT); i++) {
469         if (s->B(i) & 0x80)
470             stb(a0 + i, d->B(i));
471     }
472 }
473 
glue(helper_movl_mm_T0,SUFFIX)474 void glue(helper_movl_mm_T0, SUFFIX) (Reg *d, uint32_t val)
475 {
476     d->L(0) = val;
477     d->L(1) = 0;
478 #if SHIFT == 1
479     d->Q(1) = 0;
480 #endif
481 }
482 
483 #ifdef TARGET_X86_64
glue(helper_movq_mm_T0,SUFFIX)484 void glue(helper_movq_mm_T0, SUFFIX) (Reg *d, uint64_t val)
485 {
486     d->Q(0) = val;
487 #if SHIFT == 1
488     d->Q(1) = 0;
489 #endif
490 }
491 #endif
492 
493 #if SHIFT == 0
glue(helper_pshufw,SUFFIX)494 void glue(helper_pshufw, SUFFIX) (Reg *d, Reg *s, int order)
495 {
496     Reg r;
497     r.W(0) = s->W(order & 3);
498     r.W(1) = s->W((order >> 2) & 3);
499     r.W(2) = s->W((order >> 4) & 3);
500     r.W(3) = s->W((order >> 6) & 3);
501     *d = r;
502 }
503 #else
helper_shufps(Reg * d,Reg * s,int order)504 void helper_shufps(Reg *d, Reg *s, int order)
505 {
506     Reg r;
507     r.L(0) = d->L(order & 3);
508     r.L(1) = d->L((order >> 2) & 3);
509     r.L(2) = s->L((order >> 4) & 3);
510     r.L(3) = s->L((order >> 6) & 3);
511     *d = r;
512 }
513 
helper_shufpd(Reg * d,Reg * s,int order)514 void helper_shufpd(Reg *d, Reg *s, int order)
515 {
516     Reg r;
517     r.Q(0) = d->Q(order & 1);
518     r.Q(1) = s->Q((order >> 1) & 1);
519     *d = r;
520 }
521 
glue(helper_pshufd,SUFFIX)522 void glue(helper_pshufd, SUFFIX) (Reg *d, Reg *s, int order)
523 {
524     Reg r;
525     r.L(0) = s->L(order & 3);
526     r.L(1) = s->L((order >> 2) & 3);
527     r.L(2) = s->L((order >> 4) & 3);
528     r.L(3) = s->L((order >> 6) & 3);
529     *d = r;
530 }
531 
glue(helper_pshuflw,SUFFIX)532 void glue(helper_pshuflw, SUFFIX) (Reg *d, Reg *s, int order)
533 {
534     Reg r;
535     r.W(0) = s->W(order & 3);
536     r.W(1) = s->W((order >> 2) & 3);
537     r.W(2) = s->W((order >> 4) & 3);
538     r.W(3) = s->W((order >> 6) & 3);
539     r.Q(1) = s->Q(1);
540     *d = r;
541 }
542 
glue(helper_pshufhw,SUFFIX)543 void glue(helper_pshufhw, SUFFIX) (Reg *d, Reg *s, int order)
544 {
545     Reg r;
546     r.Q(0) = s->Q(0);
547     r.W(4) = s->W(4 + (order & 3));
548     r.W(5) = s->W(4 + ((order >> 2) & 3));
549     r.W(6) = s->W(4 + ((order >> 4) & 3));
550     r.W(7) = s->W(4 + ((order >> 6) & 3));
551     *d = r;
552 }
553 #endif
554 
555 #if SHIFT == 1
556 /* FPU ops */
557 /* XXX: not accurate */
558 
559 #define SSE_HELPER_S(name, F)\
560 void helper_ ## name ## ps (Reg *d, Reg *s)\
561 {\
562     d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0));\
563     d->XMM_S(1) = F(32, d->XMM_S(1), s->XMM_S(1));\
564     d->XMM_S(2) = F(32, d->XMM_S(2), s->XMM_S(2));\
565     d->XMM_S(3) = F(32, d->XMM_S(3), s->XMM_S(3));\
566 }\
567 \
568 void helper_ ## name ## ss (Reg *d, Reg *s)\
569 {\
570     d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0));\
571 }\
572 void helper_ ## name ## pd (Reg *d, Reg *s)\
573 {\
574     d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0));\
575     d->XMM_D(1) = F(64, d->XMM_D(1), s->XMM_D(1));\
576 }\
577 \
578 void helper_ ## name ## sd (Reg *d, Reg *s)\
579 {\
580     d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0));\
581 }
582 
583 #define FPU_ADD(size, a, b) float ## size ## _add(a, b, &env->sse_status)
584 #define FPU_SUB(size, a, b) float ## size ## _sub(a, b, &env->sse_status)
585 #define FPU_MUL(size, a, b) float ## size ## _mul(a, b, &env->sse_status)
586 #define FPU_DIV(size, a, b) float ## size ## _div(a, b, &env->sse_status)
587 #define FPU_MIN(size, a, b) (a) < (b) ? (a) : (b)
588 #define FPU_MAX(size, a, b) (a) > (b) ? (a) : (b)
589 #define FPU_SQRT(size, a, b) float ## size ## _sqrt(b, &env->sse_status)
590 
SSE_HELPER_S(add,FPU_ADD)591 SSE_HELPER_S(add, FPU_ADD)
592 SSE_HELPER_S(sub, FPU_SUB)
593 SSE_HELPER_S(mul, FPU_MUL)
594 SSE_HELPER_S(div, FPU_DIV)
595 SSE_HELPER_S(min, FPU_MIN)
596 SSE_HELPER_S(max, FPU_MAX)
597 SSE_HELPER_S(sqrt, FPU_SQRT)
598 
599 
600 /* float to float conversions */
601 void helper_cvtps2pd(Reg *d, Reg *s)
602 {
603     float32 s0, s1;
604     s0 = s->XMM_S(0);
605     s1 = s->XMM_S(1);
606     d->XMM_D(0) = float32_to_float64(s0, &env->sse_status);
607     d->XMM_D(1) = float32_to_float64(s1, &env->sse_status);
608 }
609 
helper_cvtpd2ps(Reg * d,Reg * s)610 void helper_cvtpd2ps(Reg *d, Reg *s)
611 {
612     d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
613     d->XMM_S(1) = float64_to_float32(s->XMM_D(1), &env->sse_status);
614     d->Q(1) = 0;
615 }
616 
helper_cvtss2sd(Reg * d,Reg * s)617 void helper_cvtss2sd(Reg *d, Reg *s)
618 {
619     d->XMM_D(0) = float32_to_float64(s->XMM_S(0), &env->sse_status);
620 }
621 
helper_cvtsd2ss(Reg * d,Reg * s)622 void helper_cvtsd2ss(Reg *d, Reg *s)
623 {
624     d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
625 }
626 
627 /* integer to float */
helper_cvtdq2ps(Reg * d,Reg * s)628 void helper_cvtdq2ps(Reg *d, Reg *s)
629 {
630     d->XMM_S(0) = int32_to_float32(s->XMM_L(0), &env->sse_status);
631     d->XMM_S(1) = int32_to_float32(s->XMM_L(1), &env->sse_status);
632     d->XMM_S(2) = int32_to_float32(s->XMM_L(2), &env->sse_status);
633     d->XMM_S(3) = int32_to_float32(s->XMM_L(3), &env->sse_status);
634 }
635 
helper_cvtdq2pd(Reg * d,Reg * s)636 void helper_cvtdq2pd(Reg *d, Reg *s)
637 {
638     int32_t l0, l1;
639     l0 = (int32_t)s->XMM_L(0);
640     l1 = (int32_t)s->XMM_L(1);
641     d->XMM_D(0) = int32_to_float64(l0, &env->sse_status);
642     d->XMM_D(1) = int32_to_float64(l1, &env->sse_status);
643 }
644 
helper_cvtpi2ps(XMMReg * d,MMXReg * s)645 void helper_cvtpi2ps(XMMReg *d, MMXReg *s)
646 {
647     d->XMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status);
648     d->XMM_S(1) = int32_to_float32(s->MMX_L(1), &env->sse_status);
649 }
650 
helper_cvtpi2pd(XMMReg * d,MMXReg * s)651 void helper_cvtpi2pd(XMMReg *d, MMXReg *s)
652 {
653     d->XMM_D(0) = int32_to_float64(s->MMX_L(0), &env->sse_status);
654     d->XMM_D(1) = int32_to_float64(s->MMX_L(1), &env->sse_status);
655 }
656 
helper_cvtsi2ss(XMMReg * d,uint32_t val)657 void helper_cvtsi2ss(XMMReg *d, uint32_t val)
658 {
659     d->XMM_S(0) = int32_to_float32(val, &env->sse_status);
660 }
661 
helper_cvtsi2sd(XMMReg * d,uint32_t val)662 void helper_cvtsi2sd(XMMReg *d, uint32_t val)
663 {
664     d->XMM_D(0) = int32_to_float64(val, &env->sse_status);
665 }
666 
667 #ifdef TARGET_X86_64
helper_cvtsq2ss(XMMReg * d,uint64_t val)668 void helper_cvtsq2ss(XMMReg *d, uint64_t val)
669 {
670     d->XMM_S(0) = int64_to_float32(val, &env->sse_status);
671 }
672 
helper_cvtsq2sd(XMMReg * d,uint64_t val)673 void helper_cvtsq2sd(XMMReg *d, uint64_t val)
674 {
675     d->XMM_D(0) = int64_to_float64(val, &env->sse_status);
676 }
677 #endif
678 
679 /* float to integer */
helper_cvtps2dq(XMMReg * d,XMMReg * s)680 void helper_cvtps2dq(XMMReg *d, XMMReg *s)
681 {
682     d->XMM_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
683     d->XMM_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
684     d->XMM_L(2) = float32_to_int32(s->XMM_S(2), &env->sse_status);
685     d->XMM_L(3) = float32_to_int32(s->XMM_S(3), &env->sse_status);
686 }
687 
helper_cvtpd2dq(XMMReg * d,XMMReg * s)688 void helper_cvtpd2dq(XMMReg *d, XMMReg *s)
689 {
690     d->XMM_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
691     d->XMM_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
692     d->XMM_Q(1) = 0;
693 }
694 
helper_cvtps2pi(MMXReg * d,XMMReg * s)695 void helper_cvtps2pi(MMXReg *d, XMMReg *s)
696 {
697     d->MMX_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
698     d->MMX_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
699 }
700 
helper_cvtpd2pi(MMXReg * d,XMMReg * s)701 void helper_cvtpd2pi(MMXReg *d, XMMReg *s)
702 {
703     d->MMX_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
704     d->MMX_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
705 }
706 
helper_cvtss2si(XMMReg * s)707 int32_t helper_cvtss2si(XMMReg *s)
708 {
709     return float32_to_int32(s->XMM_S(0), &env->sse_status);
710 }
711 
helper_cvtsd2si(XMMReg * s)712 int32_t helper_cvtsd2si(XMMReg *s)
713 {
714     return float64_to_int32(s->XMM_D(0), &env->sse_status);
715 }
716 
717 #ifdef TARGET_X86_64
helper_cvtss2sq(XMMReg * s)718 int64_t helper_cvtss2sq(XMMReg *s)
719 {
720     return float32_to_int64(s->XMM_S(0), &env->sse_status);
721 }
722 
helper_cvtsd2sq(XMMReg * s)723 int64_t helper_cvtsd2sq(XMMReg *s)
724 {
725     return float64_to_int64(s->XMM_D(0), &env->sse_status);
726 }
727 #endif
728 
729 /* float to integer truncated */
helper_cvttps2dq(XMMReg * d,XMMReg * s)730 void helper_cvttps2dq(XMMReg *d, XMMReg *s)
731 {
732     d->XMM_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
733     d->XMM_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
734     d->XMM_L(2) = float32_to_int32_round_to_zero(s->XMM_S(2), &env->sse_status);
735     d->XMM_L(3) = float32_to_int32_round_to_zero(s->XMM_S(3), &env->sse_status);
736 }
737 
helper_cvttpd2dq(XMMReg * d,XMMReg * s)738 void helper_cvttpd2dq(XMMReg *d, XMMReg *s)
739 {
740     d->XMM_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
741     d->XMM_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
742     d->XMM_Q(1) = 0;
743 }
744 
helper_cvttps2pi(MMXReg * d,XMMReg * s)745 void helper_cvttps2pi(MMXReg *d, XMMReg *s)
746 {
747     d->MMX_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
748     d->MMX_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
749 }
750 
helper_cvttpd2pi(MMXReg * d,XMMReg * s)751 void helper_cvttpd2pi(MMXReg *d, XMMReg *s)
752 {
753     d->MMX_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
754     d->MMX_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
755 }
756 
helper_cvttss2si(XMMReg * s)757 int32_t helper_cvttss2si(XMMReg *s)
758 {
759     return float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
760 }
761 
helper_cvttsd2si(XMMReg * s)762 int32_t helper_cvttsd2si(XMMReg *s)
763 {
764     return float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
765 }
766 
767 #ifdef TARGET_X86_64
helper_cvttss2sq(XMMReg * s)768 int64_t helper_cvttss2sq(XMMReg *s)
769 {
770     return float32_to_int64_round_to_zero(s->XMM_S(0), &env->sse_status);
771 }
772 
helper_cvttsd2sq(XMMReg * s)773 int64_t helper_cvttsd2sq(XMMReg *s)
774 {
775     return float64_to_int64_round_to_zero(s->XMM_D(0), &env->sse_status);
776 }
777 #endif
778 
helper_rsqrtps(XMMReg * d,XMMReg * s)779 void helper_rsqrtps(XMMReg *d, XMMReg *s)
780 {
781     d->XMM_S(0) = approx_rsqrt(s->XMM_S(0));
782     d->XMM_S(1) = approx_rsqrt(s->XMM_S(1));
783     d->XMM_S(2) = approx_rsqrt(s->XMM_S(2));
784     d->XMM_S(3) = approx_rsqrt(s->XMM_S(3));
785 }
786 
helper_rsqrtss(XMMReg * d,XMMReg * s)787 void helper_rsqrtss(XMMReg *d, XMMReg *s)
788 {
789     d->XMM_S(0) = approx_rsqrt(s->XMM_S(0));
790 }
791 
helper_rcpps(XMMReg * d,XMMReg * s)792 void helper_rcpps(XMMReg *d, XMMReg *s)
793 {
794     d->XMM_S(0) = approx_rcp(s->XMM_S(0));
795     d->XMM_S(1) = approx_rcp(s->XMM_S(1));
796     d->XMM_S(2) = approx_rcp(s->XMM_S(2));
797     d->XMM_S(3) = approx_rcp(s->XMM_S(3));
798 }
799 
helper_rcpss(XMMReg * d,XMMReg * s)800 void helper_rcpss(XMMReg *d, XMMReg *s)
801 {
802     d->XMM_S(0) = approx_rcp(s->XMM_S(0));
803 }
804 
helper_extrq(uint64_t src,int shift,int len)805 static inline uint64_t helper_extrq(uint64_t src, int shift, int len)
806 {
807     uint64_t mask;
808 
809     if (len == 0) {
810         mask = ~0LL;
811     } else {
812         mask = (1ULL << len) - 1;
813     }
814     return (src >> shift) & mask;
815 }
816 
helper_extrq_r(XMMReg * d,XMMReg * s)817 void helper_extrq_r(XMMReg *d, XMMReg *s)
818 {
819     d->XMM_Q(0) = helper_extrq(d->XMM_Q(0), s->XMM_B(1), s->XMM_B(0));
820 }
821 
helper_extrq_i(XMMReg * d,int index,int length)822 void helper_extrq_i(XMMReg *d, int index, int length)
823 {
824     d->XMM_Q(0) = helper_extrq(d->XMM_Q(0), index, length);
825 }
826 
helper_insertq(uint64_t src,int shift,int len)827 static inline uint64_t helper_insertq(uint64_t src, int shift, int len)
828 {
829     uint64_t mask;
830 
831     if (len == 0) {
832         mask = ~0ULL;
833     } else {
834         mask = (1ULL << len) - 1;
835     }
836     return (src & ~(mask << shift)) | ((src & mask) << shift);
837 }
838 
helper_insertq_r(XMMReg * d,XMMReg * s)839 void helper_insertq_r(XMMReg *d, XMMReg *s)
840 {
841     d->XMM_Q(0) = helper_insertq(s->XMM_Q(0), s->XMM_B(9), s->XMM_B(8));
842 }
843 
helper_insertq_i(XMMReg * d,int index,int length)844 void helper_insertq_i(XMMReg *d, int index, int length)
845 {
846     d->XMM_Q(0) = helper_insertq(d->XMM_Q(0), index, length);
847 }
848 
helper_haddps(XMMReg * d,XMMReg * s)849 void helper_haddps(XMMReg *d, XMMReg *s)
850 {
851     XMMReg r;
852     r.XMM_S(0) = d->XMM_S(0) + d->XMM_S(1);
853     r.XMM_S(1) = d->XMM_S(2) + d->XMM_S(3);
854     r.XMM_S(2) = s->XMM_S(0) + s->XMM_S(1);
855     r.XMM_S(3) = s->XMM_S(2) + s->XMM_S(3);
856     *d = r;
857 }
858 
helper_haddpd(XMMReg * d,XMMReg * s)859 void helper_haddpd(XMMReg *d, XMMReg *s)
860 {
861     XMMReg r;
862     r.XMM_D(0) = d->XMM_D(0) + d->XMM_D(1);
863     r.XMM_D(1) = s->XMM_D(0) + s->XMM_D(1);
864     *d = r;
865 }
866 
helper_hsubps(XMMReg * d,XMMReg * s)867 void helper_hsubps(XMMReg *d, XMMReg *s)
868 {
869     XMMReg r;
870     r.XMM_S(0) = d->XMM_S(0) - d->XMM_S(1);
871     r.XMM_S(1) = d->XMM_S(2) - d->XMM_S(3);
872     r.XMM_S(2) = s->XMM_S(0) - s->XMM_S(1);
873     r.XMM_S(3) = s->XMM_S(2) - s->XMM_S(3);
874     *d = r;
875 }
876 
helper_hsubpd(XMMReg * d,XMMReg * s)877 void helper_hsubpd(XMMReg *d, XMMReg *s)
878 {
879     XMMReg r;
880     r.XMM_D(0) = d->XMM_D(0) - d->XMM_D(1);
881     r.XMM_D(1) = s->XMM_D(0) - s->XMM_D(1);
882     *d = r;
883 }
884 
helper_addsubps(XMMReg * d,XMMReg * s)885 void helper_addsubps(XMMReg *d, XMMReg *s)
886 {
887     d->XMM_S(0) = d->XMM_S(0) - s->XMM_S(0);
888     d->XMM_S(1) = d->XMM_S(1) + s->XMM_S(1);
889     d->XMM_S(2) = d->XMM_S(2) - s->XMM_S(2);
890     d->XMM_S(3) = d->XMM_S(3) + s->XMM_S(3);
891 }
892 
helper_addsubpd(XMMReg * d,XMMReg * s)893 void helper_addsubpd(XMMReg *d, XMMReg *s)
894 {
895     d->XMM_D(0) = d->XMM_D(0) - s->XMM_D(0);
896     d->XMM_D(1) = d->XMM_D(1) + s->XMM_D(1);
897 }
898 
899 /* XXX: unordered */
900 #define SSE_HELPER_CMP(name, F)\
901 void helper_ ## name ## ps (Reg *d, Reg *s)\
902 {\
903     d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0));\
904     d->XMM_L(1) = F(32, d->XMM_S(1), s->XMM_S(1));\
905     d->XMM_L(2) = F(32, d->XMM_S(2), s->XMM_S(2));\
906     d->XMM_L(3) = F(32, d->XMM_S(3), s->XMM_S(3));\
907 }\
908 \
909 void helper_ ## name ## ss (Reg *d, Reg *s)\
910 {\
911     d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0));\
912 }\
913 void helper_ ## name ## pd (Reg *d, Reg *s)\
914 {\
915     d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0));\
916     d->XMM_Q(1) = F(64, d->XMM_D(1), s->XMM_D(1));\
917 }\
918 \
919 void helper_ ## name ## sd (Reg *d, Reg *s)\
920 {\
921     d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0));\
922 }
923 
924 #define FPU_CMPEQ(size, a, b) float ## size ## _eq(a, b, &env->sse_status) ? -1 : 0
925 #define FPU_CMPLT(size, a, b) float ## size ## _lt(a, b, &env->sse_status) ? -1 : 0
926 #define FPU_CMPLE(size, a, b) float ## size ## _le(a, b, &env->sse_status) ? -1 : 0
927 #define FPU_CMPUNORD(size, a, b) float ## size ## _unordered(a, b, &env->sse_status) ? - 1 : 0
928 #define FPU_CMPNEQ(size, a, b) float ## size ## _eq(a, b, &env->sse_status) ? 0 : -1
929 #define FPU_CMPNLT(size, a, b) float ## size ## _lt(a, b, &env->sse_status) ? 0 : -1
930 #define FPU_CMPNLE(size, a, b) float ## size ## _le(a, b, &env->sse_status) ? 0 : -1
931 #define FPU_CMPORD(size, a, b) float ## size ## _unordered(a, b, &env->sse_status) ? 0 : -1
932 
933 SSE_HELPER_CMP(cmpeq, FPU_CMPEQ)
934 SSE_HELPER_CMP(cmplt, FPU_CMPLT)
935 SSE_HELPER_CMP(cmple, FPU_CMPLE)
936 SSE_HELPER_CMP(cmpunord, FPU_CMPUNORD)
937 SSE_HELPER_CMP(cmpneq, FPU_CMPNEQ)
938 SSE_HELPER_CMP(cmpnlt, FPU_CMPNLT)
939 SSE_HELPER_CMP(cmpnle, FPU_CMPNLE)
940 SSE_HELPER_CMP(cmpord, FPU_CMPORD)
941 
942 static const int comis_eflags[4] = {CC_C, CC_Z, 0, CC_Z | CC_P | CC_C};
943 
helper_ucomiss(Reg * d,Reg * s)944 void helper_ucomiss(Reg *d, Reg *s)
945 {
946     int ret;
947     float32 s0, s1;
948 
949     s0 = d->XMM_S(0);
950     s1 = s->XMM_S(0);
951     ret = float32_compare_quiet(s0, s1, &env->sse_status);
952     CC_SRC = comis_eflags[ret + 1];
953 }
954 
helper_comiss(Reg * d,Reg * s)955 void helper_comiss(Reg *d, Reg *s)
956 {
957     int ret;
958     float32 s0, s1;
959 
960     s0 = d->XMM_S(0);
961     s1 = s->XMM_S(0);
962     ret = float32_compare(s0, s1, &env->sse_status);
963     CC_SRC = comis_eflags[ret + 1];
964 }
965 
helper_ucomisd(Reg * d,Reg * s)966 void helper_ucomisd(Reg *d, Reg *s)
967 {
968     int ret;
969     float64 d0, d1;
970 
971     d0 = d->XMM_D(0);
972     d1 = s->XMM_D(0);
973     ret = float64_compare_quiet(d0, d1, &env->sse_status);
974     CC_SRC = comis_eflags[ret + 1];
975 }
976 
helper_comisd(Reg * d,Reg * s)977 void helper_comisd(Reg *d, Reg *s)
978 {
979     int ret;
980     float64 d0, d1;
981 
982     d0 = d->XMM_D(0);
983     d1 = s->XMM_D(0);
984     ret = float64_compare(d0, d1, &env->sse_status);
985     CC_SRC = comis_eflags[ret + 1];
986 }
987 
helper_movmskps(Reg * s)988 uint32_t helper_movmskps(Reg *s)
989 {
990     int b0, b1, b2, b3;
991     b0 = s->XMM_L(0) >> 31;
992     b1 = s->XMM_L(1) >> 31;
993     b2 = s->XMM_L(2) >> 31;
994     b3 = s->XMM_L(3) >> 31;
995     return b0 | (b1 << 1) | (b2 << 2) | (b3 << 3);
996 }
997 
helper_movmskpd(Reg * s)998 uint32_t helper_movmskpd(Reg *s)
999 {
1000     int b0, b1;
1001     b0 = s->XMM_L(1) >> 31;
1002     b1 = s->XMM_L(3) >> 31;
1003     return b0 | (b1 << 1);
1004 }
1005 
1006 #endif
1007 
glue(helper_pmovmskb,SUFFIX)1008 uint32_t glue(helper_pmovmskb, SUFFIX)(Reg *s)
1009 {
1010     uint32_t val;
1011     val = 0;
1012     val |= (s->B(0) >> 7);
1013     val |= (s->B(1) >> 6) & 0x02;
1014     val |= (s->B(2) >> 5) & 0x04;
1015     val |= (s->B(3) >> 4) & 0x08;
1016     val |= (s->B(4) >> 3) & 0x10;
1017     val |= (s->B(5) >> 2) & 0x20;
1018     val |= (s->B(6) >> 1) & 0x40;
1019     val |= (s->B(7)) & 0x80;
1020 #if SHIFT == 1
1021     val |= (s->B(8) << 1) & 0x0100;
1022     val |= (s->B(9) << 2) & 0x0200;
1023     val |= (s->B(10) << 3) & 0x0400;
1024     val |= (s->B(11) << 4) & 0x0800;
1025     val |= (s->B(12) << 5) & 0x1000;
1026     val |= (s->B(13) << 6) & 0x2000;
1027     val |= (s->B(14) << 7) & 0x4000;
1028     val |= (s->B(15) << 8) & 0x8000;
1029 #endif
1030     return val;
1031 }
1032 
glue(helper_packsswb,SUFFIX)1033 void glue(helper_packsswb, SUFFIX) (Reg *d, Reg *s)
1034 {
1035     Reg r;
1036 
1037     r.B(0) = satsb((int16_t)d->W(0));
1038     r.B(1) = satsb((int16_t)d->W(1));
1039     r.B(2) = satsb((int16_t)d->W(2));
1040     r.B(3) = satsb((int16_t)d->W(3));
1041 #if SHIFT == 1
1042     r.B(4) = satsb((int16_t)d->W(4));
1043     r.B(5) = satsb((int16_t)d->W(5));
1044     r.B(6) = satsb((int16_t)d->W(6));
1045     r.B(7) = satsb((int16_t)d->W(7));
1046 #endif
1047     r.B((4 << SHIFT) + 0) = satsb((int16_t)s->W(0));
1048     r.B((4 << SHIFT) + 1) = satsb((int16_t)s->W(1));
1049     r.B((4 << SHIFT) + 2) = satsb((int16_t)s->W(2));
1050     r.B((4 << SHIFT) + 3) = satsb((int16_t)s->W(3));
1051 #if SHIFT == 1
1052     r.B(12) = satsb((int16_t)s->W(4));
1053     r.B(13) = satsb((int16_t)s->W(5));
1054     r.B(14) = satsb((int16_t)s->W(6));
1055     r.B(15) = satsb((int16_t)s->W(7));
1056 #endif
1057     *d = r;
1058 }
1059 
glue(helper_packuswb,SUFFIX)1060 void glue(helper_packuswb, SUFFIX) (Reg *d, Reg *s)
1061 {
1062     Reg r;
1063 
1064     r.B(0) = satub((int16_t)d->W(0));
1065     r.B(1) = satub((int16_t)d->W(1));
1066     r.B(2) = satub((int16_t)d->W(2));
1067     r.B(3) = satub((int16_t)d->W(3));
1068 #if SHIFT == 1
1069     r.B(4) = satub((int16_t)d->W(4));
1070     r.B(5) = satub((int16_t)d->W(5));
1071     r.B(6) = satub((int16_t)d->W(6));
1072     r.B(7) = satub((int16_t)d->W(7));
1073 #endif
1074     r.B((4 << SHIFT) + 0) = satub((int16_t)s->W(0));
1075     r.B((4 << SHIFT) + 1) = satub((int16_t)s->W(1));
1076     r.B((4 << SHIFT) + 2) = satub((int16_t)s->W(2));
1077     r.B((4 << SHIFT) + 3) = satub((int16_t)s->W(3));
1078 #if SHIFT == 1
1079     r.B(12) = satub((int16_t)s->W(4));
1080     r.B(13) = satub((int16_t)s->W(5));
1081     r.B(14) = satub((int16_t)s->W(6));
1082     r.B(15) = satub((int16_t)s->W(7));
1083 #endif
1084     *d = r;
1085 }
1086 
glue(helper_packssdw,SUFFIX)1087 void glue(helper_packssdw, SUFFIX) (Reg *d, Reg *s)
1088 {
1089     Reg r;
1090 
1091     r.W(0) = satsw(d->L(0));
1092     r.W(1) = satsw(d->L(1));
1093 #if SHIFT == 1
1094     r.W(2) = satsw(d->L(2));
1095     r.W(3) = satsw(d->L(3));
1096 #endif
1097     r.W((2 << SHIFT) + 0) = satsw(s->L(0));
1098     r.W((2 << SHIFT) + 1) = satsw(s->L(1));
1099 #if SHIFT == 1
1100     r.W(6) = satsw(s->L(2));
1101     r.W(7) = satsw(s->L(3));
1102 #endif
1103     *d = r;
1104 }
1105 
1106 #define UNPCK_OP(base_name, base)                               \
1107                                                                 \
1108 void glue(helper_punpck ## base_name ## bw, SUFFIX) (Reg *d, Reg *s)   \
1109 {                                                               \
1110     Reg r;                                              \
1111                                                                 \
1112     r.B(0) = d->B((base << (SHIFT + 2)) + 0);                   \
1113     r.B(1) = s->B((base << (SHIFT + 2)) + 0);                   \
1114     r.B(2) = d->B((base << (SHIFT + 2)) + 1);                   \
1115     r.B(3) = s->B((base << (SHIFT + 2)) + 1);                   \
1116     r.B(4) = d->B((base << (SHIFT + 2)) + 2);                   \
1117     r.B(5) = s->B((base << (SHIFT + 2)) + 2);                   \
1118     r.B(6) = d->B((base << (SHIFT + 2)) + 3);                   \
1119     r.B(7) = s->B((base << (SHIFT + 2)) + 3);                   \
1120 XMM_ONLY(                                                       \
1121     r.B(8) = d->B((base << (SHIFT + 2)) + 4);                   \
1122     r.B(9) = s->B((base << (SHIFT + 2)) + 4);                   \
1123     r.B(10) = d->B((base << (SHIFT + 2)) + 5);                  \
1124     r.B(11) = s->B((base << (SHIFT + 2)) + 5);                  \
1125     r.B(12) = d->B((base << (SHIFT + 2)) + 6);                  \
1126     r.B(13) = s->B((base << (SHIFT + 2)) + 6);                  \
1127     r.B(14) = d->B((base << (SHIFT + 2)) + 7);                  \
1128     r.B(15) = s->B((base << (SHIFT + 2)) + 7);                  \
1129 )                                                               \
1130     *d = r;                                                     \
1131 }                                                               \
1132                                                                 \
1133 void glue(helper_punpck ## base_name ## wd, SUFFIX) (Reg *d, Reg *s)   \
1134 {                                                               \
1135     Reg r;                                              \
1136                                                                 \
1137     r.W(0) = d->W((base << (SHIFT + 1)) + 0);                   \
1138     r.W(1) = s->W((base << (SHIFT + 1)) + 0);                   \
1139     r.W(2) = d->W((base << (SHIFT + 1)) + 1);                   \
1140     r.W(3) = s->W((base << (SHIFT + 1)) + 1);                   \
1141 XMM_ONLY(                                                       \
1142     r.W(4) = d->W((base << (SHIFT + 1)) + 2);                   \
1143     r.W(5) = s->W((base << (SHIFT + 1)) + 2);                   \
1144     r.W(6) = d->W((base << (SHIFT + 1)) + 3);                   \
1145     r.W(7) = s->W((base << (SHIFT + 1)) + 3);                   \
1146 )                                                               \
1147     *d = r;                                                     \
1148 }                                                               \
1149                                                                 \
1150 void glue(helper_punpck ## base_name ## dq, SUFFIX) (Reg *d, Reg *s)   \
1151 {                                                               \
1152     Reg r;                                              \
1153                                                                 \
1154     r.L(0) = d->L((base << SHIFT) + 0);                         \
1155     r.L(1) = s->L((base << SHIFT) + 0);                         \
1156 XMM_ONLY(                                                       \
1157     r.L(2) = d->L((base << SHIFT) + 1);                         \
1158     r.L(3) = s->L((base << SHIFT) + 1);                         \
1159 )                                                               \
1160     *d = r;                                                     \
1161 }                                                               \
1162                                                                 \
1163 XMM_ONLY(                                                       \
1164 void glue(helper_punpck ## base_name ## qdq, SUFFIX) (Reg *d, Reg *s)  \
1165 {                                                               \
1166     Reg r;                                              \
1167                                                                 \
1168     r.Q(0) = d->Q(base);                                        \
1169     r.Q(1) = s->Q(base);                                        \
1170     *d = r;                                                     \
1171 }                                                               \
1172 )
1173 
1174 UNPCK_OP(l, 0)
1175 UNPCK_OP(h, 1)
1176 
1177 /* 3DNow! float ops */
1178 #if SHIFT == 0
1179 void helper_pi2fd(MMXReg *d, MMXReg *s)
1180 {
1181     d->MMX_S(0) = int32_to_float32(s->MMX_L(0), &env->mmx_status);
1182     d->MMX_S(1) = int32_to_float32(s->MMX_L(1), &env->mmx_status);
1183 }
1184 
helper_pi2fw(MMXReg * d,MMXReg * s)1185 void helper_pi2fw(MMXReg *d, MMXReg *s)
1186 {
1187     d->MMX_S(0) = int32_to_float32((int16_t)s->MMX_W(0), &env->mmx_status);
1188     d->MMX_S(1) = int32_to_float32((int16_t)s->MMX_W(2), &env->mmx_status);
1189 }
1190 
helper_pf2id(MMXReg * d,MMXReg * s)1191 void helper_pf2id(MMXReg *d, MMXReg *s)
1192 {
1193     d->MMX_L(0) = float32_to_int32_round_to_zero(s->MMX_S(0), &env->mmx_status);
1194     d->MMX_L(1) = float32_to_int32_round_to_zero(s->MMX_S(1), &env->mmx_status);
1195 }
1196 
helper_pf2iw(MMXReg * d,MMXReg * s)1197 void helper_pf2iw(MMXReg *d, MMXReg *s)
1198 {
1199     d->MMX_L(0) = satsw(float32_to_int32_round_to_zero(s->MMX_S(0), &env->mmx_status));
1200     d->MMX_L(1) = satsw(float32_to_int32_round_to_zero(s->MMX_S(1), &env->mmx_status));
1201 }
1202 
helper_pfacc(MMXReg * d,MMXReg * s)1203 void helper_pfacc(MMXReg *d, MMXReg *s)
1204 {
1205     MMXReg r;
1206     r.MMX_S(0) = float32_add(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
1207     r.MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
1208     *d = r;
1209 }
1210 
helper_pfadd(MMXReg * d,MMXReg * s)1211 void helper_pfadd(MMXReg *d, MMXReg *s)
1212 {
1213     d->MMX_S(0) = float32_add(d->MMX_S(0), s->MMX_S(0), &env->mmx_status);
1214     d->MMX_S(1) = float32_add(d->MMX_S(1), s->MMX_S(1), &env->mmx_status);
1215 }
1216 
helper_pfcmpeq(MMXReg * d,MMXReg * s)1217 void helper_pfcmpeq(MMXReg *d, MMXReg *s)
1218 {
1219     d->MMX_L(0) = float32_eq(d->MMX_S(0), s->MMX_S(0), &env->mmx_status) ? -1 : 0;
1220     d->MMX_L(1) = float32_eq(d->MMX_S(1), s->MMX_S(1), &env->mmx_status) ? -1 : 0;
1221 }
1222 
helper_pfcmpge(MMXReg * d,MMXReg * s)1223 void helper_pfcmpge(MMXReg *d, MMXReg *s)
1224 {
1225     d->MMX_L(0) = float32_le(s->MMX_S(0), d->MMX_S(0), &env->mmx_status) ? -1 : 0;
1226     d->MMX_L(1) = float32_le(s->MMX_S(1), d->MMX_S(1), &env->mmx_status) ? -1 : 0;
1227 }
1228 
helper_pfcmpgt(MMXReg * d,MMXReg * s)1229 void helper_pfcmpgt(MMXReg *d, MMXReg *s)
1230 {
1231     d->MMX_L(0) = float32_lt(s->MMX_S(0), d->MMX_S(0), &env->mmx_status) ? -1 : 0;
1232     d->MMX_L(1) = float32_lt(s->MMX_S(1), d->MMX_S(1), &env->mmx_status) ? -1 : 0;
1233 }
1234 
helper_pfmax(MMXReg * d,MMXReg * s)1235 void helper_pfmax(MMXReg *d, MMXReg *s)
1236 {
1237     if (float32_lt(d->MMX_S(0), s->MMX_S(0), &env->mmx_status))
1238         d->MMX_S(0) = s->MMX_S(0);
1239     if (float32_lt(d->MMX_S(1), s->MMX_S(1), &env->mmx_status))
1240         d->MMX_S(1) = s->MMX_S(1);
1241 }
1242 
helper_pfmin(MMXReg * d,MMXReg * s)1243 void helper_pfmin(MMXReg *d, MMXReg *s)
1244 {
1245     if (float32_lt(s->MMX_S(0), d->MMX_S(0), &env->mmx_status))
1246         d->MMX_S(0) = s->MMX_S(0);
1247     if (float32_lt(s->MMX_S(1), d->MMX_S(1), &env->mmx_status))
1248         d->MMX_S(1) = s->MMX_S(1);
1249 }
1250 
helper_pfmul(MMXReg * d,MMXReg * s)1251 void helper_pfmul(MMXReg *d, MMXReg *s)
1252 {
1253     d->MMX_S(0) = float32_mul(d->MMX_S(0), s->MMX_S(0), &env->mmx_status);
1254     d->MMX_S(1) = float32_mul(d->MMX_S(1), s->MMX_S(1), &env->mmx_status);
1255 }
1256 
helper_pfnacc(MMXReg * d,MMXReg * s)1257 void helper_pfnacc(MMXReg *d, MMXReg *s)
1258 {
1259     MMXReg r;
1260     r.MMX_S(0) = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
1261     r.MMX_S(1) = float32_sub(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
1262     *d = r;
1263 }
1264 
helper_pfpnacc(MMXReg * d,MMXReg * s)1265 void helper_pfpnacc(MMXReg *d, MMXReg *s)
1266 {
1267     MMXReg r;
1268     r.MMX_S(0) = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
1269     r.MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
1270     *d = r;
1271 }
1272 
helper_pfrcp(MMXReg * d,MMXReg * s)1273 void helper_pfrcp(MMXReg *d, MMXReg *s)
1274 {
1275     d->MMX_S(0) = approx_rcp(s->MMX_S(0));
1276     d->MMX_S(1) = d->MMX_S(0);
1277 }
1278 
helper_pfrsqrt(MMXReg * d,MMXReg * s)1279 void helper_pfrsqrt(MMXReg *d, MMXReg *s)
1280 {
1281     d->MMX_L(1) = s->MMX_L(0) & 0x7fffffff;
1282     d->MMX_S(1) = approx_rsqrt(d->MMX_S(1));
1283     d->MMX_L(1) |= s->MMX_L(0) & 0x80000000;
1284     d->MMX_L(0) = d->MMX_L(1);
1285 }
1286 
helper_pfsub(MMXReg * d,MMXReg * s)1287 void helper_pfsub(MMXReg *d, MMXReg *s)
1288 {
1289     d->MMX_S(0) = float32_sub(d->MMX_S(0), s->MMX_S(0), &env->mmx_status);
1290     d->MMX_S(1) = float32_sub(d->MMX_S(1), s->MMX_S(1), &env->mmx_status);
1291 }
1292 
helper_pfsubr(MMXReg * d,MMXReg * s)1293 void helper_pfsubr(MMXReg *d, MMXReg *s)
1294 {
1295     d->MMX_S(0) = float32_sub(s->MMX_S(0), d->MMX_S(0), &env->mmx_status);
1296     d->MMX_S(1) = float32_sub(s->MMX_S(1), d->MMX_S(1), &env->mmx_status);
1297 }
1298 
helper_pswapd(MMXReg * d,MMXReg * s)1299 void helper_pswapd(MMXReg *d, MMXReg *s)
1300 {
1301     MMXReg r;
1302     r.MMX_L(0) = s->MMX_L(1);
1303     r.MMX_L(1) = s->MMX_L(0);
1304     *d = r;
1305 }
1306 #endif
1307 
1308 /* SSSE3 op helpers */
glue(helper_pshufb,SUFFIX)1309 void glue(helper_pshufb, SUFFIX) (Reg *d, Reg *s)
1310 {
1311     int i;
1312     Reg r;
1313 
1314     for (i = 0; i < (8 << SHIFT); i++)
1315         r.B(i) = (s->B(i) & 0x80) ? 0 : (d->B(s->B(i) & ((8 << SHIFT) - 1)));
1316 
1317     *d = r;
1318 }
1319 
glue(helper_phaddw,SUFFIX)1320 void glue(helper_phaddw, SUFFIX) (Reg *d, Reg *s)
1321 {
1322     d->W(0) = (int16_t)d->W(0) + (int16_t)d->W(1);
1323     d->W(1) = (int16_t)d->W(2) + (int16_t)d->W(3);
1324     XMM_ONLY(d->W(2) = (int16_t)d->W(4) + (int16_t)d->W(5));
1325     XMM_ONLY(d->W(3) = (int16_t)d->W(6) + (int16_t)d->W(7));
1326     d->W((2 << SHIFT) + 0) = (int16_t)s->W(0) + (int16_t)s->W(1);
1327     d->W((2 << SHIFT) + 1) = (int16_t)s->W(2) + (int16_t)s->W(3);
1328     XMM_ONLY(d->W(6) = (int16_t)s->W(4) + (int16_t)s->W(5));
1329     XMM_ONLY(d->W(7) = (int16_t)s->W(6) + (int16_t)s->W(7));
1330 }
1331 
glue(helper_phaddd,SUFFIX)1332 void glue(helper_phaddd, SUFFIX) (Reg *d, Reg *s)
1333 {
1334     d->L(0) = (int32_t)d->L(0) + (int32_t)d->L(1);
1335     XMM_ONLY(d->L(1) = (int32_t)d->L(2) + (int32_t)d->L(3));
1336     d->L((1 << SHIFT) + 0) = (int32_t)s->L(0) + (int32_t)s->L(1);
1337     XMM_ONLY(d->L(3) = (int32_t)s->L(2) + (int32_t)s->L(3));
1338 }
1339 
glue(helper_phaddsw,SUFFIX)1340 void glue(helper_phaddsw, SUFFIX) (Reg *d, Reg *s)
1341 {
1342     d->W(0) = satsw((int16_t)d->W(0) + (int16_t)d->W(1));
1343     d->W(1) = satsw((int16_t)d->W(2) + (int16_t)d->W(3));
1344     XMM_ONLY(d->W(2) = satsw((int16_t)d->W(4) + (int16_t)d->W(5)));
1345     XMM_ONLY(d->W(3) = satsw((int16_t)d->W(6) + (int16_t)d->W(7)));
1346     d->W((2 << SHIFT) + 0) = satsw((int16_t)s->W(0) + (int16_t)s->W(1));
1347     d->W((2 << SHIFT) + 1) = satsw((int16_t)s->W(2) + (int16_t)s->W(3));
1348     XMM_ONLY(d->W(6) = satsw((int16_t)s->W(4) + (int16_t)s->W(5)));
1349     XMM_ONLY(d->W(7) = satsw((int16_t)s->W(6) + (int16_t)s->W(7)));
1350 }
1351 
glue(helper_pmaddubsw,SUFFIX)1352 void glue(helper_pmaddubsw, SUFFIX) (Reg *d, Reg *s)
1353 {
1354     d->W(0) = satsw((int8_t)s->B( 0) * (uint8_t)d->B( 0) +
1355                     (int8_t)s->B( 1) * (uint8_t)d->B( 1));
1356     d->W(1) = satsw((int8_t)s->B( 2) * (uint8_t)d->B( 2) +
1357                     (int8_t)s->B( 3) * (uint8_t)d->B( 3));
1358     d->W(2) = satsw((int8_t)s->B( 4) * (uint8_t)d->B( 4) +
1359                     (int8_t)s->B( 5) * (uint8_t)d->B( 5));
1360     d->W(3) = satsw((int8_t)s->B( 6) * (uint8_t)d->B( 6) +
1361                     (int8_t)s->B( 7) * (uint8_t)d->B( 7));
1362 #if SHIFT == 1
1363     d->W(4) = satsw((int8_t)s->B( 8) * (uint8_t)d->B( 8) +
1364                     (int8_t)s->B( 9) * (uint8_t)d->B( 9));
1365     d->W(5) = satsw((int8_t)s->B(10) * (uint8_t)d->B(10) +
1366                     (int8_t)s->B(11) * (uint8_t)d->B(11));
1367     d->W(6) = satsw((int8_t)s->B(12) * (uint8_t)d->B(12) +
1368                     (int8_t)s->B(13) * (uint8_t)d->B(13));
1369     d->W(7) = satsw((int8_t)s->B(14) * (uint8_t)d->B(14) +
1370                     (int8_t)s->B(15) * (uint8_t)d->B(15));
1371 #endif
1372 }
1373 
glue(helper_phsubw,SUFFIX)1374 void glue(helper_phsubw, SUFFIX) (Reg *d, Reg *s)
1375 {
1376     d->W(0) = (int16_t)d->W(0) - (int16_t)d->W(1);
1377     d->W(1) = (int16_t)d->W(2) - (int16_t)d->W(3);
1378     XMM_ONLY(d->W(2) = (int16_t)d->W(4) - (int16_t)d->W(5));
1379     XMM_ONLY(d->W(3) = (int16_t)d->W(6) - (int16_t)d->W(7));
1380     d->W((2 << SHIFT) + 0) = (int16_t)s->W(0) - (int16_t)s->W(1);
1381     d->W((2 << SHIFT) + 1) = (int16_t)s->W(2) - (int16_t)s->W(3);
1382     XMM_ONLY(d->W(6) = (int16_t)s->W(4) - (int16_t)s->W(5));
1383     XMM_ONLY(d->W(7) = (int16_t)s->W(6) - (int16_t)s->W(7));
1384 }
1385 
glue(helper_phsubd,SUFFIX)1386 void glue(helper_phsubd, SUFFIX) (Reg *d, Reg *s)
1387 {
1388     d->L(0) = (int32_t)d->L(0) - (int32_t)d->L(1);
1389     XMM_ONLY(d->L(1) = (int32_t)d->L(2) - (int32_t)d->L(3));
1390     d->L((1 << SHIFT) + 0) = (int32_t)s->L(0) - (int32_t)s->L(1);
1391     XMM_ONLY(d->L(3) = (int32_t)s->L(2) - (int32_t)s->L(3));
1392 }
1393 
glue(helper_phsubsw,SUFFIX)1394 void glue(helper_phsubsw, SUFFIX) (Reg *d, Reg *s)
1395 {
1396     d->W(0) = satsw((int16_t)d->W(0) - (int16_t)d->W(1));
1397     d->W(1) = satsw((int16_t)d->W(2) - (int16_t)d->W(3));
1398     XMM_ONLY(d->W(2) = satsw((int16_t)d->W(4) - (int16_t)d->W(5)));
1399     XMM_ONLY(d->W(3) = satsw((int16_t)d->W(6) - (int16_t)d->W(7)));
1400     d->W((2 << SHIFT) + 0) = satsw((int16_t)s->W(0) - (int16_t)s->W(1));
1401     d->W((2 << SHIFT) + 1) = satsw((int16_t)s->W(2) - (int16_t)s->W(3));
1402     XMM_ONLY(d->W(6) = satsw((int16_t)s->W(4) - (int16_t)s->W(5)));
1403     XMM_ONLY(d->W(7) = satsw((int16_t)s->W(6) - (int16_t)s->W(7)));
1404 }
1405 
1406 #define FABSB(_, x) x > INT8_MAX  ? -(int8_t ) x : x
1407 #define FABSW(_, x) x > INT16_MAX ? -(int16_t) x : x
1408 #define FABSL(_, x) x > INT32_MAX ? -(int32_t) x : x
SSE_HELPER_B(helper_pabsb,FABSB)1409 SSE_HELPER_B(helper_pabsb, FABSB)
1410 SSE_HELPER_W(helper_pabsw, FABSW)
1411 SSE_HELPER_L(helper_pabsd, FABSL)
1412 
1413 #define FMULHRSW(d, s) ((int16_t) d * (int16_t) s + 0x4000) >> 15
1414 SSE_HELPER_W(helper_pmulhrsw, FMULHRSW)
1415 
1416 #define FSIGNB(d, s) s <= INT8_MAX  ? s ? d : 0 : -(int8_t ) d
1417 #define FSIGNW(d, s) s <= INT16_MAX ? s ? d : 0 : -(int16_t) d
1418 #define FSIGNL(d, s) s <= INT32_MAX ? s ? d : 0 : -(int32_t) d
1419 SSE_HELPER_B(helper_psignb, FSIGNB)
1420 SSE_HELPER_W(helper_psignw, FSIGNW)
1421 SSE_HELPER_L(helper_psignd, FSIGNL)
1422 
1423 void glue(helper_palignr, SUFFIX) (Reg *d, Reg *s, int32_t shift)
1424 {
1425     Reg r;
1426 
1427     /* XXX could be checked during translation */
1428     if (shift >= (16 << SHIFT)) {
1429         r.Q(0) = 0;
1430         XMM_ONLY(r.Q(1) = 0);
1431     } else {
1432         shift <<= 3;
1433 #define SHR(v, i) (i < 64 && i > -64 ? i > 0 ? v >> (i) : (v << -(i)) : 0)
1434 #if SHIFT == 0
1435         r.Q(0) = SHR(s->Q(0), shift -   0) |
1436                  SHR(d->Q(0), shift -  64);
1437 #else
1438         r.Q(0) = SHR(s->Q(0), shift -   0) |
1439                  SHR(s->Q(1), shift -  64) |
1440                  SHR(d->Q(0), shift - 128) |
1441                  SHR(d->Q(1), shift - 192);
1442         r.Q(1) = SHR(s->Q(0), shift +  64) |
1443                  SHR(s->Q(1), shift -   0) |
1444                  SHR(d->Q(0), shift -  64) |
1445                  SHR(d->Q(1), shift - 128);
1446 #endif
1447 #undef SHR
1448     }
1449 
1450     *d = r;
1451 }
1452 
1453 #define XMM0 env->xmm_regs[0]
1454 
1455 #if SHIFT == 1
1456 #define SSE_HELPER_V(name, elem, num, F)\
1457 void glue(name, SUFFIX) (Reg *d, Reg *s)\
1458 {\
1459     d->elem(0) = F(d->elem(0), s->elem(0), XMM0.elem(0));\
1460     d->elem(1) = F(d->elem(1), s->elem(1), XMM0.elem(1));\
1461     if (num > 2) {\
1462         d->elem(2) = F(d->elem(2), s->elem(2), XMM0.elem(2));\
1463         d->elem(3) = F(d->elem(3), s->elem(3), XMM0.elem(3));\
1464         if (num > 4) {\
1465             d->elem(4) = F(d->elem(4), s->elem(4), XMM0.elem(4));\
1466             d->elem(5) = F(d->elem(5), s->elem(5), XMM0.elem(5));\
1467             d->elem(6) = F(d->elem(6), s->elem(6), XMM0.elem(6));\
1468             d->elem(7) = F(d->elem(7), s->elem(7), XMM0.elem(7));\
1469             if (num > 8) {\
1470                 d->elem(8) = F(d->elem(8), s->elem(8), XMM0.elem(8));\
1471                 d->elem(9) = F(d->elem(9), s->elem(9), XMM0.elem(9));\
1472                 d->elem(10) = F(d->elem(10), s->elem(10), XMM0.elem(10));\
1473                 d->elem(11) = F(d->elem(11), s->elem(11), XMM0.elem(11));\
1474                 d->elem(12) = F(d->elem(12), s->elem(12), XMM0.elem(12));\
1475                 d->elem(13) = F(d->elem(13), s->elem(13), XMM0.elem(13));\
1476                 d->elem(14) = F(d->elem(14), s->elem(14), XMM0.elem(14));\
1477                 d->elem(15) = F(d->elem(15), s->elem(15), XMM0.elem(15));\
1478             }\
1479         }\
1480     }\
1481 }
1482 
1483 #define SSE_HELPER_I(name, elem, num, F)\
1484 void glue(name, SUFFIX) (Reg *d, Reg *s, uint32_t imm)\
1485 {\
1486     d->elem(0) = F(d->elem(0), s->elem(0), ((imm >> 0) & 1));\
1487     d->elem(1) = F(d->elem(1), s->elem(1), ((imm >> 1) & 1));\
1488     if (num > 2) {\
1489         d->elem(2) = F(d->elem(2), s->elem(2), ((imm >> 2) & 1));\
1490         d->elem(3) = F(d->elem(3), s->elem(3), ((imm >> 3) & 1));\
1491         if (num > 4) {\
1492             d->elem(4) = F(d->elem(4), s->elem(4), ((imm >> 4) & 1));\
1493             d->elem(5) = F(d->elem(5), s->elem(5), ((imm >> 5) & 1));\
1494             d->elem(6) = F(d->elem(6), s->elem(6), ((imm >> 6) & 1));\
1495             d->elem(7) = F(d->elem(7), s->elem(7), ((imm >> 7) & 1));\
1496             if (num > 8) {\
1497                 d->elem(8) = F(d->elem(8), s->elem(8), ((imm >> 8) & 1));\
1498                 d->elem(9) = F(d->elem(9), s->elem(9), ((imm >> 9) & 1));\
1499                 d->elem(10) = F(d->elem(10), s->elem(10), ((imm >> 10) & 1));\
1500                 d->elem(11) = F(d->elem(11), s->elem(11), ((imm >> 11) & 1));\
1501                 d->elem(12) = F(d->elem(12), s->elem(12), ((imm >> 12) & 1));\
1502                 d->elem(13) = F(d->elem(13), s->elem(13), ((imm >> 13) & 1));\
1503                 d->elem(14) = F(d->elem(14), s->elem(14), ((imm >> 14) & 1));\
1504                 d->elem(15) = F(d->elem(15), s->elem(15), ((imm >> 15) & 1));\
1505             }\
1506         }\
1507     }\
1508 }
1509 
1510 /* SSE4.1 op helpers */
1511 #define FBLENDVB(d, s, m) (m & 0x80) ? s : d
1512 #define FBLENDVPS(d, s, m) (m & 0x80000000) ? s : d
1513 #define FBLENDVPD(d, s, m) (m & 0x8000000000000000LL) ? s : d
1514 SSE_HELPER_V(helper_pblendvb, B, 16, FBLENDVB)
1515 SSE_HELPER_V(helper_blendvps, L, 4, FBLENDVPS)
1516 SSE_HELPER_V(helper_blendvpd, Q, 2, FBLENDVPD)
1517 
1518 void glue(helper_ptest, SUFFIX) (Reg *d, Reg *s)
1519 {
1520     uint64_t zf = (s->Q(0) &  d->Q(0)) | (s->Q(1) &  d->Q(1));
1521     uint64_t cf = (s->Q(0) & ~d->Q(0)) | (s->Q(1) & ~d->Q(1));
1522 
1523     CC_SRC = (zf ? 0 : CC_Z) | (cf ? 0 : CC_C);
1524 }
1525 
1526 #define SSE_HELPER_F(name, elem, num, F)\
1527 void glue(name, SUFFIX) (Reg *d, Reg *s)\
1528 {\
1529     d->elem(0) = F(0);\
1530     d->elem(1) = F(1);\
1531     if (num > 2) {\
1532         d->elem(2) = F(2);\
1533         d->elem(3) = F(3);\
1534         if (num > 4) {\
1535             d->elem(4) = F(4);\
1536             d->elem(5) = F(5);\
1537             d->elem(6) = F(6);\
1538             d->elem(7) = F(7);\
1539         }\
1540     }\
1541 }
1542 
1543 SSE_HELPER_F(helper_pmovsxbw, W, 8, (int8_t) s->B)
1544 SSE_HELPER_F(helper_pmovsxbd, L, 4, (int8_t) s->B)
1545 SSE_HELPER_F(helper_pmovsxbq, Q, 2, (int8_t) s->B)
1546 SSE_HELPER_F(helper_pmovsxwd, L, 4, (int16_t) s->W)
1547 SSE_HELPER_F(helper_pmovsxwq, Q, 2, (int16_t) s->W)
1548 SSE_HELPER_F(helper_pmovsxdq, Q, 2, (int32_t) s->L)
1549 SSE_HELPER_F(helper_pmovzxbw, W, 8, s->B)
1550 SSE_HELPER_F(helper_pmovzxbd, L, 4, s->B)
1551 SSE_HELPER_F(helper_pmovzxbq, Q, 2, s->B)
1552 SSE_HELPER_F(helper_pmovzxwd, L, 4, s->W)
1553 SSE_HELPER_F(helper_pmovzxwq, Q, 2, s->W)
1554 SSE_HELPER_F(helper_pmovzxdq, Q, 2, s->L)
1555 
1556 void glue(helper_pmuldq, SUFFIX) (Reg *d, Reg *s)
1557 {
1558     d->Q(0) = (int64_t) (int32_t) d->L(0) * (int32_t) s->L(0);
1559     d->Q(1) = (int64_t) (int32_t) d->L(2) * (int32_t) s->L(2);
1560 }
1561 
1562 #define FCMPEQQ(d, s) d == s ? -1 : 0
SSE_HELPER_Q(helper_pcmpeqq,FCMPEQQ)1563 SSE_HELPER_Q(helper_pcmpeqq, FCMPEQQ)
1564 
1565 void glue(helper_packusdw, SUFFIX) (Reg *d, Reg *s)
1566 {
1567     d->W(0) = satuw((int32_t) d->L(0));
1568     d->W(1) = satuw((int32_t) d->L(1));
1569     d->W(2) = satuw((int32_t) d->L(2));
1570     d->W(3) = satuw((int32_t) d->L(3));
1571     d->W(4) = satuw((int32_t) s->L(0));
1572     d->W(5) = satuw((int32_t) s->L(1));
1573     d->W(6) = satuw((int32_t) s->L(2));
1574     d->W(7) = satuw((int32_t) s->L(3));
1575 }
1576 
1577 #define FMINSB(d, s) MIN((int8_t) d, (int8_t) s)
1578 #define FMINSD(d, s) MIN((int32_t) d, (int32_t) s)
1579 #define FMAXSB(d, s) MAX((int8_t) d, (int8_t) s)
1580 #define FMAXSD(d, s) MAX((int32_t) d, (int32_t) s)
SSE_HELPER_B(helper_pminsb,FMINSB)1581 SSE_HELPER_B(helper_pminsb, FMINSB)
1582 SSE_HELPER_L(helper_pminsd, FMINSD)
1583 SSE_HELPER_W(helper_pminuw, MIN)
1584 SSE_HELPER_L(helper_pminud, MIN)
1585 SSE_HELPER_B(helper_pmaxsb, FMAXSB)
1586 SSE_HELPER_L(helper_pmaxsd, FMAXSD)
1587 SSE_HELPER_W(helper_pmaxuw, MAX)
1588 SSE_HELPER_L(helper_pmaxud, MAX)
1589 
1590 #define FMULLD(d, s) (int32_t) d * (int32_t) s
1591 SSE_HELPER_L(helper_pmulld, FMULLD)
1592 
1593 void glue(helper_phminposuw, SUFFIX) (Reg *d, Reg *s)
1594 {
1595     int idx = 0;
1596 
1597     if (s->W(1) < s->W(idx))
1598         idx = 1;
1599     if (s->W(2) < s->W(idx))
1600         idx = 2;
1601     if (s->W(3) < s->W(idx))
1602         idx = 3;
1603     if (s->W(4) < s->W(idx))
1604         idx = 4;
1605     if (s->W(5) < s->W(idx))
1606         idx = 5;
1607     if (s->W(6) < s->W(idx))
1608         idx = 6;
1609     if (s->W(7) < s->W(idx))
1610         idx = 7;
1611 
1612     d->Q(1) = 0;
1613     d->L(1) = 0;
1614     d->W(1) = idx;
1615     d->W(0) = s->W(idx);
1616 }
1617 
glue(helper_roundps,SUFFIX)1618 void glue(helper_roundps, SUFFIX) (Reg *d, Reg *s, uint32_t mode)
1619 {
1620     signed char prev_rounding_mode;
1621 
1622     prev_rounding_mode = env->sse_status.float_rounding_mode;
1623     if (!(mode & (1 << 2)))
1624         switch (mode & 3) {
1625         case 0:
1626             set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
1627             break;
1628         case 1:
1629             set_float_rounding_mode(float_round_down, &env->sse_status);
1630             break;
1631         case 2:
1632             set_float_rounding_mode(float_round_up, &env->sse_status);
1633             break;
1634         case 3:
1635             set_float_rounding_mode(float_round_to_zero, &env->sse_status);
1636             break;
1637         }
1638 
1639     d->L(0) = float64_round_to_int(s->L(0), &env->sse_status);
1640     d->L(1) = float64_round_to_int(s->L(1), &env->sse_status);
1641     d->L(2) = float64_round_to_int(s->L(2), &env->sse_status);
1642     d->L(3) = float64_round_to_int(s->L(3), &env->sse_status);
1643 
1644 #if 0 /* TODO */
1645     if (mode & (1 << 3))
1646         set_float_exception_flags(
1647                         get_float_exception_flags(&env->sse_status) &
1648                         ~float_flag_inexact,
1649                         &env->sse_status);
1650 #endif
1651     env->sse_status.float_rounding_mode = prev_rounding_mode;
1652 }
1653 
glue(helper_roundpd,SUFFIX)1654 void glue(helper_roundpd, SUFFIX) (Reg *d, Reg *s, uint32_t mode)
1655 {
1656     signed char prev_rounding_mode;
1657 
1658     prev_rounding_mode = env->sse_status.float_rounding_mode;
1659     if (!(mode & (1 << 2)))
1660         switch (mode & 3) {
1661         case 0:
1662             set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
1663             break;
1664         case 1:
1665             set_float_rounding_mode(float_round_down, &env->sse_status);
1666             break;
1667         case 2:
1668             set_float_rounding_mode(float_round_up, &env->sse_status);
1669             break;
1670         case 3:
1671             set_float_rounding_mode(float_round_to_zero, &env->sse_status);
1672             break;
1673         }
1674 
1675     d->Q(0) = float64_round_to_int(s->Q(0), &env->sse_status);
1676     d->Q(1) = float64_round_to_int(s->Q(1), &env->sse_status);
1677 
1678 #if 0 /* TODO */
1679     if (mode & (1 << 3))
1680         set_float_exception_flags(
1681                         get_float_exception_flags(&env->sse_status) &
1682                         ~float_flag_inexact,
1683                         &env->sse_status);
1684 #endif
1685     env->sse_status.float_rounding_mode = prev_rounding_mode;
1686 }
1687 
glue(helper_roundss,SUFFIX)1688 void glue(helper_roundss, SUFFIX) (Reg *d, Reg *s, uint32_t mode)
1689 {
1690     signed char prev_rounding_mode;
1691 
1692     prev_rounding_mode = env->sse_status.float_rounding_mode;
1693     if (!(mode & (1 << 2)))
1694         switch (mode & 3) {
1695         case 0:
1696             set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
1697             break;
1698         case 1:
1699             set_float_rounding_mode(float_round_down, &env->sse_status);
1700             break;
1701         case 2:
1702             set_float_rounding_mode(float_round_up, &env->sse_status);
1703             break;
1704         case 3:
1705             set_float_rounding_mode(float_round_to_zero, &env->sse_status);
1706             break;
1707         }
1708 
1709     d->L(0) = float64_round_to_int(s->L(0), &env->sse_status);
1710 
1711 #if 0 /* TODO */
1712     if (mode & (1 << 3))
1713         set_float_exception_flags(
1714                         get_float_exception_flags(&env->sse_status) &
1715                         ~float_flag_inexact,
1716                         &env->sse_status);
1717 #endif
1718     env->sse_status.float_rounding_mode = prev_rounding_mode;
1719 }
1720 
glue(helper_roundsd,SUFFIX)1721 void glue(helper_roundsd, SUFFIX) (Reg *d, Reg *s, uint32_t mode)
1722 {
1723     signed char prev_rounding_mode;
1724 
1725     prev_rounding_mode = env->sse_status.float_rounding_mode;
1726     if (!(mode & (1 << 2)))
1727         switch (mode & 3) {
1728         case 0:
1729             set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
1730             break;
1731         case 1:
1732             set_float_rounding_mode(float_round_down, &env->sse_status);
1733             break;
1734         case 2:
1735             set_float_rounding_mode(float_round_up, &env->sse_status);
1736             break;
1737         case 3:
1738             set_float_rounding_mode(float_round_to_zero, &env->sse_status);
1739             break;
1740         }
1741 
1742     d->Q(0) = float64_round_to_int(s->Q(0), &env->sse_status);
1743 
1744 #if 0 /* TODO */
1745     if (mode & (1 << 3))
1746         set_float_exception_flags(
1747                         get_float_exception_flags(&env->sse_status) &
1748                         ~float_flag_inexact,
1749                         &env->sse_status);
1750 #endif
1751     env->sse_status.float_rounding_mode = prev_rounding_mode;
1752 }
1753 
1754 #define FBLENDP(d, s, m) m ? s : d
1755 SSE_HELPER_I(helper_blendps, L, 4, FBLENDP)
1756 SSE_HELPER_I(helper_blendpd, Q, 2, FBLENDP)
1757 SSE_HELPER_I(helper_pblendw, W, 8, FBLENDP)
1758 
1759 void glue(helper_dpps, SUFFIX) (Reg *d, Reg *s, uint32_t mask)
1760 {
1761     float32 iresult = 0 /*float32_zero*/;
1762 
1763     if (mask & (1 << 4))
1764         iresult = float32_add(iresult,
1765                         float32_mul(d->L(0), s->L(0), &env->sse_status),
1766                         &env->sse_status);
1767     if (mask & (1 << 5))
1768         iresult = float32_add(iresult,
1769                         float32_mul(d->L(1), s->L(1), &env->sse_status),
1770                         &env->sse_status);
1771     if (mask & (1 << 6))
1772         iresult = float32_add(iresult,
1773                         float32_mul(d->L(2), s->L(2), &env->sse_status),
1774                         &env->sse_status);
1775     if (mask & (1 << 7))
1776         iresult = float32_add(iresult,
1777                         float32_mul(d->L(3), s->L(3), &env->sse_status),
1778                         &env->sse_status);
1779     d->L(0) = (mask & (1 << 0)) ? iresult : 0 /*float32_zero*/;
1780     d->L(1) = (mask & (1 << 1)) ? iresult : 0 /*float32_zero*/;
1781     d->L(2) = (mask & (1 << 2)) ? iresult : 0 /*float32_zero*/;
1782     d->L(3) = (mask & (1 << 3)) ? iresult : 0 /*float32_zero*/;
1783 }
1784 
glue(helper_dppd,SUFFIX)1785 void glue(helper_dppd, SUFFIX) (Reg *d, Reg *s, uint32_t mask)
1786 {
1787     float64 iresult = 0 /*float64_zero*/;
1788 
1789     if (mask & (1 << 4))
1790         iresult = float64_add(iresult,
1791                         float64_mul(d->Q(0), s->Q(0), &env->sse_status),
1792                         &env->sse_status);
1793     if (mask & (1 << 5))
1794         iresult = float64_add(iresult,
1795                         float64_mul(d->Q(1), s->Q(1), &env->sse_status),
1796                         &env->sse_status);
1797     d->Q(0) = (mask & (1 << 0)) ? iresult : 0 /*float64_zero*/;
1798     d->Q(1) = (mask & (1 << 1)) ? iresult : 0 /*float64_zero*/;
1799 }
1800 
glue(helper_mpsadbw,SUFFIX)1801 void glue(helper_mpsadbw, SUFFIX) (Reg *d, Reg *s, uint32_t offset)
1802 {
1803     int s0 = (offset & 3) << 2;
1804     int d0 = (offset & 4) << 0;
1805     int i;
1806     Reg r;
1807 
1808     for (i = 0; i < 8; i++, d0++) {
1809         r.W(i) = 0;
1810         r.W(i) += abs1(d->B(d0 + 0) - s->B(s0 + 0));
1811         r.W(i) += abs1(d->B(d0 + 1) - s->B(s0 + 1));
1812         r.W(i) += abs1(d->B(d0 + 2) - s->B(s0 + 2));
1813         r.W(i) += abs1(d->B(d0 + 3) - s->B(s0 + 3));
1814     }
1815 
1816     *d = r;
1817 }
1818 
1819 /* SSE4.2 op helpers */
1820 /* it's unclear whether signed or unsigned */
1821 #define FCMPGTQ(d, s) d > s ? -1 : 0
SSE_HELPER_Q(helper_pcmpgtq,FCMPGTQ)1822 SSE_HELPER_Q(helper_pcmpgtq, FCMPGTQ)
1823 
1824 static inline int pcmp_elen(int reg, uint32_t ctrl)
1825 {
1826     int val;
1827 
1828     /* Presence of REX.W is indicated by a bit higher than 7 set */
1829     if (ctrl >> 8)
1830         val = abs1((int64_t) env->regs[reg]);
1831     else
1832         val = abs1((int32_t) env->regs[reg]);
1833 
1834     if (ctrl & 1) {
1835         if (val > 8)
1836             return 8;
1837     } else
1838         if (val > 16)
1839             return 16;
1840 
1841     return val;
1842 }
1843 
pcmp_ilen(Reg * r,uint8_t ctrl)1844 static inline int pcmp_ilen(Reg *r, uint8_t ctrl)
1845 {
1846     int val = 0;
1847 
1848     if (ctrl & 1) {
1849         while (val < 8 && r->W(val))
1850             val++;
1851     } else
1852         while (val < 16 && r->B(val))
1853             val++;
1854 
1855     return val;
1856 }
1857 
pcmp_val(Reg * r,uint8_t ctrl,int i)1858 static inline int pcmp_val(Reg *r, uint8_t ctrl, int i)
1859 {
1860     switch ((ctrl >> 0) & 3) {
1861     case 0:
1862         return r->B(i);
1863     case 1:
1864         return r->W(i);
1865     case 2:
1866         return (int8_t) r->B(i);
1867     case 3:
1868     default:
1869         return (int16_t) r->W(i);
1870     }
1871 }
1872 
pcmpxstrx(Reg * d,Reg * s,int8_t ctrl,int valids,int validd)1873 static inline unsigned pcmpxstrx(Reg *d, Reg *s,
1874                 int8_t ctrl, int valids, int validd)
1875 {
1876     unsigned int res = 0;
1877     int v;
1878     int j, i;
1879     int upper = (ctrl & 1) ? 7 : 15;
1880 
1881     valids--;
1882     validd--;
1883 
1884     CC_SRC = (valids < upper ? CC_Z : 0) | (validd < upper ? CC_S : 0);
1885 
1886     switch ((ctrl >> 2) & 3) {
1887     case 0:
1888         for (j = valids; j >= 0; j--) {
1889             res <<= 1;
1890             v = pcmp_val(s, ctrl, j);
1891             for (i = validd; i >= 0; i--)
1892                 res |= (v == pcmp_val(d, ctrl, i));
1893         }
1894         break;
1895     case 1:
1896         for (j = valids; j >= 0; j--) {
1897             res <<= 1;
1898             v = pcmp_val(s, ctrl, j);
1899             for (i = ((validd - 1) | 1); i >= 0;