31 #define round(bias) ((int)(((bias)+0.5) * (1<<ROW_SHIFT)))
32 #define rounder(bias) {round (bias), round (bias)}
37 static inline void idct_row (int16_t * row,
int offset,
38 int16_t * table, int32_t *
rounder)
53 a0 = C4*row[0] + C2*row[2] + C4*row[4] + C6*row[6] + *
rounder;
54 a1 = C4*row[0] + C6*row[2] - C4*row[4] - C2*row[6] + *
rounder;
55 a2 = C4*row[0] - C6*row[2] - C4*row[4] + C2*row[6] + *
rounder;
56 a3 = C4*row[0] - C2*row[2] + C4*row[4] - C6*row[6] + *
rounder;
58 b0 = C1*row[1] + C3*row[3] + C5*row[5] + C7*row[7];
59 b1 = C3*row[1] - C7*row[3] - C1*row[5] - C5*row[7];
60 b2 = C5*row[1] - C1*row[3] + C7*row[5] + C3*row[7];
61 b3 = C7*row[1] - C5*row[3] + C3*row[5] - C1*row[7];
77 #define mmxext_table(c1,c2,c3,c4,c5,c6,c7) { c4, c2, -c4, -c2, \
87 const int16_t *
const table)
90 "movq (%0), %%mm2 \n\t"
92 "movq 8(%0), %%mm5 \n\t"
93 "movq %%mm2, %%mm0 \n\t"
95 "movq (%1), %%mm3 \n\t"
96 "movq %%mm5, %%mm6 \n\t"
98 "movq 8(%1), %%mm4 \n\t"
99 "pmaddwd %%mm0, %%mm3 \n\t"
101 "pshufw $0x4e, %%mm2, %%mm2 \n\t"
102 ::
"r" ((row+offset)),
"r" (table)
110 "movq 16(%0), %%mm1 \n\t"
111 "pmaddwd %%mm2, %%mm4 \n\t"
113 "pmaddwd 32(%0), %%mm0 \n\t"
114 "pshufw $0x4e, %%mm6, %%mm6 \n\t"
116 "movq 24(%0), %%mm7 \n\t"
117 "pmaddwd %%mm5, %%mm1 \n\t"
119 "paddd (%1), %%mm3 \n\t"
120 "pmaddwd %%mm6, %%mm7 \n\t"
122 "pmaddwd 40(%0), %%mm2 \n\t"
123 "paddd %%mm4, %%mm3 \n\t"
125 "pmaddwd 48(%0), %%mm5 \n\t"
126 "movq %%mm3, %%mm4 \n\t"
128 "pmaddwd 56(%0), %%mm6 \n\t"
129 "paddd %%mm7, %%mm1 \n\t"
131 "paddd (%1), %%mm0 \n\t"
132 "psubd %%mm1, %%mm3 \n\t"
135 "paddd %%mm4, %%mm1 \n\t"
137 "paddd %%mm2, %%mm0 \n\t"
140 "paddd %%mm6, %%mm5 \n\t"
141 "movq %%mm0, %%mm4 \n\t"
143 "paddd %%mm5, %%mm0 \n\t"
144 "psubd %%mm5, %%mm4 \n\t"
145 : :
"r" (table),
"r" (rounder));
155 "packssdw %%mm0, %%mm1 \n\t"
157 "packssdw %%mm3, %%mm4 \n\t"
159 "movq %%mm1, (%0) \n\t"
160 "pshufw $0xb1, %%mm4, %%mm4 \n\t"
164 "movq %%mm4, 8(%0) \n\t"
171 const int16_t *
const table)
174 "movq (%0,%1), %%mm2 \n\t"
177 "movq 8(%0,%1), %%mm5 \n\t"
180 "packssdw %%mm0, %%mm1 \n\t"
181 "movq %%mm5, %%mm6 \n\t"
183 "packssdw %%mm3, %%mm4 \n\t"
184 "movq %%mm2, %%mm0 \n\t"
186 "movq %%mm1, (%0,%2) \n\t"
187 "pshufw $0xb1, %%mm4, %%mm4\n\t"
189 "movq (%3), %%mm3 \n\t"
190 "movq %%mm4, 8(%0,%2) \n\t"
192 "pmaddwd %%mm0, %%mm3 \n\t"
194 "movq 8(%3), %%mm4 \n\t"
195 "pshufw $0x4e, %%mm2, %%mm2\n\t"
196 ::
"r" (row),
"r" ((
x86_reg) (2*offset)),
"r" ((
x86_reg) (2*store)),
"r" (table)
203 #define mmx_table(c1,c2,c3,c4,c5,c6,c7) { c4, c2, c4, c6, \
212 static inline void mmx_row_head (int16_t *
const row,
const int offset,
213 const int16_t *
const table)
216 "movq (%0), %%mm2 \n\t"
218 "movq 8(%0), %%mm5 \n\t"
219 "movq %%mm2, %%mm0 \n\t"
221 "movq (%1), %%mm3 \n\t"
222 "movq %%mm5, %%mm6 \n\t"
224 "punpckldq %%mm0, %%mm0 \n\t"
226 "movq 8(%1), %%mm4 \n\t"
227 "pmaddwd %%mm0, %%mm3 \n\t"
229 "movq 16(%1), %%mm1 \n\t"
230 "punpckhdq %%mm2, %%mm2 \n\t"
231 ::
"r" ((row+offset)),
"r" (table)
235 static inline void mmx_row (
const int16_t *
const table,
239 "pmaddwd %%mm2, %%mm4 \n\t"
240 "punpckldq %%mm5, %%mm5 \n\t"
242 "pmaddwd 32(%0), %%mm0 \n\t"
243 "punpckhdq %%mm6, %%mm6 \n\t"
245 "movq 24(%0), %%mm7 \n\t"
246 "pmaddwd %%mm5, %%mm1 \n\t"
248 "paddd (%1), %%mm3 \n\t"
249 "pmaddwd %%mm6, %%mm7 \n\t"
251 "pmaddwd 40(%0), %%mm2 \n\t"
252 "paddd %%mm4, %%mm3 \n\t"
254 "pmaddwd 48(%0), %%mm5 \n\t"
255 "movq %%mm3, %%mm4 \n\t"
257 "pmaddwd 56(%0), %%mm6 \n\t"
258 "paddd %%mm7, %%mm1 \n\t"
260 "paddd (%1), %%mm0 \n\t"
261 "psubd %%mm1, %%mm3 \n\t"
264 "paddd %%mm4, %%mm1 \n\t"
266 "paddd %%mm2, %%mm0 \n\t"
269 "paddd %%mm6, %%mm5 \n\t"
270 "movq %%mm0, %%mm7 \n\t"
272 "paddd %%mm5, %%mm0 \n\t"
273 "psubd %%mm5, %%mm7 \n\t"
274 ::
"r" (table),
"r" (rounder)
285 "packssdw %%mm0, %%mm1 \n\t"
287 "packssdw %%mm3, %%mm7 \n\t"
289 "movq %%mm1, (%0) \n\t"
290 "movq %%mm7, %%mm4 \n\t"
292 "pslld $16, %%mm7 \n\t"
294 "psrld $16, %%mm4 \n\t"
296 "por %%mm4, %%mm7 \n\t"
300 "movq %%mm7, 8(%0) \n\t"
305 static inline void mmx_row_mid (int16_t *
const row,
const int store,
306 const int offset,
const int16_t *
const table)
310 "movq (%0,%1), %%mm2 \n\t"
313 "movq 8(%0,%1), %%mm5 \n\t"
316 "packssdw %%mm0, %%mm1 \n\t"
317 "movq %%mm5, %%mm6 \n\t"
319 "packssdw %%mm3, %%mm7 \n\t"
320 "movq %%mm2, %%mm0 \n\t"
322 "movq %%mm1, (%0,%2) \n\t"
323 "movq %%mm7, %%mm1 \n\t"
325 "punpckldq %%mm0, %%mm0 \n\t"
326 "psrld $16, %%mm7 \n\t"
328 "movq (%3), %%mm3 \n\t"
329 "pslld $16, %%mm1 \n\t"
331 "movq 8(%3), %%mm4 \n\t"
332 "por %%mm1, %%mm7 \n\t"
334 "movq 16(%3), %%mm1 \n\t"
335 "punpckhdq %%mm2, %%mm2 \n\t"
337 "movq %%mm7, 8(%0,%2) \n\t"
338 "pmaddwd %%mm0, %%mm3 \n\t"
339 : :
"r" (row),
"r" ((
x86_reg) (2*offset)),
"r" ((
x86_reg) (2*store)),
"r" (table)
346 static inline void idct_col (int16_t * col,
int offset)
349 #define F(c,x) (((c) * (x)) >> 16)
352 #define S(x) (((x)>32767) ? 32767 : ((x)<-32768) ? -32768 : (x))
354 int16_t x0, x1, x2, x3, x4, x5, x6, x7;
355 int16_t y0, y1, y2, y3, y4, y5, y6, y7;
356 int16_t
a0,
a1,
a2,
a3, b0, b1, b2, b3;
357 int16_t u04, v04, u26, v26, u17, v17, u35, v35, u12, v12;
372 u26 =
S (
F (T2, x6) + x2);
373 v26 =
S (
F (T2, x2) - x6);
380 u17 =
S (
F (T1, x7) + x1);
381 v17 =
S (
F (T1, x1) - x7);
382 u35 =
S (
F (T3, x5) + x3);
383 v35 =
S (
F (T3, x3) - x5);
389 u12 =
S (2 *
F (C4, u12));
390 v12 =
S (2 *
F (C4, v12));
417 static inline void idct_col (int16_t *
const col,
const int offset)
435 "movq (%0), %%mm0 \n\t"
437 "movq 2*8(%1), %%mm1 \n\t"
438 "movq %%mm0, %%mm2 \n\t"
440 "movq 7*2*8(%1), %%mm4 \n\t"
441 "pmulhw %%mm1, %%mm0 \n\t"
443 "movq 16(%0), %%mm5 \n\t"
444 "pmulhw %%mm4, %%mm2 \n\t"
446 "movq 2*5*8(%1), %%mm6 \n\t"
447 "movq %%mm5, %%mm7 \n\t"
449 "movq 3*8*2(%1), %%mm3 \n\t"
450 "psubsw %%mm4, %%mm0 \n\t"
452 "movq 8(%0), %%mm4 \n\t"
453 "pmulhw %%mm3, %%mm5 \n\t"
455 "paddsw %%mm2, %%mm1 \n\t"
456 "pmulhw %%mm6, %%mm7 \n\t"
460 "movq %%mm4, %%mm2 \n\t"
461 "paddsw %%mm3, %%mm5 \n\t"
463 "pmulhw 2*8*2(%1), %%mm4 \n\t"
464 "paddsw %%mm6, %%mm7 \n\t"
466 "psubsw %%mm6, %%mm5 \n\t"
467 "paddsw %%mm3, %%mm7 \n\t"
469 "movq 6*8*2(%1), %%mm3 \n\t"
470 "movq %%mm0, %%mm6 \n\t"
472 "pmulhw %%mm3, %%mm2 \n\t"
473 "psubsw %%mm5, %%mm0 \n\t"
475 "psubsw %%mm3, %%mm4 \n\t"
476 "paddsw %%mm6, %%mm5 \n\t"
478 "movq %%mm0, 3*8*2(%1)\n\t"
479 "movq %%mm1, %%mm6 \n\t"
481 "paddsw 2*8*2(%1), %%mm2 \n\t"
482 "paddsw %%mm7, %%mm6 \n\t"
484 "psubsw %%mm7, %%mm1 \n\t"
485 "movq %%mm1, %%mm7 \n\t"
487 "movq 0*8(%1), %%mm3 \n\t"
488 "paddsw %%mm5, %%mm1 \n\t"
490 "movq 24(%0), %%mm0 \n\t"
491 "psubsw %%mm5, %%mm7 \n\t"
493 "movq %%mm6, 5*8*2(%1)\n\t"
494 "pmulhw %%mm0, %%mm1 \n\t"
496 "movq %%mm4, %%mm6 \n\t"
497 "pmulhw %%mm0, %%mm7 \n\t"
499 "movq 4*8*2(%1), %%mm5 \n\t"
500 "movq %%mm3, %%mm0 \n\t"
502 "psubsw %%mm5, %%mm3 \n\t"
503 "paddsw %%mm5, %%mm0 \n\t"
505 "paddsw %%mm3, %%mm4 \n\t"
506 "movq %%mm0, %%mm5 \n\t"
508 "psubsw %%mm6, %%mm3 \n\t"
509 "paddsw %%mm2, %%mm5 \n\t"
511 "paddsw %%mm1, %%mm1 \n\t"
512 "psubsw %%mm2, %%mm0 \n\t"
514 "paddsw %%mm7, %%mm7 \n\t"
515 "movq %%mm3, %%mm2 \n\t"
517 "movq %%mm4, %%mm6 \n\t"
518 "paddsw %%mm7, %%mm3 \n\t"
521 "paddsw %%mm1, %%mm4\n\t"
524 "psubsw %%mm1, %%mm6 \n\t"
526 "movq 5*8*2(%1), %%mm1 \n\t"
527 "psubsw %%mm7, %%mm2 \n\t"
530 "movq %%mm5, %%mm7 \n\t"
532 "movq %%mm4, 1*8*2(%1)\n\t"
535 "movq %%mm3, 2*8*2(%1)\n\t"
536 "paddsw %%mm1, %%mm5 \n\t"
538 "movq 3*8*2(%1), %%mm4 \n\t"
539 "psubsw %%mm1, %%mm7 \n\t"
542 "movq %%mm0, %%mm3 \n\t"
544 "movq %%mm2, 5*8*2(%1)\n\t"
545 "psubsw %%mm4, %%mm3 \n\t"
548 "paddsw %%mm0, %%mm4 \n\t"
550 "movq %%mm5, 0*8*2(%1)\n\t"
553 "movq %%mm6, 6*8*2(%1)\n\t"
556 "movq %%mm7, 7*8*2(%1)\n\t"
558 "movq %%mm3, 4*8*2(%1)\n\t"
560 "movq %%mm4, 3*8*2(%1)\n\t"
561 ::
"r" (t1_vector),
"r" (col+offset)
590 #define declare_idct(idct,table,idct_row_head,idct_row,idct_row_tail,idct_row_mid) \
591 void idct (int16_t * const block) \
593 DECLARE_ALIGNED(16, static const int16_t, table04)[] = \
594 table (22725, 21407, 19266, 16384, 12873, 8867, 4520); \
595 DECLARE_ALIGNED(16, static const int16_t, table17)[] = \
596 table (31521, 29692, 26722, 22725, 17855, 12299, 6270); \
597 DECLARE_ALIGNED(16, static const int16_t, table26)[] = \
598 table (29692, 27969, 25172, 21407, 16819, 11585, 5906); \
599 DECLARE_ALIGNED(16, static const int16_t, table35)[] = \
600 table (26722, 25172, 22654, 19266, 15137, 10426, 5315); \
602 idct_row_head (block, 0*8, table04); \
603 idct_row (table04, rounder0); \
604 idct_row_mid (block, 0*8, 4*8, table04); \
605 idct_row (table04, rounder4); \
606 idct_row_mid (block, 4*8, 1*8, table17); \
607 idct_row (table17, rounder1); \
608 idct_row_mid (block, 1*8, 7*8, table17); \
609 idct_row (table17, rounder7); \
610 idct_row_mid (block, 7*8, 2*8, table26); \
611 idct_row (table26, rounder2); \
612 idct_row_mid (block, 2*8, 6*8, table26); \
613 idct_row (table26, rounder6); \
614 idct_row_mid (block, 6*8, 3*8, table35); \
615 idct_row (table35, rounder3); \
616 idct_row_mid (block, 3*8, 5*8, table35); \
617 idct_row (table35, rounder5); \
618 idct_row_tail (block, 5*8); \
620 idct_col (block, 0); \
621 idct_col (block, 4); \