36 #define C0 23170 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
37 #define C1 22725 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
38 #define C2 21407 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
39 #define C3 19266 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
40 #define C4 16383 //cos(i*M_PI/16)*sqrt(2)*(1<<14) - 0.5
41 #define C5 12873 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
42 #define C6 8867 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
43 #define C7 4520 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
46 #define COL_SHIFT 20 // 6
82 int16_t *
const temp= (int16_t*)align_tmp;
85 #if 0 //Alternative, simpler variant
87 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
88 "movq " #src0 ", %%mm0 \n\t" \
89 "movq " #src4 ", %%mm1 \n\t" \
90 "movq " #src1 ", %%mm2 \n\t" \
91 "movq " #src5 ", %%mm3 \n\t" \
92 "movq 16(%2), %%mm4 \n\t" \
93 "pmaddwd %%mm0, %%mm4 \n\t" \
94 "movq 24(%2), %%mm5 \n\t" \
95 "pmaddwd %%mm5, %%mm0 \n\t" \
96 "movq 32(%2), %%mm5 \n\t" \
97 "pmaddwd %%mm1, %%mm5 \n\t" \
98 "movq 40(%2), %%mm6 \n\t" \
99 "pmaddwd %%mm6, %%mm1 \n\t" \
100 "movq 48(%2), %%mm7 \n\t" \
101 "pmaddwd %%mm2, %%mm7 \n\t" \
102 #rounder ", %%mm4 \n\t"\
103 "movq %%mm4, %%mm6 \n\t" \
104 "paddd %%mm5, %%mm4 \n\t" \
105 "psubd %%mm5, %%mm6 \n\t" \
106 "movq 56(%2), %%mm5 \n\t" \
107 "pmaddwd %%mm3, %%mm5 \n\t" \
108 #rounder ", %%mm0 \n\t"\
109 "paddd %%mm0, %%mm1 \n\t" \
110 "paddd %%mm0, %%mm0 \n\t" \
111 "psubd %%mm1, %%mm0 \n\t" \
112 "pmaddwd 64(%2), %%mm2 \n\t" \
113 "paddd %%mm5, %%mm7 \n\t" \
114 "movq 72(%2), %%mm5 \n\t" \
115 "pmaddwd %%mm3, %%mm5 \n\t" \
116 "paddd %%mm4, %%mm7 \n\t" \
117 "paddd %%mm4, %%mm4 \n\t" \
118 "psubd %%mm7, %%mm4 \n\t" \
119 "paddd %%mm2, %%mm5 \n\t" \
120 "psrad $" #shift ", %%mm7 \n\t"\
121 "psrad $" #shift ", %%mm4 \n\t"\
122 "movq %%mm1, %%mm2 \n\t" \
123 "paddd %%mm5, %%mm1 \n\t" \
124 "psubd %%mm5, %%mm2 \n\t" \
125 "psrad $" #shift ", %%mm1 \n\t"\
126 "psrad $" #shift ", %%mm2 \n\t"\
127 "packssdw %%mm1, %%mm7 \n\t" \
128 "packssdw %%mm4, %%mm2 \n\t" \
129 "movq %%mm7, " #dst " \n\t"\
130 "movq " #src1 ", %%mm1 \n\t" \
131 "movq 80(%2), %%mm4 \n\t" \
132 "movq %%mm2, 24+" #dst " \n\t"\
133 "pmaddwd %%mm1, %%mm4 \n\t" \
134 "movq 88(%2), %%mm7 \n\t" \
135 "pmaddwd 96(%2), %%mm1 \n\t" \
136 "pmaddwd %%mm3, %%mm7 \n\t" \
137 "movq %%mm0, %%mm2 \n\t" \
138 "pmaddwd 104(%2), %%mm3 \n\t" \
139 "paddd %%mm7, %%mm4 \n\t" \
140 "paddd %%mm4, %%mm2 \n\t" \
141 "psubd %%mm4, %%mm0 \n\t" \
142 "psrad $" #shift ", %%mm2 \n\t"\
143 "psrad $" #shift ", %%mm0 \n\t"\
144 "movq %%mm6, %%mm4 \n\t" \
145 "paddd %%mm1, %%mm3 \n\t" \
146 "paddd %%mm3, %%mm6 \n\t" \
147 "psubd %%mm3, %%mm4 \n\t" \
148 "psrad $" #shift ", %%mm6 \n\t"\
149 "packssdw %%mm6, %%mm2 \n\t" \
150 "movq %%mm2, 8+" #dst " \n\t"\
151 "psrad $" #shift ", %%mm4 \n\t"\
152 "packssdw %%mm0, %%mm4 \n\t" \
153 "movq %%mm4, 16+" #dst " \n\t"\
155 #define COL_IDCT(src0, src4, src1, src5, dst, shift) \
156 "movq " #src0 ", %%mm0 \n\t" \
157 "movq " #src4 ", %%mm1 \n\t" \
158 "movq " #src1 ", %%mm2 \n\t" \
159 "movq " #src5 ", %%mm3 \n\t" \
160 "movq 16(%2), %%mm4 \n\t" \
161 "pmaddwd %%mm0, %%mm4 \n\t" \
162 "movq 24(%2), %%mm5 \n\t" \
163 "pmaddwd %%mm5, %%mm0 \n\t" \
164 "movq 32(%2), %%mm5 \n\t" \
165 "pmaddwd %%mm1, %%mm5 \n\t" \
166 "movq 40(%2), %%mm6 \n\t" \
167 "pmaddwd %%mm6, %%mm1 \n\t" \
168 "movq %%mm4, %%mm6 \n\t" \
169 "movq 48(%2), %%mm7 \n\t" \
170 "pmaddwd %%mm2, %%mm7 \n\t" \
171 "paddd %%mm5, %%mm4 \n\t" \
172 "psubd %%mm5, %%mm6 \n\t" \
173 "movq %%mm0, %%mm5 \n\t" \
174 "paddd %%mm1, %%mm0 \n\t" \
175 "psubd %%mm1, %%mm5 \n\t" \
176 "movq 56(%2), %%mm1 \n\t" \
177 "pmaddwd %%mm3, %%mm1 \n\t" \
178 "pmaddwd 64(%2), %%mm2 \n\t" \
179 "paddd %%mm1, %%mm7 \n\t" \
180 "movq 72(%2), %%mm1 \n\t" \
181 "pmaddwd %%mm3, %%mm1 \n\t" \
182 "paddd %%mm4, %%mm7 \n\t" \
183 "paddd %%mm4, %%mm4 \n\t" \
184 "psubd %%mm7, %%mm4 \n\t" \
185 "paddd %%mm2, %%mm1 \n\t" \
186 "psrad $" #shift ", %%mm7 \n\t"\
187 "psrad $" #shift ", %%mm4 \n\t"\
188 "movq %%mm0, %%mm2 \n\t" \
189 "paddd %%mm1, %%mm0 \n\t" \
190 "psubd %%mm1, %%mm2 \n\t" \
191 "psrad $" #shift ", %%mm0 \n\t"\
192 "psrad $" #shift ", %%mm2 \n\t"\
193 "packssdw %%mm7, %%mm7 \n\t" \
194 "movd %%mm7, " #dst " \n\t"\
195 "packssdw %%mm0, %%mm0 \n\t" \
196 "movd %%mm0, 16+" #dst " \n\t"\
197 "packssdw %%mm2, %%mm2 \n\t" \
198 "movd %%mm2, 96+" #dst " \n\t"\
199 "packssdw %%mm4, %%mm4 \n\t" \
200 "movd %%mm4, 112+" #dst " \n\t"\
201 "movq " #src1 ", %%mm0 \n\t" \
202 "movq 80(%2), %%mm4 \n\t" \
203 "pmaddwd %%mm0, %%mm4 \n\t" \
204 "movq 88(%2), %%mm7 \n\t" \
205 "pmaddwd 96(%2), %%mm0 \n\t" \
206 "pmaddwd %%mm3, %%mm7 \n\t" \
207 "movq %%mm5, %%mm2 \n\t" \
208 "pmaddwd 104(%2), %%mm3 \n\t" \
209 "paddd %%mm7, %%mm4 \n\t" \
210 "paddd %%mm4, %%mm2 \n\t" \
211 "psubd %%mm4, %%mm5 \n\t" \
212 "psrad $" #shift ", %%mm2 \n\t"\
213 "psrad $" #shift ", %%mm5 \n\t"\
214 "movq %%mm6, %%mm4 \n\t" \
215 "paddd %%mm0, %%mm3 \n\t" \
216 "paddd %%mm3, %%mm6 \n\t" \
217 "psubd %%mm3, %%mm4 \n\t" \
218 "psrad $" #shift ", %%mm6 \n\t"\
219 "psrad $" #shift ", %%mm4 \n\t"\
220 "packssdw %%mm2, %%mm2 \n\t" \
221 "packssdw %%mm6, %%mm6 \n\t" \
222 "movd %%mm2, 32+" #dst " \n\t"\
223 "packssdw %%mm4, %%mm4 \n\t" \
224 "packssdw %%mm5, %%mm5 \n\t" \
225 "movd %%mm6, 48+" #dst " \n\t"\
226 "movd %%mm4, 64+" #dst " \n\t"\
227 "movd %%mm5, 80+" #dst " \n\t"\
230 #define DC_COND_ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
231 "movq " #src0 ", %%mm0 \n\t" \
232 "movq " #src4 ", %%mm1 \n\t" \
233 "movq " #src1 ", %%mm2 \n\t" \
234 "movq " #src5 ", %%mm3 \n\t" \
235 "movq "MANGLE(wm1010)", %%mm4 \n\t"\
236 "pand %%mm0, %%mm4 \n\t"\
237 "por %%mm1, %%mm4 \n\t"\
238 "por %%mm2, %%mm4 \n\t"\
239 "por %%mm3, %%mm4 \n\t"\
240 "packssdw %%mm4,%%mm4 \n\t"\
241 "movd %%mm4, %%eax \n\t"\
242 "orl %%eax, %%eax \n\t"\
244 "movq 16(%2), %%mm4 \n\t" \
245 "pmaddwd %%mm0, %%mm4 \n\t" \
246 "movq 24(%2), %%mm5 \n\t" \
247 "pmaddwd %%mm5, %%mm0 \n\t" \
248 "movq 32(%2), %%mm5 \n\t" \
249 "pmaddwd %%mm1, %%mm5 \n\t" \
250 "movq 40(%2), %%mm6 \n\t" \
251 "pmaddwd %%mm6, %%mm1 \n\t" \
252 "movq 48(%2), %%mm7 \n\t" \
253 "pmaddwd %%mm2, %%mm7 \n\t" \
254 #rounder ", %%mm4 \n\t"\
255 "movq %%mm4, %%mm6 \n\t" \
256 "paddd %%mm5, %%mm4 \n\t" \
257 "psubd %%mm5, %%mm6 \n\t" \
258 "movq 56(%2), %%mm5 \n\t" \
259 "pmaddwd %%mm3, %%mm5 \n\t" \
260 #rounder ", %%mm0 \n\t"\
261 "paddd %%mm0, %%mm1 \n\t" \
262 "paddd %%mm0, %%mm0 \n\t" \
263 "psubd %%mm1, %%mm0 \n\t" \
264 "pmaddwd 64(%2), %%mm2 \n\t" \
265 "paddd %%mm5, %%mm7 \n\t" \
266 "movq 72(%2), %%mm5 \n\t" \
267 "pmaddwd %%mm3, %%mm5 \n\t" \
268 "paddd %%mm4, %%mm7 \n\t" \
269 "paddd %%mm4, %%mm4 \n\t" \
270 "psubd %%mm7, %%mm4 \n\t" \
271 "paddd %%mm2, %%mm5 \n\t" \
272 "psrad $" #shift ", %%mm7 \n\t"\
273 "psrad $" #shift ", %%mm4 \n\t"\
274 "movq %%mm1, %%mm2 \n\t" \
275 "paddd %%mm5, %%mm1 \n\t" \
276 "psubd %%mm5, %%mm2 \n\t" \
277 "psrad $" #shift ", %%mm1 \n\t"\
278 "psrad $" #shift ", %%mm2 \n\t"\
279 "packssdw %%mm1, %%mm7 \n\t" \
280 "packssdw %%mm4, %%mm2 \n\t" \
281 "movq %%mm7, " #dst " \n\t"\
282 "movq " #src1 ", %%mm1 \n\t" \
283 "movq 80(%2), %%mm4 \n\t" \
284 "movq %%mm2, 24+" #dst " \n\t"\
285 "pmaddwd %%mm1, %%mm4 \n\t" \
286 "movq 88(%2), %%mm7 \n\t" \
287 "pmaddwd 96(%2), %%mm1 \n\t" \
288 "pmaddwd %%mm3, %%mm7 \n\t" \
289 "movq %%mm0, %%mm2 \n\t" \
290 "pmaddwd 104(%2), %%mm3 \n\t" \
291 "paddd %%mm7, %%mm4 \n\t" \
292 "paddd %%mm4, %%mm2 \n\t" \
293 "psubd %%mm4, %%mm0 \n\t" \
294 "psrad $" #shift ", %%mm2 \n\t"\
295 "psrad $" #shift ", %%mm0 \n\t"\
296 "movq %%mm6, %%mm4 \n\t" \
297 "paddd %%mm1, %%mm3 \n\t" \
298 "paddd %%mm3, %%mm6 \n\t" \
299 "psubd %%mm3, %%mm4 \n\t" \
300 "psrad $" #shift ", %%mm6 \n\t"\
301 "packssdw %%mm6, %%mm2 \n\t" \
302 "movq %%mm2, 8+" #dst " \n\t"\
303 "psrad $" #shift ", %%mm4 \n\t"\
304 "packssdw %%mm0, %%mm4 \n\t" \
305 "movq %%mm4, 16+" #dst " \n\t"\
308 "pslld $16, %%mm0 \n\t"\
309 "#paddd "MANGLE(d40000)", %%mm0 \n\t"\
310 "psrad $13, %%mm0 \n\t"\
311 "packssdw %%mm0, %%mm0 \n\t"\
312 "movq %%mm0, " #dst " \n\t"\
313 "movq %%mm0, 8+" #dst " \n\t"\
314 "movq %%mm0, 16+" #dst " \n\t"\
315 "movq %%mm0, 24+" #dst " \n\t"\
320 ROW_IDCT( (%0), 8(%0), 16(%0), 24(%0), 0(%1),paddd 8(%2), 11)
325 DC_COND_ROW_IDCT( 32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11)
326 DC_COND_ROW_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11)
327 DC_COND_ROW_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11)
331 COL_IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
332 COL_IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
333 COL_IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
334 COL_IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
338 #define DC_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
339 "movq " #src0 ", %%mm0 \n\t" \
340 "movq " #src4 ", %%mm1 \n\t" \
341 "movq " #src1 ", %%mm2 \n\t" \
342 "movq " #src5 ", %%mm3 \n\t" \
343 "movq "MANGLE(wm1010)", %%mm4 \n\t"\
344 "pand %%mm0, %%mm4 \n\t"\
345 "por %%mm1, %%mm4 \n\t"\
346 "por %%mm2, %%mm4 \n\t"\
347 "por %%mm3, %%mm4 \n\t"\
348 "packssdw %%mm4,%%mm4 \n\t"\
349 "movd %%mm4, %%eax \n\t"\
350 "orl %%eax, %%eax \n\t"\
352 "movq 16(%2), %%mm4 \n\t" \
353 "pmaddwd %%mm0, %%mm4 \n\t" \
354 "movq 24(%2), %%mm5 \n\t" \
355 "pmaddwd %%mm5, %%mm0 \n\t" \
356 "movq 32(%2), %%mm5 \n\t" \
357 "pmaddwd %%mm1, %%mm5 \n\t" \
358 "movq 40(%2), %%mm6 \n\t" \
359 "pmaddwd %%mm6, %%mm1 \n\t" \
360 "movq 48(%2), %%mm7 \n\t" \
361 "pmaddwd %%mm2, %%mm7 \n\t" \
362 #rounder ", %%mm4 \n\t"\
363 "movq %%mm4, %%mm6 \n\t" \
364 "paddd %%mm5, %%mm4 \n\t" \
365 "psubd %%mm5, %%mm6 \n\t" \
366 "movq 56(%2), %%mm5 \n\t" \
367 "pmaddwd %%mm3, %%mm5 \n\t" \
368 #rounder ", %%mm0 \n\t"\
369 "paddd %%mm0, %%mm1 \n\t" \
370 "paddd %%mm0, %%mm0 \n\t" \
371 "psubd %%mm1, %%mm0 \n\t" \
372 "pmaddwd 64(%2), %%mm2 \n\t" \
373 "paddd %%mm5, %%mm7 \n\t" \
374 "movq 72(%2), %%mm5 \n\t" \
375 "pmaddwd %%mm3, %%mm5 \n\t" \
376 "paddd %%mm4, %%mm7 \n\t" \
377 "paddd %%mm4, %%mm4 \n\t" \
378 "psubd %%mm7, %%mm4 \n\t" \
379 "paddd %%mm2, %%mm5 \n\t" \
380 "psrad $" #shift ", %%mm7 \n\t"\
381 "psrad $" #shift ", %%mm4 \n\t"\
382 "movq %%mm1, %%mm2 \n\t" \
383 "paddd %%mm5, %%mm1 \n\t" \
384 "psubd %%mm5, %%mm2 \n\t" \
385 "psrad $" #shift ", %%mm1 \n\t"\
386 "psrad $" #shift ", %%mm2 \n\t"\
387 "packssdw %%mm1, %%mm7 \n\t" \
388 "packssdw %%mm4, %%mm2 \n\t" \
389 "movq %%mm7, " #dst " \n\t"\
390 "movq " #src1 ", %%mm1 \n\t" \
391 "movq 80(%2), %%mm4 \n\t" \
392 "movq %%mm2, 24+" #dst " \n\t"\
393 "pmaddwd %%mm1, %%mm4 \n\t" \
394 "movq 88(%2), %%mm7 \n\t" \
395 "pmaddwd 96(%2), %%mm1 \n\t" \
396 "pmaddwd %%mm3, %%mm7 \n\t" \
397 "movq %%mm0, %%mm2 \n\t" \
398 "pmaddwd 104(%2), %%mm3 \n\t" \
399 "paddd %%mm7, %%mm4 \n\t" \
400 "paddd %%mm4, %%mm2 \n\t" \
401 "psubd %%mm4, %%mm0 \n\t" \
402 "psrad $" #shift ", %%mm2 \n\t"\
403 "psrad $" #shift ", %%mm0 \n\t"\
404 "movq %%mm6, %%mm4 \n\t" \
405 "paddd %%mm1, %%mm3 \n\t" \
406 "paddd %%mm3, %%mm6 \n\t" \
407 "psubd %%mm3, %%mm4 \n\t" \
408 "psrad $" #shift ", %%mm6 \n\t"\
409 "packssdw %%mm6, %%mm2 \n\t" \
410 "movq %%mm2, 8+" #dst " \n\t"\
411 "psrad $" #shift ", %%mm4 \n\t"\
412 "packssdw %%mm0, %%mm4 \n\t" \
413 "movq %%mm4, 16+" #dst " \n\t"\
416 "pslld $16, %%mm0 \n\t"\
417 "paddd "MANGLE(d40000)", %%mm0 \n\t"\
418 "psrad $13, %%mm0 \n\t"\
419 "packssdw %%mm0, %%mm0 \n\t"\
420 "movq %%mm0, " #dst " \n\t"\
421 "movq %%mm0, 8+" #dst " \n\t"\
422 "movq %%mm0, 16+" #dst " \n\t"\
423 "movq %%mm0, 24+" #dst " \n\t"\
426 #define Z_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift, bt) \
427 "movq " #src0 ", %%mm0 \n\t" \
428 "movq " #src4 ", %%mm1 \n\t" \
429 "movq " #src1 ", %%mm2 \n\t" \
430 "movq " #src5 ", %%mm3 \n\t" \
431 "movq %%mm0, %%mm4 \n\t"\
432 "por %%mm1, %%mm4 \n\t"\
433 "por %%mm2, %%mm4 \n\t"\
434 "por %%mm3, %%mm4 \n\t"\
435 "packssdw %%mm4,%%mm4 \n\t"\
436 "movd %%mm4, %%eax \n\t"\
437 "orl %%eax, %%eax \n\t"\
439 "movq 16(%2), %%mm4 \n\t" \
440 "pmaddwd %%mm0, %%mm4 \n\t" \
441 "movq 24(%2), %%mm5 \n\t" \
442 "pmaddwd %%mm5, %%mm0 \n\t" \
443 "movq 32(%2), %%mm5 \n\t" \
444 "pmaddwd %%mm1, %%mm5 \n\t" \
445 "movq 40(%2), %%mm6 \n\t" \
446 "pmaddwd %%mm6, %%mm1 \n\t" \
447 "movq 48(%2), %%mm7 \n\t" \
448 "pmaddwd %%mm2, %%mm7 \n\t" \
449 #rounder ", %%mm4 \n\t"\
450 "movq %%mm4, %%mm6 \n\t" \
451 "paddd %%mm5, %%mm4 \n\t" \
452 "psubd %%mm5, %%mm6 \n\t" \
453 "movq 56(%2), %%mm5 \n\t" \
454 "pmaddwd %%mm3, %%mm5 \n\t" \
455 #rounder ", %%mm0 \n\t"\
456 "paddd %%mm0, %%mm1 \n\t" \
457 "paddd %%mm0, %%mm0 \n\t" \
458 "psubd %%mm1, %%mm0 \n\t" \
459 "pmaddwd 64(%2), %%mm2 \n\t" \
460 "paddd %%mm5, %%mm7 \n\t" \
461 "movq 72(%2), %%mm5 \n\t" \
462 "pmaddwd %%mm3, %%mm5 \n\t" \
463 "paddd %%mm4, %%mm7 \n\t" \
464 "paddd %%mm4, %%mm4 \n\t" \
465 "psubd %%mm7, %%mm4 \n\t" \
466 "paddd %%mm2, %%mm5 \n\t" \
467 "psrad $" #shift ", %%mm7 \n\t"\
468 "psrad $" #shift ", %%mm4 \n\t"\
469 "movq %%mm1, %%mm2 \n\t" \
470 "paddd %%mm5, %%mm1 \n\t" \
471 "psubd %%mm5, %%mm2 \n\t" \
472 "psrad $" #shift ", %%mm1 \n\t"\
473 "psrad $" #shift ", %%mm2 \n\t"\
474 "packssdw %%mm1, %%mm7 \n\t" \
475 "packssdw %%mm4, %%mm2 \n\t" \
476 "movq %%mm7, " #dst " \n\t"\
477 "movq " #src1 ", %%mm1 \n\t" \
478 "movq 80(%2), %%mm4 \n\t" \
479 "movq %%mm2, 24+" #dst " \n\t"\
480 "pmaddwd %%mm1, %%mm4 \n\t" \
481 "movq 88(%2), %%mm7 \n\t" \
482 "pmaddwd 96(%2), %%mm1 \n\t" \
483 "pmaddwd %%mm3, %%mm7 \n\t" \
484 "movq %%mm0, %%mm2 \n\t" \
485 "pmaddwd 104(%2), %%mm3 \n\t" \
486 "paddd %%mm7, %%mm4 \n\t" \
487 "paddd %%mm4, %%mm2 \n\t" \
488 "psubd %%mm4, %%mm0 \n\t" \
489 "psrad $" #shift ", %%mm2 \n\t"\
490 "psrad $" #shift ", %%mm0 \n\t"\
491 "movq %%mm6, %%mm4 \n\t" \
492 "paddd %%mm1, %%mm3 \n\t" \
493 "paddd %%mm3, %%mm6 \n\t" \
494 "psubd %%mm3, %%mm4 \n\t" \
495 "psrad $" #shift ", %%mm6 \n\t"\
496 "packssdw %%mm6, %%mm2 \n\t" \
497 "movq %%mm2, 8+" #dst " \n\t"\
498 "psrad $" #shift ", %%mm4 \n\t"\
499 "packssdw %%mm0, %%mm4 \n\t" \
500 "movq %%mm4, 16+" #dst " \n\t"\
502 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
503 "movq " #src0 ", %%mm0 \n\t" \
504 "movq " #src4 ", %%mm1 \n\t" \
505 "movq " #src1 ", %%mm2 \n\t" \
506 "movq " #src5 ", %%mm3 \n\t" \
507 "movq 16(%2), %%mm4 \n\t" \
508 "pmaddwd %%mm0, %%mm4 \n\t" \
509 "movq 24(%2), %%mm5 \n\t" \
510 "pmaddwd %%mm5, %%mm0 \n\t" \
511 "movq 32(%2), %%mm5 \n\t" \
512 "pmaddwd %%mm1, %%mm5 \n\t" \
513 "movq 40(%2), %%mm6 \n\t" \
514 "pmaddwd %%mm6, %%mm1 \n\t" \
515 "movq 48(%2), %%mm7 \n\t" \
516 "pmaddwd %%mm2, %%mm7 \n\t" \
517 #rounder ", %%mm4 \n\t"\
518 "movq %%mm4, %%mm6 \n\t" \
519 "paddd %%mm5, %%mm4 \n\t" \
520 "psubd %%mm5, %%mm6 \n\t" \
521 "movq 56(%2), %%mm5 \n\t" \
522 "pmaddwd %%mm3, %%mm5 \n\t" \
523 #rounder ", %%mm0 \n\t"\
524 "paddd %%mm0, %%mm1 \n\t" \
525 "paddd %%mm0, %%mm0 \n\t" \
526 "psubd %%mm1, %%mm0 \n\t" \
527 "pmaddwd 64(%2), %%mm2 \n\t" \
528 "paddd %%mm5, %%mm7 \n\t" \
529 "movq 72(%2), %%mm5 \n\t" \
530 "pmaddwd %%mm3, %%mm5 \n\t" \
531 "paddd %%mm4, %%mm7 \n\t" \
532 "paddd %%mm4, %%mm4 \n\t" \
533 "psubd %%mm7, %%mm4 \n\t" \
534 "paddd %%mm2, %%mm5 \n\t" \
535 "psrad $" #shift ", %%mm7 \n\t"\
536 "psrad $" #shift ", %%mm4 \n\t"\
537 "movq %%mm1, %%mm2 \n\t" \
538 "paddd %%mm5, %%mm1 \n\t" \
539 "psubd %%mm5, %%mm2 \n\t" \
540 "psrad $" #shift ", %%mm1 \n\t"\
541 "psrad $" #shift ", %%mm2 \n\t"\
542 "packssdw %%mm1, %%mm7 \n\t" \
543 "packssdw %%mm4, %%mm2 \n\t" \
544 "movq %%mm7, " #dst " \n\t"\
545 "movq " #src1 ", %%mm1 \n\t" \
546 "movq 80(%2), %%mm4 \n\t" \
547 "movq %%mm2, 24+" #dst " \n\t"\
548 "pmaddwd %%mm1, %%mm4 \n\t" \
549 "movq 88(%2), %%mm7 \n\t" \
550 "pmaddwd 96(%2), %%mm1 \n\t" \
551 "pmaddwd %%mm3, %%mm7 \n\t" \
552 "movq %%mm0, %%mm2 \n\t" \
553 "pmaddwd 104(%2), %%mm3 \n\t" \
554 "paddd %%mm7, %%mm4 \n\t" \
555 "paddd %%mm4, %%mm2 \n\t" \
556 "psubd %%mm4, %%mm0 \n\t" \
557 "psrad $" #shift ", %%mm2 \n\t"\
558 "psrad $" #shift ", %%mm0 \n\t"\
559 "movq %%mm6, %%mm4 \n\t" \
560 "paddd %%mm1, %%mm3 \n\t" \
561 "paddd %%mm3, %%mm6 \n\t" \
562 "psubd %%mm3, %%mm4 \n\t" \
563 "psrad $" #shift ", %%mm6 \n\t"\
564 "packssdw %%mm6, %%mm2 \n\t" \
565 "movq %%mm2, 8+" #dst " \n\t"\
566 "psrad $" #shift ", %%mm4 \n\t"\
567 "packssdw %%mm0, %%mm4 \n\t" \
568 "movq %%mm4, 16+" #dst " \n\t"\
571 DC_COND_IDCT( 0(%0), 8(%0), 16(%0), 24(%0), 0(%1),paddd 8(%2), 11)
572 Z_COND_IDCT( 32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11, 4
f)
573 Z_COND_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 2
f)
574 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 1
f)
577 #define IDCT(src0, src4, src1, src5, dst, shift) \
578 "movq " #src0 ", %%mm0 \n\t" \
579 "movq " #src4 ", %%mm1 \n\t" \
580 "movq " #src1 ", %%mm2 \n\t" \
581 "movq " #src5 ", %%mm3 \n\t" \
582 "movq 16(%2), %%mm4 \n\t" \
583 "pmaddwd %%mm0, %%mm4 \n\t" \
584 "movq 24(%2), %%mm5 \n\t" \
585 "pmaddwd %%mm5, %%mm0 \n\t" \
586 "movq 32(%2), %%mm5 \n\t" \
587 "pmaddwd %%mm1, %%mm5 \n\t" \
588 "movq 40(%2), %%mm6 \n\t" \
589 "pmaddwd %%mm6, %%mm1 \n\t" \
590 "movq %%mm4, %%mm6 \n\t" \
591 "movq 48(%2), %%mm7 \n\t" \
592 "pmaddwd %%mm2, %%mm7 \n\t" \
593 "paddd %%mm5, %%mm4 \n\t" \
594 "psubd %%mm5, %%mm6 \n\t" \
595 "movq %%mm0, %%mm5 \n\t" \
596 "paddd %%mm1, %%mm0 \n\t" \
597 "psubd %%mm1, %%mm5 \n\t" \
598 "movq 56(%2), %%mm1 \n\t" \
599 "pmaddwd %%mm3, %%mm1 \n\t" \
600 "pmaddwd 64(%2), %%mm2 \n\t" \
601 "paddd %%mm1, %%mm7 \n\t" \
602 "movq 72(%2), %%mm1 \n\t" \
603 "pmaddwd %%mm3, %%mm1 \n\t" \
604 "paddd %%mm4, %%mm7 \n\t" \
605 "paddd %%mm4, %%mm4 \n\t" \
606 "psubd %%mm7, %%mm4 \n\t" \
607 "paddd %%mm2, %%mm1 \n\t" \
608 "psrad $" #shift ", %%mm7 \n\t"\
609 "psrad $" #shift ", %%mm4 \n\t"\
610 "movq %%mm0, %%mm2 \n\t" \
611 "paddd %%mm1, %%mm0 \n\t" \
612 "psubd %%mm1, %%mm2 \n\t" \
613 "psrad $" #shift ", %%mm0 \n\t"\
614 "psrad $" #shift ", %%mm2 \n\t"\
615 "packssdw %%mm7, %%mm7 \n\t" \
616 "movd %%mm7, " #dst " \n\t"\
617 "packssdw %%mm0, %%mm0 \n\t" \
618 "movd %%mm0, 16+" #dst " \n\t"\
619 "packssdw %%mm2, %%mm2 \n\t" \
620 "movd %%mm2, 96+" #dst " \n\t"\
621 "packssdw %%mm4, %%mm4 \n\t" \
622 "movd %%mm4, 112+" #dst " \n\t"\
623 "movq " #src1 ", %%mm0 \n\t" \
624 "movq 80(%2), %%mm4 \n\t" \
625 "pmaddwd %%mm0, %%mm4 \n\t" \
626 "movq 88(%2), %%mm7 \n\t" \
627 "pmaddwd 96(%2), %%mm0 \n\t" \
628 "pmaddwd %%mm3, %%mm7 \n\t" \
629 "movq %%mm5, %%mm2 \n\t" \
630 "pmaddwd 104(%2), %%mm3 \n\t" \
631 "paddd %%mm7, %%mm4 \n\t" \
632 "paddd %%mm4, %%mm2 \n\t" \
633 "psubd %%mm4, %%mm5 \n\t" \
634 "psrad $" #shift ", %%mm2 \n\t"\
635 "psrad $" #shift ", %%mm5 \n\t"\
636 "movq %%mm6, %%mm4 \n\t" \
637 "paddd %%mm0, %%mm3 \n\t" \
638 "paddd %%mm3, %%mm6 \n\t" \
639 "psubd %%mm3, %%mm4 \n\t" \
640 "psrad $" #shift ", %%mm6 \n\t"\
641 "psrad $" #shift ", %%mm4 \n\t"\
642 "packssdw %%mm2, %%mm2 \n\t" \
643 "packssdw %%mm6, %%mm6 \n\t" \
644 "movd %%mm2, 32+" #dst " \n\t"\
645 "packssdw %%mm4, %%mm4 \n\t" \
646 "packssdw %%mm5, %%mm5 \n\t" \
647 "movd %%mm6, 48+" #dst " \n\t"\
648 "movd %%mm4, 64+" #dst " \n\t"\
649 "movd %%mm5, 80+" #dst " \n\t"
653 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
654 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
655 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
656 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
661 Z_COND_IDCT( 64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 6
f)
662 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 5
f)
665 #define IDCT(src0, src4, src1, src5, dst, shift) \
666 "movq " #src0 ", %%mm0 \n\t" \
667 "movq " #src4 ", %%mm1 \n\t" \
668 "movq " #src5 ", %%mm3 \n\t" \
669 "movq 16(%2), %%mm4 \n\t" \
670 "pmaddwd %%mm0, %%mm4 \n\t" \
671 "movq 24(%2), %%mm5 \n\t" \
672 "pmaddwd %%mm5, %%mm0 \n\t" \
673 "movq 32(%2), %%mm5 \n\t" \
674 "pmaddwd %%mm1, %%mm5 \n\t" \
675 "movq 40(%2), %%mm6 \n\t" \
676 "pmaddwd %%mm6, %%mm1 \n\t" \
677 "movq %%mm4, %%mm6 \n\t" \
678 "paddd %%mm5, %%mm4 \n\t" \
679 "psubd %%mm5, %%mm6 \n\t" \
680 "movq %%mm0, %%mm5 \n\t" \
681 "paddd %%mm1, %%mm0 \n\t" \
682 "psubd %%mm1, %%mm5 \n\t" \
683 "movq 56(%2), %%mm1 \n\t" \
684 "pmaddwd %%mm3, %%mm1 \n\t" \
685 "movq 72(%2), %%mm7 \n\t" \
686 "pmaddwd %%mm3, %%mm7 \n\t" \
687 "paddd %%mm4, %%mm1 \n\t" \
688 "paddd %%mm4, %%mm4 \n\t" \
689 "psubd %%mm1, %%mm4 \n\t" \
690 "psrad $" #shift ", %%mm1 \n\t"\
691 "psrad $" #shift ", %%mm4 \n\t"\
692 "movq %%mm0, %%mm2 \n\t" \
693 "paddd %%mm7, %%mm0 \n\t" \
694 "psubd %%mm7, %%mm2 \n\t" \
695 "psrad $" #shift ", %%mm0 \n\t"\
696 "psrad $" #shift ", %%mm2 \n\t"\
697 "packssdw %%mm1, %%mm1 \n\t" \
698 "movd %%mm1, " #dst " \n\t"\
699 "packssdw %%mm0, %%mm0 \n\t" \
700 "movd %%mm0, 16+" #dst " \n\t"\
701 "packssdw %%mm2, %%mm2 \n\t" \
702 "movd %%mm2, 96+" #dst " \n\t"\
703 "packssdw %%mm4, %%mm4 \n\t" \
704 "movd %%mm4, 112+" #dst " \n\t"\
705 "movq 88(%2), %%mm1 \n\t" \
706 "pmaddwd %%mm3, %%mm1 \n\t" \
707 "movq %%mm5, %%mm2 \n\t" \
708 "pmaddwd 104(%2), %%mm3 \n\t" \
709 "paddd %%mm1, %%mm2 \n\t" \
710 "psubd %%mm1, %%mm5 \n\t" \
711 "psrad $" #shift ", %%mm2 \n\t"\
712 "psrad $" #shift ", %%mm5 \n\t"\
713 "movq %%mm6, %%mm1 \n\t" \
714 "paddd %%mm3, %%mm6 \n\t" \
715 "psubd %%mm3, %%mm1 \n\t" \
716 "psrad $" #shift ", %%mm6 \n\t"\
717 "psrad $" #shift ", %%mm1 \n\t"\
718 "packssdw %%mm2, %%mm2 \n\t" \
719 "packssdw %%mm6, %%mm6 \n\t" \
720 "movd %%mm2, 32+" #dst " \n\t"\
721 "packssdw %%mm1, %%mm1 \n\t" \
722 "packssdw %%mm5, %%mm5 \n\t" \
723 "movd %%mm6, 48+" #dst " \n\t"\
724 "movd %%mm1, 64+" #dst " \n\t"\
725 "movd %%mm5, 80+" #dst " \n\t"
728 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
729 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
730 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
731 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
736 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 7
f)
739 #define IDCT(src0, src4, src1, src5, dst, shift) \
740 "movq " #src0 ", %%mm0 \n\t" \
741 "movq " #src5 ", %%mm3 \n\t" \
742 "movq 16(%2), %%mm4 \n\t" \
743 "pmaddwd %%mm0, %%mm4 \n\t" \
744 "movq 24(%2), %%mm5 \n\t" \
745 "pmaddwd %%mm5, %%mm0 \n\t" \
746 "movq %%mm4, %%mm6 \n\t" \
747 "movq %%mm0, %%mm5 \n\t" \
748 "movq 56(%2), %%mm1 \n\t" \
749 "pmaddwd %%mm3, %%mm1 \n\t" \
750 "movq 72(%2), %%mm7 \n\t" \
751 "pmaddwd %%mm3, %%mm7 \n\t" \
752 "paddd %%mm4, %%mm1 \n\t" \
753 "paddd %%mm4, %%mm4 \n\t" \
754 "psubd %%mm1, %%mm4 \n\t" \
755 "psrad $" #shift ", %%mm1 \n\t"\
756 "psrad $" #shift ", %%mm4 \n\t"\
757 "movq %%mm0, %%mm2 \n\t" \
758 "paddd %%mm7, %%mm0 \n\t" \
759 "psubd %%mm7, %%mm2 \n\t" \
760 "psrad $" #shift ", %%mm0 \n\t"\
761 "psrad $" #shift ", %%mm2 \n\t"\
762 "packssdw %%mm1, %%mm1 \n\t" \
763 "movd %%mm1, " #dst " \n\t"\
764 "packssdw %%mm0, %%mm0 \n\t" \
765 "movd %%mm0, 16+" #dst " \n\t"\
766 "packssdw %%mm2, %%mm2 \n\t" \
767 "movd %%mm2, 96+" #dst " \n\t"\
768 "packssdw %%mm4, %%mm4 \n\t" \
769 "movd %%mm4, 112+" #dst " \n\t"\
770 "movq 88(%2), %%mm1 \n\t" \
771 "pmaddwd %%mm3, %%mm1 \n\t" \
772 "movq %%mm5, %%mm2 \n\t" \
773 "pmaddwd 104(%2), %%mm3 \n\t" \
774 "paddd %%mm1, %%mm2 \n\t" \
775 "psubd %%mm1, %%mm5 \n\t" \
776 "psrad $" #shift ", %%mm2 \n\t"\
777 "psrad $" #shift ", %%mm5 \n\t"\
778 "movq %%mm6, %%mm1 \n\t" \
779 "paddd %%mm3, %%mm6 \n\t" \
780 "psubd %%mm3, %%mm1 \n\t" \
781 "psrad $" #shift ", %%mm6 \n\t"\
782 "psrad $" #shift ", %%mm1 \n\t"\
783 "packssdw %%mm2, %%mm2 \n\t" \
784 "packssdw %%mm6, %%mm6 \n\t" \
785 "movd %%mm2, 32+" #dst " \n\t"\
786 "packssdw %%mm1, %%mm1 \n\t" \
787 "packssdw %%mm5, %%mm5 \n\t" \
788 "movd %%mm6, 48+" #dst " \n\t"\
789 "movd %%mm1, 64+" #dst " \n\t"\
790 "movd %%mm5, 80+" #dst " \n\t"
794 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
795 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
796 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
797 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
802 Z_COND_IDCT( 96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 3
f)
805 #define IDCT(src0, src4, src1, src5, dst, shift) \
806 "movq " #src0 ", %%mm0 \n\t" \
807 "movq " #src1 ", %%mm2 \n\t" \
808 "movq " #src5 ", %%mm3 \n\t" \
809 "movq 16(%2), %%mm4 \n\t" \
810 "pmaddwd %%mm0, %%mm4 \n\t" \
811 "movq 24(%2), %%mm5 \n\t" \
812 "pmaddwd %%mm5, %%mm0 \n\t" \
813 "movq %%mm4, %%mm6 \n\t" \
814 "movq 48(%2), %%mm7 \n\t" \
815 "pmaddwd %%mm2, %%mm7 \n\t" \
816 "movq %%mm0, %%mm5 \n\t" \
817 "movq 56(%2), %%mm1 \n\t" \
818 "pmaddwd %%mm3, %%mm1 \n\t" \
819 "pmaddwd 64(%2), %%mm2 \n\t" \
820 "paddd %%mm1, %%mm7 \n\t" \
821 "movq 72(%2), %%mm1 \n\t" \
822 "pmaddwd %%mm3, %%mm1 \n\t" \
823 "paddd %%mm4, %%mm7 \n\t" \
824 "paddd %%mm4, %%mm4 \n\t" \
825 "psubd %%mm7, %%mm4 \n\t" \
826 "paddd %%mm2, %%mm1 \n\t" \
827 "psrad $" #shift ", %%mm7 \n\t"\
828 "psrad $" #shift ", %%mm4 \n\t"\
829 "movq %%mm0, %%mm2 \n\t" \
830 "paddd %%mm1, %%mm0 \n\t" \
831 "psubd %%mm1, %%mm2 \n\t" \
832 "psrad $" #shift ", %%mm0 \n\t"\
833 "psrad $" #shift ", %%mm2 \n\t"\
834 "packssdw %%mm7, %%mm7 \n\t" \
835 "movd %%mm7, " #dst " \n\t"\
836 "packssdw %%mm0, %%mm0 \n\t" \
837 "movd %%mm0, 16+" #dst " \n\t"\
838 "packssdw %%mm2, %%mm2 \n\t" \
839 "movd %%mm2, 96+" #dst " \n\t"\
840 "packssdw %%mm4, %%mm4 \n\t" \
841 "movd %%mm4, 112+" #dst " \n\t"\
842 "movq " #src1 ", %%mm0 \n\t" \
843 "movq 80(%2), %%mm4 \n\t" \
844 "pmaddwd %%mm0, %%mm4 \n\t" \
845 "movq 88(%2), %%mm7 \n\t" \
846 "pmaddwd 96(%2), %%mm0 \n\t" \
847 "pmaddwd %%mm3, %%mm7 \n\t" \
848 "movq %%mm5, %%mm2 \n\t" \
849 "pmaddwd 104(%2), %%mm3 \n\t" \
850 "paddd %%mm7, %%mm4 \n\t" \
851 "paddd %%mm4, %%mm2 \n\t" \
852 "psubd %%mm4, %%mm5 \n\t" \
853 "psrad $" #shift ", %%mm2 \n\t"\
854 "psrad $" #shift ", %%mm5 \n\t"\
855 "movq %%mm6, %%mm4 \n\t" \
856 "paddd %%mm0, %%mm3 \n\t" \
857 "paddd %%mm3, %%mm6 \n\t" \
858 "psubd %%mm3, %%mm4 \n\t" \
859 "psrad $" #shift ", %%mm6 \n\t"\
860 "psrad $" #shift ", %%mm4 \n\t"\
861 "packssdw %%mm2, %%mm2 \n\t" \
862 "packssdw %%mm6, %%mm6 \n\t" \
863 "movd %%mm2, 32+" #dst " \n\t"\
864 "packssdw %%mm4, %%mm4 \n\t" \
865 "packssdw %%mm5, %%mm5 \n\t" \
866 "movd %%mm6, 48+" #dst " \n\t"\
867 "movd %%mm4, 64+" #dst " \n\t"\
868 "movd %%mm5, 80+" #dst " \n\t"
871 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
872 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
873 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
874 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
880 #define IDCT(src0, src4, src1, src5, dst, shift) \
881 "movq " #src0 ", %%mm0 \n\t" \
882 "movq " #src1 ", %%mm2 \n\t" \
883 "movq 16(%2), %%mm4 \n\t" \
884 "pmaddwd %%mm0, %%mm4 \n\t" \
885 "movq 24(%2), %%mm5 \n\t" \
886 "pmaddwd %%mm5, %%mm0 \n\t" \
887 "movq %%mm4, %%mm6 \n\t" \
888 "movq 48(%2), %%mm7 \n\t" \
889 "pmaddwd %%mm2, %%mm7 \n\t" \
890 "movq %%mm0, %%mm5 \n\t" \
891 "movq 64(%2), %%mm3 \n\t"\
892 "pmaddwd %%mm2, %%mm3 \n\t" \
893 "paddd %%mm4, %%mm7 \n\t" \
894 "paddd %%mm4, %%mm4 \n\t" \
895 "psubd %%mm7, %%mm4 \n\t" \
896 "psrad $" #shift ", %%mm7 \n\t"\
897 "psrad $" #shift ", %%mm4 \n\t"\
898 "movq %%mm0, %%mm1 \n\t" \
899 "paddd %%mm3, %%mm0 \n\t" \
900 "psubd %%mm3, %%mm1 \n\t" \
901 "psrad $" #shift ", %%mm0 \n\t"\
902 "psrad $" #shift ", %%mm1 \n\t"\
903 "packssdw %%mm7, %%mm7 \n\t" \
904 "movd %%mm7, " #dst " \n\t"\
905 "packssdw %%mm0, %%mm0 \n\t" \
906 "movd %%mm0, 16+" #dst " \n\t"\
907 "packssdw %%mm1, %%mm1 \n\t" \
908 "movd %%mm1, 96+" #dst " \n\t"\
909 "packssdw %%mm4, %%mm4 \n\t" \
910 "movd %%mm4, 112+" #dst " \n\t"\
911 "movq 80(%2), %%mm4 \n\t" \
912 "pmaddwd %%mm2, %%mm4 \n\t" \
913 "pmaddwd 96(%2), %%mm2 \n\t" \
914 "movq %%mm5, %%mm1 \n\t" \
915 "paddd %%mm4, %%mm1 \n\t" \
916 "psubd %%mm4, %%mm5 \n\t" \
917 "psrad $" #shift ", %%mm1 \n\t"\
918 "psrad $" #shift ", %%mm5 \n\t"\
919 "movq %%mm6, %%mm4 \n\t" \
920 "paddd %%mm2, %%mm6 \n\t" \
921 "psubd %%mm2, %%mm4 \n\t" \
922 "psrad $" #shift ", %%mm6 \n\t"\
923 "psrad $" #shift ", %%mm4 \n\t"\
924 "packssdw %%mm1, %%mm1 \n\t" \
925 "packssdw %%mm6, %%mm6 \n\t" \
926 "movd %%mm1, 32+" #dst " \n\t"\
927 "packssdw %%mm4, %%mm4 \n\t" \
928 "packssdw %%mm5, %%mm5 \n\t" \
929 "movd %%mm6, 48+" #dst " \n\t"\
930 "movd %%mm4, 64+" #dst " \n\t"\
931 "movd %%mm5, 80+" #dst " \n\t"
935 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
936 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
937 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
938 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
944 #define IDCT(src0, src4, src1, src5, dst, shift) \
945 "movq " #src0 ", %%mm0 \n\t" \
946 "movq " #src4 ", %%mm1 \n\t" \
947 "movq 16(%2), %%mm4 \n\t" \
948 "pmaddwd %%mm0, %%mm4 \n\t" \
949 "movq 24(%2), %%mm5 \n\t" \
950 "pmaddwd %%mm5, %%mm0 \n\t" \
951 "movq 32(%2), %%mm5 \n\t" \
952 "pmaddwd %%mm1, %%mm5 \n\t" \
953 "movq 40(%2), %%mm6 \n\t" \
954 "pmaddwd %%mm6, %%mm1 \n\t" \
955 "movq %%mm4, %%mm6 \n\t" \
956 "paddd %%mm5, %%mm4 \n\t" \
957 "psubd %%mm5, %%mm6 \n\t" \
958 "movq %%mm0, %%mm5 \n\t" \
959 "paddd %%mm1, %%mm0 \n\t" \
960 "psubd %%mm1, %%mm5 \n\t" \
961 "movq 8+" #src0 ", %%mm2 \n\t" \
962 "movq 8+" #src4 ", %%mm3 \n\t" \
963 "movq 16(%2), %%mm1 \n\t" \
964 "pmaddwd %%mm2, %%mm1 \n\t" \
965 "movq 24(%2), %%mm7 \n\t" \
966 "pmaddwd %%mm7, %%mm2 \n\t" \
967 "movq 32(%2), %%mm7 \n\t" \
968 "pmaddwd %%mm3, %%mm7 \n\t" \
969 "pmaddwd 40(%2), %%mm3 \n\t" \
970 "paddd %%mm1, %%mm7 \n\t" \
971 "paddd %%mm1, %%mm1 \n\t" \
972 "psubd %%mm7, %%mm1 \n\t" \
973 "paddd %%mm2, %%mm3 \n\t" \
974 "paddd %%mm2, %%mm2 \n\t" \
975 "psubd %%mm3, %%mm2 \n\t" \
976 "psrad $" #shift ", %%mm4 \n\t"\
977 "psrad $" #shift ", %%mm7 \n\t"\
978 "psrad $" #shift ", %%mm3 \n\t"\
979 "packssdw %%mm7, %%mm4 \n\t" \
980 "movq %%mm4, " #dst " \n\t"\
981 "psrad $" #shift ", %%mm0 \n\t"\
982 "packssdw %%mm3, %%mm0 \n\t" \
983 "movq %%mm0, 16+" #dst " \n\t"\
984 "movq %%mm0, 96+" #dst " \n\t"\
985 "movq %%mm4, 112+" #dst " \n\t"\
986 "psrad $" #shift ", %%mm5 \n\t"\
987 "psrad $" #shift ", %%mm6 \n\t"\
988 "psrad $" #shift ", %%mm2 \n\t"\
989 "packssdw %%mm2, %%mm5 \n\t" \
990 "movq %%mm5, 32+" #dst " \n\t"\
991 "psrad $" #shift ", %%mm1 \n\t"\
992 "packssdw %%mm1, %%mm6 \n\t" \
993 "movq %%mm6, 48+" #dst " \n\t"\
994 "movq %%mm6, 64+" #dst " \n\t"\
995 "movq %%mm5, 80+" #dst " \n\t"
999 IDCT( 0(%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
1001 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
1006 "# .p2align 4 \n\t"\
1009 #define IDCT(src0, src4, src1, src5, dst, shift) \
1010 "movq " #src0 ", %%mm0 \n\t" \
1011 "movq " #src4 ", %%mm1 \n\t" \
1012 "movq " #src1 ", %%mm2 \n\t" \
1013 "movq 16(%2), %%mm4 \n\t" \
1014 "pmaddwd %%mm0, %%mm4 \n\t" \
1015 "movq 24(%2), %%mm5 \n\t" \
1016 "pmaddwd %%mm5, %%mm0 \n\t" \
1017 "movq 32(%2), %%mm5 \n\t" \
1018 "pmaddwd %%mm1, %%mm5 \n\t" \
1019 "movq 40(%2), %%mm6 \n\t" \
1020 "pmaddwd %%mm6, %%mm1 \n\t" \
1021 "movq %%mm4, %%mm6 \n\t" \
1022 "movq 48(%2), %%mm7 \n\t" \
1023 "pmaddwd %%mm2, %%mm7 \n\t" \
1024 "paddd %%mm5, %%mm4 \n\t" \
1025 "psubd %%mm5, %%mm6 \n\t" \
1026 "movq %%mm0, %%mm5 \n\t" \
1027 "paddd %%mm1, %%mm0 \n\t" \
1028 "psubd %%mm1, %%mm5 \n\t" \
1029 "movq 64(%2), %%mm1 \n\t"\
1030 "pmaddwd %%mm2, %%mm1 \n\t" \
1031 "paddd %%mm4, %%mm7 \n\t" \
1032 "paddd %%mm4, %%mm4 \n\t" \
1033 "psubd %%mm7, %%mm4 \n\t" \
1034 "psrad $" #shift ", %%mm7 \n\t"\
1035 "psrad $" #shift ", %%mm4 \n\t"\
1036 "movq %%mm0, %%mm3 \n\t" \
1037 "paddd %%mm1, %%mm0 \n\t" \
1038 "psubd %%mm1, %%mm3 \n\t" \
1039 "psrad $" #shift ", %%mm0 \n\t"\
1040 "psrad $" #shift ", %%mm3 \n\t"\
1041 "packssdw %%mm7, %%mm7 \n\t" \
1042 "movd %%mm7, " #dst " \n\t"\
1043 "packssdw %%mm0, %%mm0 \n\t" \
1044 "movd %%mm0, 16+" #dst " \n\t"\
1045 "packssdw %%mm3, %%mm3 \n\t" \
1046 "movd %%mm3, 96+" #dst " \n\t"\
1047 "packssdw %%mm4, %%mm4 \n\t" \
1048 "movd %%mm4, 112+" #dst " \n\t"\
1049 "movq 80(%2), %%mm4 \n\t" \
1050 "pmaddwd %%mm2, %%mm4 \n\t" \
1051 "pmaddwd 96(%2), %%mm2 \n\t" \
1052 "movq %%mm5, %%mm3 \n\t" \
1053 "paddd %%mm4, %%mm3 \n\t" \
1054 "psubd %%mm4, %%mm5 \n\t" \
1055 "psrad $" #shift ", %%mm3 \n\t"\
1056 "psrad $" #shift ", %%mm5 \n\t"\
1057 "movq %%mm6, %%mm4 \n\t" \
1058 "paddd %%mm2, %%mm6 \n\t" \
1059 "psubd %%mm2, %%mm4 \n\t" \
1060 "psrad $" #shift ", %%mm6 \n\t"\
1061 "packssdw %%mm3, %%mm3 \n\t" \
1062 "movd %%mm3, 32+" #dst " \n\t"\
1063 "psrad $" #shift ", %%mm4 \n\t"\
1064 "packssdw %%mm6, %%mm6 \n\t" \
1065 "movd %%mm6, 48+" #dst " \n\t"\
1066 "packssdw %%mm4, %%mm4 \n\t" \
1067 "packssdw %%mm5, %%mm5 \n\t" \
1068 "movd %%mm4, 64+" #dst " \n\t"\
1069 "movd %%mm5, 80+" #dst " \n\t"
1073 IDCT( (%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
1074 IDCT( 8(%1), 72(%1), 40(%1), 104(%1), 4(%0), 20)
1075 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)
1076 IDCT( 24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
1083 #define IDCT(src0, src4, src1, src5, dst, shift) \
1084 "movq " #src0 ", %%mm0 \n\t" \
1085 "movq 16(%2), %%mm4 \n\t" \
1086 "pmaddwd %%mm0, %%mm4 \n\t" \
1087 "movq 24(%2), %%mm5 \n\t" \
1088 "pmaddwd %%mm5, %%mm0 \n\t" \
1089 "psrad $" #shift ", %%mm4 \n\t"\
1090 "psrad $" #shift ", %%mm0 \n\t"\
1091 "movq 8+" #src0 ", %%mm2 \n\t" \
1092 "movq 16(%2), %%mm1 \n\t" \
1093 "pmaddwd %%mm2, %%mm1 \n\t" \
1094 "movq 24(%2), %%mm7 \n\t" \
1095 "pmaddwd %%mm7, %%mm2 \n\t" \
1096 "movq 32(%2), %%mm7 \n\t" \
1097 "psrad $" #shift ", %%mm1 \n\t"\
1098 "packssdw %%mm1, %%mm4 \n\t" \
1099 "movq %%mm4, " #dst " \n\t"\
1100 "psrad $" #shift ", %%mm2 \n\t"\
1101 "packssdw %%mm2, %%mm0 \n\t" \
1102 "movq %%mm0, 16+" #dst " \n\t"\
1103 "movq %%mm0, 96+" #dst " \n\t"\
1104 "movq %%mm4, 112+" #dst " \n\t"\
1105 "movq %%mm0, 32+" #dst " \n\t"\
1106 "movq %%mm4, 48+" #dst " \n\t"\
1107 "movq %%mm4, 64+" #dst " \n\t"\
1108 "movq %%mm0, 80+" #dst " \n\t"
1111 IDCT( 0(%1), 64(%1), 32(%1), 96(%1), 0(%0), 20)
1113 IDCT( 16(%1), 80(%1), 48(%1), 112(%1), 8(%0), 20)