1 #include "mod_skeletal_animatevertices_sse.h"
5 #ifdef MATRIX4x4_OPENGLORIENTATION
6 #error "SSE skeletal requires D3D matrix layout"
11 void Mod_Skeletal_AnimateVertices_SSE(const dp_model_t * RESTRICT model, const frameblend_t * RESTRICT frameblend, const skeleton_t *skeleton, float * RESTRICT vertex3f, float * RESTRICT normal3f, float * RESTRICT svector3f, float * RESTRICT tvector3f)
13 // vertex weighted skeletal
16 matrix4x4_t *bonepose;
17 matrix4x4_t *boneposerelative;
19 const blendweights_t * RESTRICT weights;
20 int num_vertices_minus_one;
22 num_vertices_minus_one = model->surfmesh.num_vertices - 1;
24 //unsigned long long ts = rdtsc();
25 bonepose = (matrix4x4_t *) Mod_Skeletal_AnimateVertices_AllocBuffers(sizeof(matrix4x4_t) * (model->num_bones*2 + model->surfmesh.num_blends));
26 boneposerelative = bonepose + model->num_bones;
28 if (skeleton && !skeleton->relativetransforms)
31 // interpolate matrices
34 for (i = 0;i < model->num_bones;i++)
36 const float * RESTRICT n = model->data_baseboneposeinverse + i * 12;
37 matrix4x4_t * RESTRICT s = &skeleton->relativetransforms[i];
38 matrix4x4_t * RESTRICT b = &bonepose[i];
39 matrix4x4_t * RESTRICT r = &boneposerelative[i];
40 __m128 b0, b1, b2, b3, r0, r1, r2, r3, nr;
41 if (model->data_bones[i].parent >= 0)
43 const matrix4x4_t * RESTRICT p = &bonepose[model->data_bones[i].parent];
44 __m128 s0 = _mm_loadu_ps(s->m[0]), s1 = _mm_loadu_ps(s->m[1]), s2 = _mm_loadu_ps(s->m[2]);
45 #ifdef OPENGLORIENTATION
46 __m128 s3 = _mm_loadu_ps(s->m[3]);
47 #define SKELETON_MATRIX(r, c) _mm_shuffle_ps(s##c, s##c, _MM_SHUFFLE(r, r, r, r))
49 #define SKELETON_MATRIX(r, c) _mm_shuffle_ps(s##r, s##r, _MM_SHUFFLE(c, c, c, c))
51 __m128 pr = _mm_load_ps(p->m[0]);
52 b0 = _mm_mul_ps(pr, SKELETON_MATRIX(0, 0));
53 b1 = _mm_mul_ps(pr, SKELETON_MATRIX(0, 1));
54 b2 = _mm_mul_ps(pr, SKELETON_MATRIX(0, 2));
55 b3 = _mm_mul_ps(pr, SKELETON_MATRIX(0, 3));
56 pr = _mm_load_ps(p->m[1]);
57 b0 = _mm_add_ps(b0, _mm_mul_ps(pr, SKELETON_MATRIX(1, 0)));
58 b1 = _mm_add_ps(b1, _mm_mul_ps(pr, SKELETON_MATRIX(1, 1)));
59 b2 = _mm_add_ps(b2, _mm_mul_ps(pr, SKELETON_MATRIX(1, 2)));
60 b3 = _mm_add_ps(b3, _mm_mul_ps(pr, SKELETON_MATRIX(1, 3)));
61 pr = _mm_load_ps(p->m[2]);
62 b0 = _mm_add_ps(b0, _mm_mul_ps(pr, SKELETON_MATRIX(2, 0)));
63 b1 = _mm_add_ps(b1, _mm_mul_ps(pr, SKELETON_MATRIX(2, 1)));
64 b2 = _mm_add_ps(b2, _mm_mul_ps(pr, SKELETON_MATRIX(2, 2)));
65 b3 = _mm_add_ps(b3, _mm_mul_ps(pr, SKELETON_MATRIX(2, 3)));
66 b3 = _mm_add_ps(b3, _mm_load_ps(p->m[3]));
70 b0 = _mm_loadu_ps(s->m[0]);
71 b1 = _mm_loadu_ps(s->m[1]);
72 b2 = _mm_loadu_ps(s->m[2]);
73 b3 = _mm_loadu_ps(s->m[3]);
74 #ifndef OPENGLORIENTATION
75 _MM_TRANSPOSE4_PS(b0, b1, b2, b3);
78 _mm_store_ps(b->m[0], b0);
79 _mm_store_ps(b->m[1], b1);
80 _mm_store_ps(b->m[2], b2);
81 _mm_store_ps(b->m[3], b3);
83 r0 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0)));
84 r1 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1)));
85 r2 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2)));
86 r3 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3)));
87 nr = _mm_loadu_ps(n+4);
88 r0 = _mm_add_ps(r0, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0))));
89 r1 = _mm_add_ps(r1, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1))));
90 r2 = _mm_add_ps(r2, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2))));
91 r3 = _mm_add_ps(r3, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3))));
92 nr = _mm_loadu_ps(n+8);
93 r0 = _mm_add_ps(r0, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0))));
94 r1 = _mm_add_ps(r1, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1))));
95 r2 = _mm_add_ps(r2, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2))));
96 r3 = _mm_add_ps(r3, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3))));
97 r3 = _mm_add_ps(r3, b3);
98 _mm_store_ps(r->m[0], r0);
99 _mm_store_ps(r->m[1], r1);
100 _mm_store_ps(r->m[2], r2);
101 _mm_store_ps(r->m[3], r3);
106 for (i = 0;i < model->num_bones;i++)
108 const short * RESTRICT pose7s = model->data_poses7s + 7 * (frameblend[0].subframe * model->num_bones + i);
109 float lerp = frameblend[0].lerp,
110 tx = pose7s[0], ty = pose7s[1], tz = pose7s[2],
111 rx = pose7s[3] * lerp,
112 ry = pose7s[4] * lerp,
113 rz = pose7s[5] * lerp,
114 rw = pose7s[6] * lerp,
115 dx = tx*rw + ty*rz - tz*ry,
116 dy = -tx*rz + ty*rw + tz*rx,
117 dz = tx*ry - ty*rx + tz*rw,
118 dw = -tx*rx - ty*ry - tz*rz,
120 for (blends = 1;blends < MAX_FRAMEBLENDS && frameblend[blends].lerp > 0;blends++)
122 const short * RESTRICT pose7s = model->data_poses7s + 7 * (frameblend[blends].subframe * model->num_bones + i);
123 float lerp = frameblend[blends].lerp,
124 tx = pose7s[0], ty = pose7s[1], tz = pose7s[2],
125 qx = pose7s[3], qy = pose7s[4], qz = pose7s[5], qw = pose7s[6];
126 if(rx*qx + ry*qy + rz*qz + rw*qw < 0) lerp = -lerp;
135 dx += tx*qw + ty*qz - tz*qy;
136 dy += -tx*qz + ty*qw + tz*qx;
137 dz += tx*qy - ty*qx + tz*qw;
138 dw += -tx*qx - ty*qy - tz*qz;
140 scale = 1.0f / (rx*rx + ry*ry + rz*rz + rw*rw);
141 originscale = -model->num_posescale * scale;
142 m[0] = scale*(rw*rw + rx*rx - ry*ry - rz*rz);
143 m[1] = 2*scale*(rx*ry - rw*rz);
144 m[2] = 2*scale*(rx*rz + rw*ry);
145 m[3] = originscale*(dw*rx - dx*rw + dy*rz - dz*ry);
146 m[4] = 2*scale*(rx*ry + rw*rz);
147 m[5] = scale*(rw*rw + ry*ry - rx*rx - rz*rz);
148 m[6] = 2*scale*(ry*rz - rw*rx);
149 m[7] = originscale*(dw*ry - dx*rz - dy*rw + dz*rx);
150 m[8] = 2*scale*(rx*rz - rw*ry);
151 m[9] = 2*scale*(ry*rz + rw*rx);
152 m[10] = scale*(rw*rw + rz*rz - rx*rx - ry*ry);
153 m[11] = originscale*(dw*rz + dx*ry - dy*rx - dz*rw);
154 if (i == r_skeletal_debugbone.integer)
155 m[r_skeletal_debugbonecomponent.integer % 12] += r_skeletal_debugbonevalue.value;
156 m[3] *= r_skeletal_debugtranslatex.value;
157 m[7] *= r_skeletal_debugtranslatey.value;
158 m[11] *= r_skeletal_debugtranslatez.value;
160 const float * RESTRICT n = model->data_baseboneposeinverse + i * 12;
161 matrix4x4_t * RESTRICT b = &bonepose[i];
162 matrix4x4_t * RESTRICT r = &boneposerelative[i];
163 __m128 b0, b1, b2, b3, r0, r1, r2, r3, nr;
164 if (model->data_bones[i].parent >= 0)
166 const matrix4x4_t * RESTRICT p = &bonepose[model->data_bones[i].parent];
167 __m128 pr = _mm_load_ps(p->m[0]);
168 b0 = _mm_mul_ps(pr, _mm_set1_ps(m[0]));
169 b1 = _mm_mul_ps(pr, _mm_set1_ps(m[1]));
170 b2 = _mm_mul_ps(pr, _mm_set1_ps(m[2]));
171 b3 = _mm_mul_ps(pr, _mm_set1_ps(m[3]));
172 pr = _mm_load_ps(p->m[1]);
173 b0 = _mm_add_ps(b0, _mm_mul_ps(pr, _mm_set1_ps(m[4])));
174 b1 = _mm_add_ps(b1, _mm_mul_ps(pr, _mm_set1_ps(m[5])));
175 b2 = _mm_add_ps(b2, _mm_mul_ps(pr, _mm_set1_ps(m[6])));
176 b3 = _mm_add_ps(b3, _mm_mul_ps(pr, _mm_set1_ps(m[7])));
177 pr = _mm_load_ps(p->m[2]);
178 b0 = _mm_add_ps(b0, _mm_mul_ps(pr, _mm_set1_ps(m[8])));
179 b1 = _mm_add_ps(b1, _mm_mul_ps(pr, _mm_set1_ps(m[9])));
180 b2 = _mm_add_ps(b2, _mm_mul_ps(pr, _mm_set1_ps(m[10])));
181 b3 = _mm_add_ps(b3, _mm_mul_ps(pr, _mm_set1_ps(m[11])));
182 b3 = _mm_add_ps(b3, _mm_load_ps(p->m[3]));
186 b0 = _mm_setr_ps(m[0], m[4], m[8], 0.0f);
187 b1 = _mm_setr_ps(m[1], m[5], m[9], 0.0f);
188 b2 = _mm_setr_ps(m[2], m[6], m[10], 0.0f);
189 b3 = _mm_setr_ps(m[3], m[7], m[11], 1.0f);
191 _mm_store_ps(b->m[0], b0);
192 _mm_store_ps(b->m[1], b1);
193 _mm_store_ps(b->m[2], b2);
194 _mm_store_ps(b->m[3], b3);
195 nr = _mm_loadu_ps(n);
196 r0 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0)));
197 r1 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1)));
198 r2 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2)));
199 r3 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3)));
200 nr = _mm_loadu_ps(n+4);
201 r0 = _mm_add_ps(r0, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0))));
202 r1 = _mm_add_ps(r1, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1))));
203 r2 = _mm_add_ps(r2, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2))));
204 r3 = _mm_add_ps(r3, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3))));
205 nr = _mm_loadu_ps(n+8);
206 r0 = _mm_add_ps(r0, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0))));
207 r1 = _mm_add_ps(r1, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1))));
208 r2 = _mm_add_ps(r2, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2))));
209 r3 = _mm_add_ps(r3, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3))));
210 r3 = _mm_add_ps(r3, b3);
211 _mm_store_ps(r->m[0], r0);
212 _mm_store_ps(r->m[1], r1);
213 _mm_store_ps(r->m[2], r2);
214 _mm_store_ps(r->m[3], r3);
219 // generate matrices for all blend combinations
220 weights = model->surfmesh.data_blendweights;
221 for (i = 0;i < model->surfmesh.num_blends;i++, weights++)
223 float * RESTRICT b = &boneposerelative[model->num_bones + i].m[0][0];
224 const float * RESTRICT m = &boneposerelative[weights->index[0]].m[0][0];
225 float f = weights->influence[0] * (1.0f / 255.0f);
226 __m128 fv = _mm_set_ps1(f);
227 __m128 b0 = _mm_load_ps(m);
228 __m128 b1 = _mm_load_ps(m+4);
229 __m128 b2 = _mm_load_ps(m+8);
230 __m128 b3 = _mm_load_ps(m+12);
231 __m128 m0, m1, m2, m3;
232 b0 = _mm_mul_ps(b0, fv);
233 b1 = _mm_mul_ps(b1, fv);
234 b2 = _mm_mul_ps(b2, fv);
235 b3 = _mm_mul_ps(b3, fv);
236 for (k = 1;k < 4 && weights->influence[k];k++)
238 m = &boneposerelative[weights->index[k]].m[0][0];
239 f = weights->influence[k] * (1.0f / 255.0f);
242 m1 = _mm_load_ps(m+4);
243 m2 = _mm_load_ps(m+8);
244 m3 = _mm_load_ps(m+12);
245 m0 = _mm_mul_ps(m0, fv);
246 m1 = _mm_mul_ps(m1, fv);
247 m2 = _mm_mul_ps(m2, fv);
248 m3 = _mm_mul_ps(m3, fv);
249 b0 = _mm_add_ps(m0, b0);
250 b1 = _mm_add_ps(m1, b1);
251 b2 = _mm_add_ps(m2, b2);
252 b3 = _mm_add_ps(m3, b3);
255 _mm_store_ps(b+4, b1);
256 _mm_store_ps(b+8, b2);
257 _mm_store_ps(b+12, b3);
260 #define LOAD_MATRIX_SCALAR() const float * RESTRICT m = &boneposerelative[*b].m[0][0]
262 #define LOAD_MATRIX3() \
263 const float * RESTRICT m = &boneposerelative[*b].m[0][0]; \
264 /* bonepose array is 16 byte aligned */ \
265 __m128 m1 = _mm_load_ps((m)); \
266 __m128 m2 = _mm_load_ps((m)+4); \
267 __m128 m3 = _mm_load_ps((m)+8);
268 #define LOAD_MATRIX4() \
269 const float * RESTRICT m = &boneposerelative[*b].m[0][0]; \
270 /* bonepose array is 16 byte aligned */ \
271 __m128 m1 = _mm_load_ps((m)); \
272 __m128 m2 = _mm_load_ps((m)+4); \
273 __m128 m3 = _mm_load_ps((m)+8); \
274 __m128 m4 = _mm_load_ps((m)+12)
276 /* Note that matrix is 4x4 and transposed compared to non-USE_SSE codepath */
277 #define TRANSFORM_POSITION_SCALAR(in, out) \
278 (out)[0] = ((in)[0] * m[0] + (in)[1] * m[4] + (in)[2] * m[ 8] + m[12]); \
279 (out)[1] = ((in)[0] * m[1] + (in)[1] * m[5] + (in)[2] * m[ 9] + m[13]); \
280 (out)[2] = ((in)[0] * m[2] + (in)[1] * m[6] + (in)[2] * m[10] + m[14]);
281 #define TRANSFORM_VECTOR_SCALAR(in, out) \
282 (out)[0] = ((in)[0] * m[0] + (in)[1] * m[4] + (in)[2] * m[ 8]); \
283 (out)[1] = ((in)[0] * m[1] + (in)[1] * m[5] + (in)[2] * m[ 9]); \
284 (out)[2] = ((in)[0] * m[2] + (in)[1] * m[6] + (in)[2] * m[10]);
286 #define TRANSFORM_POSITION(in, out) { \
287 __m128 pin = _mm_loadu_ps(in); /* we ignore the value in the last element (x from the next vertex) */ \
288 __m128 x = _mm_shuffle_ps(pin, pin, 0x0); \
289 __m128 t1 = _mm_mul_ps(x, m1); \
292 __m128 y = _mm_shuffle_ps(pin, pin, 0x55); \
293 __m128 t2 = _mm_mul_ps(y, m2); \
294 __m128 t3 = _mm_add_ps(t1, t2); \
297 __m128 z = _mm_shuffle_ps(pin, pin, 0xaa); \
298 __m128 t4 = _mm_mul_ps(z, m3); \
299 __m128 t5 = _mm_add_ps(t3, t4); \
302 __m128 pout = _mm_add_ps(t5, m4); \
303 _mm_storeu_ps((out), pout); \
306 #define TRANSFORM_VECTOR(in, out) { \
307 __m128 vin = _mm_loadu_ps(in); \
310 __m128 x = _mm_shuffle_ps(vin, vin, 0x0); \
311 __m128 t1 = _mm_mul_ps(x, m1); \
314 __m128 y = _mm_shuffle_ps(vin, vin, 0x55); \
315 __m128 t2 = _mm_mul_ps(y, m2); \
316 __m128 t3 = _mm_add_ps(t1, t2); \
318 /* nz, + (ny + nx) */ \
319 __m128 z = _mm_shuffle_ps(vin, vin, 0xaa); \
320 __m128 t4 = _mm_mul_ps(z, m3); \
321 __m128 vout = _mm_add_ps(t3, t4); \
322 _mm_storeu_ps((out), vout); \
325 // transform vertex attributes by blended matrices
328 const float * RESTRICT v = model->surfmesh.data_vertex3f;
329 const unsigned short * RESTRICT b = model->surfmesh.blends;
330 // special case common combinations of attributes to avoid repeated loading of matrices
333 const float * RESTRICT n = model->surfmesh.data_normal3f;
334 if (svector3f && tvector3f)
336 const float * RESTRICT sv = model->surfmesh.data_svector3f;
337 const float * RESTRICT tv = model->surfmesh.data_tvector3f;
339 // Note that for SSE each iteration stores one element past end, so we break one vertex short
340 // and handle that with scalars in that case
341 for (i = 0; i < num_vertices_minus_one; i++, v += 3, n += 3, sv += 3, tv += 3, b++,
342 vertex3f += 3, normal3f += 3, svector3f += 3, tvector3f += 3)
345 TRANSFORM_POSITION(v, vertex3f);
346 TRANSFORM_VECTOR(n, normal3f);
347 TRANSFORM_VECTOR(sv, svector3f);
348 TRANSFORM_VECTOR(tv, tvector3f);
351 // Last vertex needs to be done with scalars to avoid reading/writing 1 word past end of arrays
353 LOAD_MATRIX_SCALAR();
354 TRANSFORM_POSITION_SCALAR(v, vertex3f);
355 TRANSFORM_VECTOR_SCALAR(n, normal3f);
356 TRANSFORM_VECTOR_SCALAR(sv, svector3f);
357 TRANSFORM_VECTOR_SCALAR(tv, tvector3f);
359 //printf("elapsed ticks: %llu\n", rdtsc() - ts); // XXX
363 for (i = 0;i < num_vertices_minus_one; i++, v += 3, n += 3, b++, vertex3f += 3, normal3f += 3)
366 TRANSFORM_POSITION(v, vertex3f);
367 TRANSFORM_VECTOR(n, normal3f);
370 LOAD_MATRIX_SCALAR();
371 TRANSFORM_POSITION_SCALAR(v, vertex3f);
372 TRANSFORM_VECTOR_SCALAR(n, normal3f);
377 for (i = 0;i < num_vertices_minus_one; i++, v += 3, b++, vertex3f += 3)
380 TRANSFORM_POSITION(v, vertex3f);
383 LOAD_MATRIX_SCALAR();
384 TRANSFORM_POSITION_SCALAR(v, vertex3f);
391 const float * RESTRICT n = model->surfmesh.data_normal3f;
392 const unsigned short * RESTRICT b = model->surfmesh.blends;
393 for (i = 0; i < num_vertices_minus_one; i++, n += 3, b++, normal3f += 3)
396 TRANSFORM_VECTOR(n, normal3f);
399 LOAD_MATRIX_SCALAR();
400 TRANSFORM_VECTOR_SCALAR(n, normal3f);
406 const float * RESTRICT sv = model->surfmesh.data_svector3f;
407 const unsigned short * RESTRICT b = model->surfmesh.blends;
408 for (i = 0; i < num_vertices_minus_one; i++, sv += 3, b++, svector3f += 3)
411 TRANSFORM_VECTOR(sv, svector3f);
414 LOAD_MATRIX_SCALAR();
415 TRANSFORM_VECTOR_SCALAR(sv, svector3f);
421 const float * RESTRICT tv = model->surfmesh.data_tvector3f;
422 const unsigned short * RESTRICT b = model->surfmesh.blends;
423 for (i = 0; i < num_vertices_minus_one; i++, tv += 3, b++, tvector3f += 3)
426 TRANSFORM_VECTOR(tv, tvector3f);
429 LOAD_MATRIX_SCALAR();
430 TRANSFORM_VECTOR_SCALAR(tv, tvector3f);
436 #undef TRANSFORM_POSITION
437 #undef TRANSFORM_VECTOR
438 #undef LOAD_MATRIX_SCALAR
439 #undef TRANSFORM_POSITION_SCALAR
440 #undef TRANSFORM_VECTOR_SCALAR