mod_skeletal_animatevertices_sse.c

   1 #include "mod_skeletal_animatevertices_sse.h"
   2
   3 #ifdef SSE_POSSIBLE
   4
   5 #ifdef MATRIX4x4_OPENGLORIENTATION
   6 #error "SSE skeletal requires D3D matrix layout"
   7 #endif
   8
   9 #include <xmmintrin.h>
  10
  11 void Mod_Skeletal_AnimateVertices_SSE(const dp_model_t * RESTRICT model, const frameblend_t * RESTRICT frameblend, const skeleton_t *skeleton, float * RESTRICT vertex3f, float * RESTRICT normal3f, float * RESTRICT svector3f, float * RESTRICT tvector3f)
  12 {
  13         // vertex weighted skeletal
  14         int i, k;
  15         int blends;
  16         matrix4x4_t *bonepose;
  17         matrix4x4_t *boneposerelative;
  18         float m[12];
  19         const blendweights_t * RESTRICT weights;
  20         int num_vertices_minus_one;
  21
  22         num_vertices_minus_one = model->surfmesh.num_vertices - 1;
  23
  24         //unsigned long long ts = rdtsc();
  25         bonepose = (matrix4x4_t *) Mod_Skeletal_AnimateVertices_AllocBuffers(sizeof(matrix4x4_t) * (model->num_bones*2 + model->surfmesh.num_blends));
  26         boneposerelative = bonepose + model->num_bones;
  27
  28         if (skeleton && !skeleton->relativetransforms)
  29                 skeleton = NULL;
  30
  31         // interpolate matrices
  32         if (skeleton)
  33         {
  34                 for (i = 0;i < model->num_bones;i++)
  35                 {
  36                         const float * RESTRICT n = model->data_baseboneposeinverse + i * 12;
  37                         matrix4x4_t * RESTRICT s = &skeleton->relativetransforms[i];
  38                         matrix4x4_t * RESTRICT b = &bonepose[i];
  39                         matrix4x4_t * RESTRICT r = &boneposerelative[i];
  40                         __m128 b0, b1, b2, b3, r0, r1, r2, r3, nr;
  41                         if (model->data_bones[i].parent >= 0)
  42                         {
  43                                 const matrix4x4_t * RESTRICT p = &bonepose[model->data_bones[i].parent];
  44                                 __m128 s0 = _mm_loadu_ps(s->m[0]), s1 = _mm_loadu_ps(s->m[1]), s2 = _mm_loadu_ps(s->m[2]);
  45 #ifdef OPENGLORIENTATION
  46                                 __m128 s3 = _mm_loadu_ps(s->m[3]);
  47 #define SKELETON_MATRIX(r, c) _mm_shuffle_ps(s##c, s##c, _MM_SHUFFLE(r, r, r, r))
  48 #else
  49 #define SKELETON_MATRIX(r, c) _mm_shuffle_ps(s##r, s##r, _MM_SHUFFLE(c, c, c, c))
  50 #endif
  51                                 __m128 pr = _mm_load_ps(p->m[0]);
  52                                 b0 = _mm_mul_ps(pr, SKELETON_MATRIX(0, 0));
  53                                 b1 = _mm_mul_ps(pr, SKELETON_MATRIX(0, 1));
  54                                 b2 = _mm_mul_ps(pr, SKELETON_MATRIX(0, 2));
  55                                 b3 = _mm_mul_ps(pr, SKELETON_MATRIX(0, 3));
  56                                 pr = _mm_load_ps(p->m[1]);
  57                                 b0 = _mm_add_ps(b0, _mm_mul_ps(pr, SKELETON_MATRIX(1, 0)));
  58                                 b1 = _mm_add_ps(b1, _mm_mul_ps(pr, SKELETON_MATRIX(1, 1)));
  59                                 b2 = _mm_add_ps(b2, _mm_mul_ps(pr, SKELETON_MATRIX(1, 2)));
  60                                 b3 = _mm_add_ps(b3, _mm_mul_ps(pr, SKELETON_MATRIX(1, 3)));
  61                                 pr = _mm_load_ps(p->m[2]);
  62                                 b0 = _mm_add_ps(b0, _mm_mul_ps(pr, SKELETON_MATRIX(2, 0)));
  63                                 b1 = _mm_add_ps(b1, _mm_mul_ps(pr, SKELETON_MATRIX(2, 1)));
  64                                 b2 = _mm_add_ps(b2, _mm_mul_ps(pr, SKELETON_MATRIX(2, 2)));
  65                                 b3 = _mm_add_ps(b3, _mm_mul_ps(pr, SKELETON_MATRIX(2, 3)));
  66                                 b3 = _mm_add_ps(b3, _mm_load_ps(p->m[3]));
  67                         }
  68                         else
  69                         {
  70                                 b0 = _mm_loadu_ps(s->m[0]);
  71                                 b1 = _mm_loadu_ps(s->m[1]);
  72                                 b2 = _mm_loadu_ps(s->m[2]);
  73                                 b3 = _mm_loadu_ps(s->m[3]);
  74 #ifndef OPENGLORIENTATION
  75                                 _MM_TRANSPOSE4_PS(b0, b1, b2, b3);
  76 #endif
  77                         }
  78                         _mm_store_ps(b->m[0], b0);
  79                         _mm_store_ps(b->m[1], b1);
  80                         _mm_store_ps(b->m[2], b2);
  81                         _mm_store_ps(b->m[3], b3);
  82                         nr = _mm_loadu_ps(n);
  83                         r0 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0)));
  84                         r1 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1)));
  85                         r2 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2)));
  86                         r3 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3)));
  87                         nr = _mm_loadu_ps(n+4);
  88                         r0 = _mm_add_ps(r0, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0))));
  89                         r1 = _mm_add_ps(r1, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1))));
  90                         r2 = _mm_add_ps(r2, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2))));
  91                         r3 = _mm_add_ps(r3, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3))));
  92                         nr = _mm_loadu_ps(n+8);
  93                         r0 = _mm_add_ps(r0, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0))));
  94                         r1 = _mm_add_ps(r1, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1))));
  95                         r2 = _mm_add_ps(r2, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2))));
  96                         r3 = _mm_add_ps(r3, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3))));
  97                         r3 = _mm_add_ps(r3, b3);
  98                         _mm_store_ps(r->m[0], r0);
  99                         _mm_store_ps(r->m[1], r1);
 100                         _mm_store_ps(r->m[2], r2);
 101                         _mm_store_ps(r->m[3], r3);
 102                 }
 103         }
 104         else
 105         {
 106                 for (i = 0;i < model->num_bones;i++)
 107                 {
 108                         const short * RESTRICT pose7s = model->data_poses7s + 7 * (frameblend[0].subframe * model->num_bones + i);
 109                         float lerp = frameblend[0].lerp,
 110                                 tx = pose7s[0], ty = pose7s[1], tz = pose7s[2],
 111                                 rx = pose7s[3] * lerp,
 112                                 ry = pose7s[4] * lerp,
 113                                 rz = pose7s[5] * lerp,
 114                                 rw = pose7s[6] * lerp,
 115                                 dx = tx*rw + ty*rz - tz*ry,
 116                                 dy = -tx*rz + ty*rw + tz*rx,
 117                                 dz = tx*ry - ty*rx + tz*rw,
 118                                 dw = -tx*rx - ty*ry - tz*rz,
 119                                 scale, originscale;
 120                         for (blends = 1;blends < MAX_FRAMEBLENDS && frameblend[blends].lerp > 0;blends++)
 121                         {
 122                                 const short * RESTRICT pose7s = model->data_poses7s + 7 * (frameblend[blends].subframe * model->num_bones + i);
 123                                 float lerp = frameblend[blends].lerp,
 124                                         tx = pose7s[0], ty = pose7s[1], tz = pose7s[2],
 125                                         qx = pose7s[3], qy = pose7s[4], qz = pose7s[5], qw = pose7s[6];
 126                                 if(rx*qx + ry*qy + rz*qz + rw*qw < 0) lerp = -lerp;
 127                                 qx *= lerp;
 128                                 qy *= lerp;
 129                                 qz *= lerp;
 130                                 qw *= lerp;
 131                                 rx += qx;
 132                                 ry += qy;
 133                                 rz += qz;
 134                                 rw += qw;
 135                                 dx += tx*qw + ty*qz - tz*qy;
 136                                 dy += -tx*qz + ty*qw + tz*qx;
 137                                 dz += tx*qy - ty*qx + tz*qw;
 138                                 dw += -tx*qx - ty*qy - tz*qz;
 139                         }
 140                         scale = 1.0f / (rx*rx + ry*ry + rz*rz + rw*rw);
 141                         originscale = -model->num_posescale * scale;
 142                         m[0] = scale*(rw*rw + rx*rx - ry*ry - rz*rz);
 143                         m[1] = 2*scale*(rx*ry - rw*rz);
 144                         m[2] = 2*scale*(rx*rz + rw*ry);
 145                         m[3] = originscale*(dw*rx - dx*rw + dy*rz - dz*ry);
 146                         m[4] = 2*scale*(rx*ry + rw*rz);
 147                         m[5] = scale*(rw*rw + ry*ry - rx*rx - rz*rz);
 148                         m[6] = 2*scale*(ry*rz - rw*rx);
 149                         m[7] = originscale*(dw*ry - dx*rz - dy*rw + dz*rx);
 150                         m[8] = 2*scale*(rx*rz - rw*ry);
 151                         m[9] = 2*scale*(ry*rz + rw*rx);
 152                         m[10] = scale*(rw*rw + rz*rz - rx*rx - ry*ry);
 153                         m[11] = originscale*(dw*rz + dx*ry - dy*rx - dz*rw);
 154                         if (i == r_skeletal_debugbone.integer)
 155                                 m[r_skeletal_debugbonecomponent.integer % 12] += r_skeletal_debugbonevalue.value;
 156                         m[3] *= r_skeletal_debugtranslatex.value;
 157                         m[7] *= r_skeletal_debugtranslatey.value;
 158                         m[11] *= r_skeletal_debugtranslatez.value;
 159                         {
 160                                 const float * RESTRICT n = model->data_baseboneposeinverse + i * 12;
 161                                 matrix4x4_t * RESTRICT b = &bonepose[i];
 162                                 matrix4x4_t * RESTRICT r = &boneposerelative[i];
 163                                 __m128 b0, b1, b2, b3, r0, r1, r2, r3, nr;
 164                                 if (model->data_bones[i].parent >= 0)
 165                                 {
 166                                         const matrix4x4_t * RESTRICT p = &bonepose[model->data_bones[i].parent];
 167                                         __m128 pr = _mm_load_ps(p->m[0]);
 168                                         b0 = _mm_mul_ps(pr, _mm_set1_ps(m[0]));
 169                                         b1 = _mm_mul_ps(pr, _mm_set1_ps(m[1]));
 170                                         b2 = _mm_mul_ps(pr, _mm_set1_ps(m[2]));
 171                                         b3 = _mm_mul_ps(pr, _mm_set1_ps(m[3]));
 172                                         pr = _mm_load_ps(p->m[1]);
 173                                         b0 = _mm_add_ps(b0, _mm_mul_ps(pr, _mm_set1_ps(m[4])));
 174                                         b1 = _mm_add_ps(b1, _mm_mul_ps(pr, _mm_set1_ps(m[5])));
 175                                         b2 = _mm_add_ps(b2, _mm_mul_ps(pr, _mm_set1_ps(m[6])));
 176                                         b3 = _mm_add_ps(b3, _mm_mul_ps(pr, _mm_set1_ps(m[7])));
 177                                         pr = _mm_load_ps(p->m[2]);
 178                                         b0 = _mm_add_ps(b0, _mm_mul_ps(pr, _mm_set1_ps(m[8])));
 179                                         b1 = _mm_add_ps(b1, _mm_mul_ps(pr, _mm_set1_ps(m[9])));
 180                                         b2 = _mm_add_ps(b2, _mm_mul_ps(pr, _mm_set1_ps(m[10])));
 181                                         b3 = _mm_add_ps(b3, _mm_mul_ps(pr, _mm_set1_ps(m[11])));
 182                                         b3 = _mm_add_ps(b3, _mm_load_ps(p->m[3]));
 183                                 }
 184                                 else
 185                                 {
 186                                         b0 = _mm_setr_ps(m[0], m[4], m[8], 0.0f);
 187                                         b1 = _mm_setr_ps(m[1], m[5], m[9], 0.0f);
 188                                         b2 = _mm_setr_ps(m[2], m[6], m[10], 0.0f);
 189                                         b3 = _mm_setr_ps(m[3], m[7], m[11], 1.0f);
 190                                 }
 191                                 _mm_store_ps(b->m[0], b0);
 192                                 _mm_store_ps(b->m[1], b1);
 193                                 _mm_store_ps(b->m[2], b2);
 194                                 _mm_store_ps(b->m[3], b3);
 195                                 nr = _mm_loadu_ps(n);
 196                                 r0 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0)));
 197                                 r1 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1)));
 198                                 r2 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2)));
 199                                 r3 = _mm_mul_ps(b0, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3)));
 200                                 nr = _mm_loadu_ps(n+4);
 201                                 r0 = _mm_add_ps(r0, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0))));
 202                                 r1 = _mm_add_ps(r1, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1))));
 203                                 r2 = _mm_add_ps(r2, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2))));
 204                                 r3 = _mm_add_ps(r3, _mm_mul_ps(b1, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3))));
 205                                 nr = _mm_loadu_ps(n+8);
 206                                 r0 = _mm_add_ps(r0, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(0, 0, 0, 0))));
 207                                 r1 = _mm_add_ps(r1, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(1, 1, 1, 1))));
 208                                 r2 = _mm_add_ps(r2, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(2, 2, 2, 2))));
 209                                 r3 = _mm_add_ps(r3, _mm_mul_ps(b2, _mm_shuffle_ps(nr, nr, _MM_SHUFFLE(3, 3, 3, 3))));
 210                                 r3 = _mm_add_ps(r3, b3);
 211                                 _mm_store_ps(r->m[0], r0);
 212                                 _mm_store_ps(r->m[1], r1);
 213                                 _mm_store_ps(r->m[2], r2);
 214                                 _mm_store_ps(r->m[3], r3);
 215                         }
 216                 }
 217         }
 218
 219         // generate matrices for all blend combinations
 220         weights = model->surfmesh.data_blendweights;
 221         for (i = 0;i < model->surfmesh.num_blends;i++, weights++)
 222         {
 223                 float * RESTRICT b = &boneposerelative[model->num_bones + i].m[0][0];
 224                 const float * RESTRICT m = &boneposerelative[weights->index[0]].m[0][0];
 225                 float f = weights->influence[0] * (1.0f / 255.0f);
 226                 __m128 fv = _mm_set_ps1(f);
 227                 __m128 b0 = _mm_load_ps(m);
 228                 __m128 b1 = _mm_load_ps(m+4);
 229                 __m128 b2 = _mm_load_ps(m+8);
 230                 __m128 b3 = _mm_load_ps(m+12);
 231                 __m128 m0, m1, m2, m3;
 232                 b0 = _mm_mul_ps(b0, fv);
 233                 b1 = _mm_mul_ps(b1, fv);
 234                 b2 = _mm_mul_ps(b2, fv);
 235                 b3 = _mm_mul_ps(b3, fv);
 236                 for (k = 1;k < 4 && weights->influence[k];k++)
 237                 {
 238                         m = &boneposerelative[weights->index[k]].m[0][0];
 239                         f = weights->influence[k] * (1.0f / 255.0f);
 240                         fv = _mm_set_ps1(f);
 241                         m0 = _mm_load_ps(m);
 242                         m1 = _mm_load_ps(m+4);
 243                         m2 = _mm_load_ps(m+8);
 244                         m3 = _mm_load_ps(m+12);
 245                         m0 = _mm_mul_ps(m0, fv);
 246                         m1 = _mm_mul_ps(m1, fv);
 247                         m2 = _mm_mul_ps(m2, fv);
 248                         m3 = _mm_mul_ps(m3, fv);
 249                         b0 = _mm_add_ps(m0, b0);
 250                         b1 = _mm_add_ps(m1, b1);
 251                         b2 = _mm_add_ps(m2, b2);
 252                         b3 = _mm_add_ps(m3, b3);
 253                 }
 254                 _mm_store_ps(b, b0);
 255                 _mm_store_ps(b+4, b1);
 256                 _mm_store_ps(b+8, b2);
 257                 _mm_store_ps(b+12, b3);
 258         }
 259
 260 #define LOAD_MATRIX_SCALAR() const float * RESTRICT m = &boneposerelative[*b].m[0][0]
 261
 262 #define LOAD_MATRIX3() \
 263         const float * RESTRICT m = &boneposerelative[*b].m[0][0]; \
 264         /* bonepose array is 16 byte aligned */ \
 265         __m128 m1 = _mm_load_ps((m)); \
 266         __m128 m2 = _mm_load_ps((m)+4); \
 267         __m128 m3 = _mm_load_ps((m)+8);
 268 #define LOAD_MATRIX4() \
 269         const float * RESTRICT m = &boneposerelative[*b].m[0][0]; \
 270         /* bonepose array is 16 byte aligned */ \
 271         __m128 m1 = _mm_load_ps((m)); \
 272         __m128 m2 = _mm_load_ps((m)+4); \
 273         __m128 m3 = _mm_load_ps((m)+8); \
 274         __m128 m4 = _mm_load_ps((m)+12)
 275
 276         /* Note that matrix is 4x4 and transposed compared to non-USE_SSE codepath */
 277 #define TRANSFORM_POSITION_SCALAR(in, out) \
 278         (out)[0] = ((in)[0] * m[0] + (in)[1] * m[4] + (in)[2] * m[ 8] + m[12]); \
 279         (out)[1] = ((in)[0] * m[1] + (in)[1] * m[5] + (in)[2] * m[ 9] + m[13]); \
 280         (out)[2] = ((in)[0] * m[2] + (in)[1] * m[6] + (in)[2] * m[10] + m[14]);
 281 #define TRANSFORM_VECTOR_SCALAR(in, out) \
 282         (out)[0] = ((in)[0] * m[0] + (in)[1] * m[4] + (in)[2] * m[ 8]); \
 283         (out)[1] = ((in)[0] * m[1] + (in)[1] * m[5] + (in)[2] * m[ 9]); \
 284         (out)[2] = ((in)[0] * m[2] + (in)[1] * m[6] + (in)[2] * m[10]);
 285
 286 #define TRANSFORM_POSITION(in, out) { \
 287                 __m128 pin = _mm_loadu_ps(in); /* we ignore the value in the last element (x from the next vertex) */ \
 288                 __m128 x = _mm_shuffle_ps(pin, pin, 0x0); \
 289                 __m128 t1 = _mm_mul_ps(x, m1); \
 290                 \
 291                 /* y, + x */ \
 292                 __m128 y = _mm_shuffle_ps(pin, pin, 0x55); \
 293                 __m128 t2 = _mm_mul_ps(y, m2); \
 294                 __m128 t3 = _mm_add_ps(t1, t2); \
 295                 \
 296                 /* z, + (y+x) */ \
 297                 __m128 z = _mm_shuffle_ps(pin, pin, 0xaa); \
 298                 __m128 t4 = _mm_mul_ps(z, m3); \
 299                 __m128 t5 = _mm_add_ps(t3, t4); \
 300                 \
 301                 /* + m3 */ \
 302                 __m128 pout = _mm_add_ps(t5, m4); \
 303                 _mm_storeu_ps((out), pout); \
 304         }
 305
 306 #define TRANSFORM_VECTOR(in, out) { \
 307                 __m128 vin = _mm_loadu_ps(in); \
 308                 \
 309                 /* x */ \
 310                 __m128 x = _mm_shuffle_ps(vin, vin, 0x0); \
 311                 __m128 t1 = _mm_mul_ps(x, m1); \
 312                 \
 313                 /* y, + x */ \
 314                 __m128 y = _mm_shuffle_ps(vin, vin, 0x55); \
 315                 __m128 t2 = _mm_mul_ps(y, m2); \
 316                 __m128 t3 = _mm_add_ps(t1, t2); \
 317                 \
 318                 /* nz, + (ny + nx) */ \
 319                 __m128 z = _mm_shuffle_ps(vin, vin, 0xaa); \
 320                 __m128 t4 = _mm_mul_ps(z, m3); \
 321                 __m128 vout = _mm_add_ps(t3, t4); \
 322                 _mm_storeu_ps((out), vout); \
 323         }
 324
 325         // transform vertex attributes by blended matrices
 326         if (vertex3f)
 327         {
 328                 const float * RESTRICT v = model->surfmesh.data_vertex3f;
 329                 const unsigned short * RESTRICT b = model->surfmesh.blends;
 330                 // special case common combinations of attributes to avoid repeated loading of matrices
 331                 if (normal3f)
 332                 {
 333                         const float * RESTRICT n = model->surfmesh.data_normal3f;
 334                         if (svector3f && tvector3f)
 335                         {
 336                                 const float * RESTRICT sv = model->surfmesh.data_svector3f;
 337                                 const float * RESTRICT tv = model->surfmesh.data_tvector3f;
 338
 339                                 // Note that for SSE each iteration stores one element past end, so we break one vertex short
 340                                 // and handle that with scalars in that case
 341                                 for (i = 0; i < num_vertices_minus_one; i++, v += 3, n += 3, sv += 3, tv += 3, b++,
 342                                                 vertex3f += 3, normal3f += 3, svector3f += 3, tvector3f += 3)
 343                                 {
 344                                         LOAD_MATRIX4();
 345                                         TRANSFORM_POSITION(v, vertex3f);
 346                                         TRANSFORM_VECTOR(n, normal3f);
 347                                         TRANSFORM_VECTOR(sv, svector3f);
 348                                         TRANSFORM_VECTOR(tv, tvector3f);
 349                                 }
 350
 351                                 // Last vertex needs to be done with scalars to avoid reading/writing 1 word past end of arrays
 352                                 {
 353                                         LOAD_MATRIX_SCALAR();
 354                                         TRANSFORM_POSITION_SCALAR(v, vertex3f);
 355                                         TRANSFORM_VECTOR_SCALAR(n, normal3f);
 356                                         TRANSFORM_VECTOR_SCALAR(sv, svector3f);
 357                                         TRANSFORM_VECTOR_SCALAR(tv, tvector3f);
 358                                 }
 359                                 //printf("elapsed ticks: %llu\n", rdtsc() - ts); // XXX
 360                                 return;
 361                         }
 362
 363                         for (i = 0;i < num_vertices_minus_one; i++, v += 3, n += 3, b++, vertex3f += 3, normal3f += 3)
 364                         {
 365                                 LOAD_MATRIX4();
 366                                 TRANSFORM_POSITION(v, vertex3f);
 367                                 TRANSFORM_VECTOR(n, normal3f);
 368                         }
 369                         {
 370                                 LOAD_MATRIX_SCALAR();
 371                                 TRANSFORM_POSITION_SCALAR(v, vertex3f);
 372                                 TRANSFORM_VECTOR_SCALAR(n, normal3f);
 373                         }
 374                 }
 375                 else
 376                 {
 377                         for (i = 0;i < num_vertices_minus_one; i++, v += 3, b++, vertex3f += 3)
 378                         {
 379                                 LOAD_MATRIX4();
 380                                 TRANSFORM_POSITION(v, vertex3f);
 381                         }
 382                         {
 383                                 LOAD_MATRIX_SCALAR();
 384                                 TRANSFORM_POSITION_SCALAR(v, vertex3f);
 385                         }
 386                 }
 387         }
 388
 389         else if (normal3f)
 390         {
 391                 const float * RESTRICT n = model->surfmesh.data_normal3f;
 392                 const unsigned short * RESTRICT b = model->surfmesh.blends;
 393                 for (i = 0; i < num_vertices_minus_one; i++, n += 3, b++, normal3f += 3)
 394                 {
 395                         LOAD_MATRIX3();
 396                         TRANSFORM_VECTOR(n, normal3f);
 397                 }
 398                 {
 399                         LOAD_MATRIX_SCALAR();
 400                         TRANSFORM_VECTOR_SCALAR(n, normal3f);
 401                 }
 402         }
 403
 404         if (svector3f)
 405         {
 406                 const float * RESTRICT sv = model->surfmesh.data_svector3f;
 407                 const unsigned short * RESTRICT b = model->surfmesh.blends;
 408                 for (i = 0; i < num_vertices_minus_one; i++, sv += 3, b++, svector3f += 3)
 409                 {
 410                         LOAD_MATRIX3();
 411                         TRANSFORM_VECTOR(sv, svector3f);
 412                 }
 413                 {
 414                         LOAD_MATRIX_SCALAR();
 415                         TRANSFORM_VECTOR_SCALAR(sv, svector3f);
 416                 }
 417         }
 418
 419         if (tvector3f)
 420         {
 421                 const float * RESTRICT tv = model->surfmesh.data_tvector3f;
 422                 const unsigned short * RESTRICT b = model->surfmesh.blends;
 423                 for (i = 0; i < num_vertices_minus_one; i++, tv += 3, b++, tvector3f += 3)
 424                 {
 425                         LOAD_MATRIX3();
 426                         TRANSFORM_VECTOR(tv, tvector3f);
 427                 }
 428                 {
 429                         LOAD_MATRIX_SCALAR();
 430                         TRANSFORM_VECTOR_SCALAR(tv, tvector3f);
 431                 }
 432         }
 433
 434 #undef LOAD_MATRIX3
 435 #undef LOAD_MATRIX4
 436 #undef TRANSFORM_POSITION
 437 #undef TRANSFORM_VECTOR
 438 #undef LOAD_MATRIX_SCALAR
 439 #undef TRANSFORM_POSITION_SCALAR
 440 #undef TRANSFORM_VECTOR_SCALAR
 441 }
 442
 443 #endif