fix crashes on Win64 builds caused by stack overflow in R_DrawRTLight

[xonotic/darkplaces.git] / r_shadow.c

diff --git a/r_shadow.c b/r_shadow.c
index 4864bb869c7030834fe71e46dee3c267e6ee827a..414560e4395e6de71a0bab9a7033fd71d132b5ee 100644 (file)
--- a/r_shadow.c
+++ b/r_shadow.c
@@ -178,6 +178,7 @@ int *vertexremap;
 int vertexupdatenum;
 
 int r_shadow_buffer_numleafpvsbytes;
+unsigned char *r_shadow_buffer_visitingleafpvs;
 unsigned char *r_shadow_buffer_leafpvs;
 int *r_shadow_buffer_leaflist;
 
@@ -318,6 +319,7 @@ void r_shadow_start(void)
        shadowmarklist = NULL;
        shadowmarkcount = 0;
        r_shadow_buffer_numleafpvsbytes = 0;
+       r_shadow_buffer_visitingleafpvs = NULL;
        r_shadow_buffer_leafpvs = NULL;
        r_shadow_buffer_leaflist = NULL;
        r_shadow_buffer_numsurfacepvsbytes = 0;
@@ -363,6 +365,9 @@ void r_shadow_shutdown(void)
        shadowmarklist = NULL;
        shadowmarkcount = 0;
        r_shadow_buffer_numleafpvsbytes = 0;
+       if (r_shadow_buffer_visitingleafpvs)
+               Mem_Free(r_shadow_buffer_visitingleafpvs);
+       r_shadow_buffer_visitingleafpvs = NULL;
        if (r_shadow_buffer_leafpvs)
                Mem_Free(r_shadow_buffer_leafpvs);
        r_shadow_buffer_leafpvs = NULL;
@@ -484,6 +489,7 @@ void R_Shadow_Init(void)
        shadowmarklist = NULL;
        shadowmarkcount = 0;
        r_shadow_buffer_numleafpvsbytes = 0;
+       r_shadow_buffer_visitingleafpvs = NULL;
        r_shadow_buffer_leafpvs = NULL;
        r_shadow_buffer_leaflist = NULL;
        r_shadow_buffer_numsurfacepvsbytes = 0;
@@ -542,11 +548,14 @@ static void R_Shadow_EnlargeLeafSurfaceTrisBuffer(int numleafs, int numsurfaces,
        int numlighttrispvsbytes = (((numlighttriangles + 7) >> 3) + 255) & ~255;
        if (r_shadow_buffer_numleafpvsbytes < numleafpvsbytes)
        {
+               if (r_shadow_buffer_visitingleafpvs)
+                       Mem_Free(r_shadow_buffer_visitingleafpvs);
                if (r_shadow_buffer_leafpvs)
                        Mem_Free(r_shadow_buffer_leafpvs);
                if (r_shadow_buffer_leaflist)
                        Mem_Free(r_shadow_buffer_leaflist);
                r_shadow_buffer_numleafpvsbytes = numleafpvsbytes;
+               r_shadow_buffer_visitingleafpvs = (unsigned char *)Mem_Alloc(r_main_mempool, r_shadow_buffer_numleafpvsbytes);
                r_shadow_buffer_leafpvs = (unsigned char *)Mem_Alloc(r_main_mempool, r_shadow_buffer_numleafpvsbytes);
                r_shadow_buffer_leaflist = (int *)Mem_Alloc(r_main_mempool, r_shadow_buffer_numleafpvsbytes * 8 * sizeof(*r_shadow_buffer_leaflist));
        }
@@ -2468,7 +2477,7 @@ void R_RTLight_Compile(rtlight_t *rtlight)
                // this variable must be set for the CompileShadowVolume code
                r_shadow_compilingrtlight = rtlight;
                R_Shadow_EnlargeLeafSurfaceTrisBuffer(model->brush.num_leafs, model->num_surfaces, model->brush.shadowmesh ? model->brush.shadowmesh->numtriangles : model->surfmesh.num_triangles, model->surfmesh.num_triangles);
-               model->GetLightInfo(ent, rtlight->shadoworigin, rtlight->radius, rtlight->cullmins, rtlight->cullmaxs, r_shadow_buffer_leaflist, r_shadow_buffer_leafpvs, &numleafs, r_shadow_buffer_surfacelist, r_shadow_buffer_surfacepvs, &numsurfaces, r_shadow_buffer_shadowtrispvs, r_shadow_buffer_lighttrispvs);
+               model->GetLightInfo(ent, rtlight->shadoworigin, rtlight->radius, rtlight->cullmins, rtlight->cullmaxs, r_shadow_buffer_leaflist, r_shadow_buffer_leafpvs, &numleafs, r_shadow_buffer_surfacelist, r_shadow_buffer_surfacepvs, &numsurfaces, r_shadow_buffer_shadowtrispvs, r_shadow_buffer_lighttrispvs, r_shadow_buffer_visitingleafpvs);
                numleafpvsbytes = (model->brush.num_leafs + 7) >> 3;
                numshadowtrispvsbytes = ((model->brush.shadowmesh ? model->brush.shadowmesh->numtriangles : model->surfmesh.num_triangles) + 7) >> 3;
                numlighttrispvsbytes = (model->surfmesh.num_triangles + 7) >> 3;
@@ -2560,7 +2569,8 @@ void R_Shadow_UncompileWorldLights(void)
 {
        size_t lightindex;
        dlight_t *light;
-       for (lightindex = 0;lightindex < Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray);lightindex++)
+       size_t range = Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray); // checked
+       for (lightindex = 0;lightindex < range;lightindex++)
        {
                light = Mem_ExpandableArray_RecordAtIndex(&r_shadow_worldlightsarray, lightindex);
                if (!light)
@@ -2850,10 +2860,10 @@ void R_DrawRTLight(rtlight_t *rtlight, qboolean visible)
        int numlightentities_noselfshadow;
        int numshadowentities;
        int numshadowentities_noselfshadow;
-       entity_render_t *lightentities[MAX_EDICTS];
-       entity_render_t *lightentities_noselfshadow[MAX_EDICTS];
-       entity_render_t *shadowentities[MAX_EDICTS];
-       entity_render_t *shadowentities_noselfshadow[MAX_EDICTS];
+       static entity_render_t *lightentities[MAX_EDICTS];
+       static entity_render_t *lightentities_noselfshadow[MAX_EDICTS];
+       static entity_render_t *shadowentities[MAX_EDICTS];
+       static entity_render_t *shadowentities_noselfshadow[MAX_EDICTS];
 
        // skip lights that don't light because of ambientscale+diffusescale+specularscale being 0 (corona only lights)
        // skip lights that are basically invisible (color 0 0 0)
@@ -2908,7 +2918,7 @@ void R_DrawRTLight(rtlight_t *rtlight, qboolean visible)
                // dynamic light, world available and can receive realtime lighting
                // calculate lit surfaces and leafs
                R_Shadow_EnlargeLeafSurfaceTrisBuffer(r_refdef.scene.worldmodel->brush.num_leafs, r_refdef.scene.worldmodel->num_surfaces, r_refdef.scene.worldmodel->brush.shadowmesh ? r_refdef.scene.worldmodel->brush.shadowmesh->numtriangles : r_refdef.scene.worldmodel->surfmesh.num_triangles, r_refdef.scene.worldmodel->surfmesh.num_triangles);
-               r_refdef.scene.worldmodel->GetLightInfo(r_refdef.scene.worldentity, rtlight->shadoworigin, rtlight->radius, rsurface.rtlight_cullmins, rsurface.rtlight_cullmaxs, r_shadow_buffer_leaflist, r_shadow_buffer_leafpvs, &numleafs, r_shadow_buffer_surfacelist, r_shadow_buffer_surfacepvs, &numsurfaces, r_shadow_buffer_shadowtrispvs, r_shadow_buffer_lighttrispvs);
+               r_refdef.scene.worldmodel->GetLightInfo(r_refdef.scene.worldentity, rtlight->shadoworigin, rtlight->radius, rsurface.rtlight_cullmins, rsurface.rtlight_cullmaxs, r_shadow_buffer_leaflist, r_shadow_buffer_leafpvs, &numleafs, r_shadow_buffer_surfacelist, r_shadow_buffer_surfacepvs, &numsurfaces, r_shadow_buffer_shadowtrispvs, r_shadow_buffer_lighttrispvs, r_shadow_buffer_visitingleafpvs);
                leaflist = r_shadow_buffer_leaflist;
                leafpvs = r_shadow_buffer_leafpvs;
                surfacelist = r_shadow_buffer_surfacelist;
@@ -3128,6 +3138,7 @@ void R_ShadowVolumeLighting(qboolean visible)
        int lnum;
        size_t lightindex;
        dlight_t *light;
+       size_t range;
 
        if (r_editlights.integer)
                R_Shadow_DrawLightSprites();
@@ -3144,7 +3155,8 @@ void R_ShadowVolumeLighting(qboolean visible)
        }
        else
        {
-               for (lightindex = 0;lightindex < Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray);lightindex++)
+               range = Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray); // checked
+               for (lightindex = 0;lightindex < range;lightindex++)
                {
                        light = Mem_ExpandableArray_RecordAtIndex(&r_shadow_worldlightsarray, lightindex);
                        if (light && (light->flags & flag))
@@ -3253,12 +3265,14 @@ void R_DrawCoronas(void)
        size_t lightindex;
        dlight_t *light;
        rtlight_t *rtlight;
+       size_t range;
        if (r_coronas.value < (1.0f / 256.0f) && !gl_flashblend.integer)
                return;
        R_Mesh_Matrix(&identitymatrix);
        flag = r_refdef.scene.rtworld ? LIGHTFLAG_REALTIMEMODE : LIGHTFLAG_NORMALMODE;
        // FIXME: these traces should scan all render entities instead of cl.world
-       for (lightindex = 0;lightindex < Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray);lightindex++)
+       range = Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray); // checked
+       for (lightindex = 0;lightindex < range;lightindex++)
        {
                light = Mem_ExpandableArray_RecordAtIndex(&r_shadow_worldlightsarray, lightindex);
                if (!light)
@@ -3494,7 +3508,8 @@ void R_Shadow_ClearWorldLights(void)
 {
        size_t lightindex;
        dlight_t *light;
-       for (lightindex = 0;lightindex < Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray);lightindex++)
+       size_t range = Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray); // checked
+       for (lightindex = 0;lightindex < range;lightindex++)
        {
                light = Mem_ExpandableArray_RecordAtIndex(&r_shadow_worldlightsarray, lightindex);
                if (light)
@@ -3557,7 +3572,8 @@ void R_Shadow_DrawLightSprites(void)
 {
        size_t lightindex;
        dlight_t *light;
-       for (lightindex = 0;lightindex < Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray);lightindex++)
+       size_t range = Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray); // checked
+       for (lightindex = 0;lightindex < range;lightindex++)
        {
                light = Mem_ExpandableArray_RecordAtIndex(&r_shadow_worldlightsarray, lightindex);
                if (light)
@@ -3572,9 +3588,10 @@ void R_Shadow_SelectLightInView(void)
        dlight_t *best;
        size_t lightindex;
        dlight_t *light;
+       size_t range = Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray); // checked
        best = NULL;
        bestrating = 0;
-       for (lightindex = 0;lightindex < Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray);lightindex++)
+       for (lightindex = 0;lightindex < range;lightindex++)
        {
                light = Mem_ExpandableArray_RecordAtIndex(&r_shadow_worldlightsarray, lightindex);
                if (!light)
@@ -3706,7 +3723,9 @@ void R_Shadow_SaveWorldLights(void)
        char *buf, *oldbuf;
        char name[MAX_QPATH];
        char line[MAX_INPUTLINE];
-       if (!Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray))
+       size_t range = Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray); // checked, assuming the dpsnprintf mess doesn't screw it up...
+       // I hate lines which are 3 times my screen size :( --blub
+       if (!range)
                return;
        if (cl.worldmodel == NULL)
        {
@@ -3717,7 +3736,7 @@ void R_Shadow_SaveWorldLights(void)
        strlcat (name, ".rtlights", sizeof (name));
        bufchars = bufmaxchars = 0;
        buf = NULL;
-       for (lightindex = 0;lightindex < Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray);lightindex++)
+       for (lightindex = 0;lightindex < range;lightindex++)
        {
                light = Mem_ExpandableArray_RecordAtIndex(&r_shadow_worldlightsarray, lightindex);
                if (!light)
@@ -4434,6 +4453,7 @@ void R_Shadow_EditLights_EditAll_f(void)
 {
        size_t lightindex;
        dlight_t *light;
+       size_t range;
 
        if (!r_editlights.integer)
        {
@@ -4441,7 +4461,9 @@ void R_Shadow_EditLights_EditAll_f(void)
                return;
        }
 
-       for (lightindex = 0;lightindex < Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray);lightindex++)
+       // EditLights doesn't seem to have a "remove" command or something so:
+       range = Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray); // checked
+       for (lightindex = 0;lightindex < range;lightindex++)
        {
                light = Mem_ExpandableArray_RecordAtIndex(&r_shadow_worldlightsarray, lightindex);
                if (!light)
@@ -4454,7 +4476,7 @@ void R_Shadow_EditLights_EditAll_f(void)
 void R_Shadow_EditLights_DrawSelectedLightProperties(void)
 {
        int lightnumber, lightcount;
-       size_t lightindex;
+       size_t lightindex, range;
        dlight_t *light;
        float x, y;
        char temp[256];
@@ -4465,7 +4487,8 @@ void R_Shadow_EditLights_DrawSelectedLightProperties(void)
        DrawQ_Pic(x-5, y-5, NULL, 250, 155, 0, 0, 0, 0.75, 0);
        lightnumber = -1;
        lightcount = 0;
-       for (lightindex = 0;lightindex < Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray);lightindex++)
+       range = Mem_ExpandableArray_IndexRange(&r_shadow_worldlightsarray); // checked
+       for (lightindex = 0;lightindex < range;lightindex++)
        {
                light = Mem_ExpandableArray_RecordAtIndex(&r_shadow_worldlightsarray, lightindex);
                if (!light)