diff --git a/mapdraw.c b/mapdraw.c
index debf9756f8..d571be65a1 100644
--- a/mapdraw.c
+++ b/mapdraw.c
@@ -742,21 +742,6 @@ int msDrawLayer(mapObj *map, layerObj *layer, imageObj *image)
       if(retcode != MS_SUCCESS) {
         return MS_FAILURE;
       }
-      /*
-       * hack to work around bug #3834: if we have use an alternate renderer, the symbolset may contain
-       * symbols that reference it. We want to remove those references before the altFormat is destroyed
-       * to avoid a segfault and/or a leak, and so the the main renderer doesn't pick the cache up thinking
-       * it's for him.
-       */
-      for(i=0; i<map->symbolset.numsymbols; i++) {
-        if (map->symbolset.symbol[i]!=NULL) {
-          symbolObj *s = map->symbolset.symbol[i];
-          if(s->renderer == MS_IMAGE_RENDERER(maskLayer->maskimage)) {
-            MS_IMAGE_RENDERER(maskLayer->maskimage)->freeSymbol(s);
-            s->renderer = NULL;
-          }
-        }
-      }
       /* set the imagetype from the original outputformat back (it was removed by msSelectOutputFormat() */
       msFree(map->imagetype);
       map->imagetype = msStrdup(image->format->name);
@@ -825,21 +810,6 @@ int msDrawLayer(mapObj *map, layerObj *layer, imageObj *image)
     altrenderer->getRasterBufferHandle(image_draw,&rb);
     renderer->mergeRasterBuffer(image,&rb,layer->opacity*0.01,0,0,0,0,rb.width,rb.height);
 
-    /*
-     * hack to work around bug #3834: if we have use an alternate renderer, the symbolset may contain
-     * symbols that reference it. We want to remove those references before the altFormat is destroyed
-     * to avoid a segfault and/or a leak, and so the the main renderer doesn't pick the cache up thinking
-     * it's for him.
-     */
-    for(i=0; i<map->symbolset.numsymbols; i++) {
-      if (map->symbolset.symbol[i]!=NULL) {
-        symbolObj *s = map->symbolset.symbol[i];
-        if(s->renderer == altrenderer) {
-          altrenderer->freeSymbol(s);
-          s->renderer = NULL;
-        }
-      }
-    }
     msFreeImage(image_draw);
 
     /* set the imagetype from the original outputformat back (it was removed by msSelectOutputFormat() */
@@ -2058,7 +2028,7 @@ int msDrawShape(mapObj *map, layerObj *layer, shapeObj *shape, imageObj *image,
       }
       symbol = map->symbolset.symbol[style->symbol];
       if (symbol->type == MS_SYMBOL_PIXMAP) {
-        if (MS_SUCCESS != msPreloadImageSymbol(MS_MAP_RENDERER(map), symbol))
+        if (MS_SUCCESS != msPreloadImageSymbol(map->outputformat, symbol))
           return MS_FAILURE;
       } else if (symbol->type == MS_SYMBOL_SVG) {
 #ifdef USE_SVG_CAIRO
diff --git a/maplegend.c b/maplegend.c
index ffbfe12314..2f70481742 100644
--- a/maplegend.c
+++ b/maplegend.c
@@ -281,21 +281,6 @@ int msDrawLegendIcon(mapObj *map, layerObj *lp, classObj *theclass,
 
     altrenderer->getRasterBufferHandle(image_draw,&rb);
     renderer->mergeRasterBuffer(image,&rb,lp->opacity*0.01,0,0,0,0,rb.width,rb.height);
-    /*
-     * hack to work around bug #3834: if we have use an alternate renderer, the symbolset may contain
-     * symbols that reference it. We want to remove those references before the altFormat is destroyed
-     * to avoid a segfault and/or a leak, and so the the main renderer doesn't pick the cache up thinking
-     * it's for him.
-     */
-    for(i=0; i<map->symbolset.numsymbols; i++) {
-      if (map->symbolset.symbol[i]!=NULL) {
-        symbolObj *s = map->symbolset.symbol[i];
-        if(s->renderer == altrenderer) {
-          altrenderer->freeSymbol(s);
-          s->renderer = NULL;
-        }
-      }
-    }
     msFreeImage(image_draw);
 
   } else if(image != image_draw) {
@@ -654,8 +639,13 @@ int msEmbedLegend(mapObj *map, imageObj *img)
   rendererVTableObj *renderer;
 
   s = msGetSymbolIndex(&(map->symbolset), "legend", MS_FALSE);
-  if(s != -1)
-    msRemoveSymbol(&(map->symbolset), s); /* solves some caching issues in AGG with long-running processes */
+  if(s != -1) {
+    legendSymbol = msRemoveSymbol(&(map->symbolset), s); /* solves some caching issues in AGG with long-running processes */
+    if (legendSymbol && legendSymbol->refcount == 0) {
+      msFreeSymbol(legendSymbol); /* clean up */
+      msFree(legendSymbol);
+    }
+  }
 
   if(msGrowSymbolSet(&map->symbolset) == NULL)
     return -1;
@@ -690,8 +680,14 @@ int msEmbedLegend(mapObj *map, imageObj *img)
 
   if(MS_SUCCESS != renderer->getRasterBufferCopy(image,legendSymbol->pixmap_buffer))
     return MS_FAILURE;
-  legendSymbol->renderer = renderer;
-
+  if (legendSymbol->format != map->outputformat) {
+    /* clean up previous output format */
+    if( legendSymbol->format && --legendSymbol->format->refcount < 1 )
+      msFreeOutputFormat( legendSymbol->format );
+    legendSymbol->format = map->outputformat;
+    legendSymbol->format->refcount++;
+  }
+  
   msFreeImage( image );
 
   if(!legendSymbol->pixmap_buffer) return(-1); /* something went wrong creating scalebar */
diff --git a/maprendering.c b/maprendering.c
index 5a6169cb24..3fc7488e7f 100644
--- a/maprendering.c
+++ b/maprendering.c
@@ -204,7 +204,7 @@ imageObj *getTile(imageObj *img, symbolObj *symbol,  symbolStyleObj *s, int widt
           renderer->renderTruetypeSymbol(tileimg, p_x, p_y, symbol, s);
           break;
         case (MS_SYMBOL_PIXMAP):
-          if(msPreloadImageSymbol(renderer,symbol) != MS_SUCCESS) {
+          if(msPreloadImageSymbol(img->format,symbol) != MS_SUCCESS) {
             return NULL; /* failed to load image, renderer should have set the error message */
           }
           renderer->renderPixmapSymbol(tileimg, p_x, p_y, symbol, s);
@@ -256,7 +256,7 @@ imageObj *getTile(imageObj *img, symbolObj *symbol,  symbolStyleObj *s, int widt
               renderer->renderTruetypeSymbol(tile3img, p_x, p_y, symbol, s);
               break;
             case (MS_SYMBOL_PIXMAP):
-              if(msPreloadImageSymbol(renderer,symbol) != MS_SUCCESS) {
+              if(msPreloadImageSymbol(img->format,symbol) != MS_SUCCESS) {
                 return NULL; /* failed to load image, renderer should have set the error message */
               }
               renderer->renderPixmapSymbol(tile3img, p_x, p_y, symbol, s);
@@ -480,7 +480,13 @@ int msDrawLineSymbol(symbolSetObj *symbolset, imageObj *image, shapeObj *p,
 
       symbol = symbolset->symbol[style->symbol];
       /* store a reference to the renderer to be used for freeing */
-      symbol->renderer = renderer;
+      if (symbol->format != image->format) {
+        /* clean up previous output format */
+        if( symbol->format && --symbol->format->refcount < 1 )
+          msFreeOutputFormat( symbol->format );
+        symbol->format = image->format;
+        symbol->format->refcount++;
+      }
 
       width = style->width * scalefactor;
       width = MS_MIN(width,style->maxwidth*image->resolutionfactor);
@@ -533,7 +539,7 @@ int msDrawLineSymbol(symbolSetObj *symbolset, imageObj *image, shapeObj *p,
           break;
           case (MS_SYMBOL_PIXMAP): {
             if(!symbol->pixmap_buffer) {
-              if(MS_SUCCESS != msPreloadImageSymbol(renderer,symbol))
+              if(MS_SUCCESS != msPreloadImageSymbol(image->format,symbol))
                 return MS_FAILURE;
             }
           }
@@ -630,8 +636,15 @@ int msDrawShadeSymbol(symbolSetObj *symbolset, imageObj *image, shapeObj *p, sty
       shapeObj *offsetPolygon = NULL;
       symbolObj *symbol = symbolset->symbol[style->symbol];
       /* store a reference to the renderer to be used for freeing */
-      if(style->symbol)
-        symbol->renderer = renderer;
+      if(style->symbol) {
+        if (symbol->format != image->format) {
+          /* clean up previous output format */
+          if( symbol->format && --symbol->format->refcount < 1 )
+            msFreeOutputFormat( symbol->format );
+          symbol->format = image->format;
+          symbol->format->refcount++;
+        }
+      }
 
       if (style->offsetx != 0 || style->offsety != 0) {
         if(style->offsety==-99)
@@ -689,7 +702,7 @@ int msDrawShadeSymbol(symbolSetObj *symbolset, imageObj *image, shapeObj *p, sty
 
         switch(symbol->type) {
           case MS_SYMBOL_PIXMAP:
-            if(MS_SUCCESS != msPreloadImageSymbol(renderer,symbol)) {
+            if(MS_SUCCESS != msPreloadImageSymbol(image->format,symbol)) {
               ret = MS_FAILURE;
               goto cleanup;
             }
@@ -799,7 +812,13 @@ int msDrawMarkerSymbol(symbolSetObj *symbolset,imageObj *image, pointObj *p, sty
       double p_x,p_y;
       symbolObj *symbol = symbolset->symbol[style->symbol];
       /* store a reference to the renderer to be used for freeing */
-      symbol->renderer = renderer;
+      if (symbol->format != image->format) {
+        /* clean up previous output format */
+        if( symbol->format && --symbol->format->refcount < 1 )
+          msFreeOutputFormat( symbol->format );
+        symbol->format = image->format;
+        symbol->format->refcount++;
+      }
       switch (symbol->type) {
         case (MS_SYMBOL_TRUETYPE): {
           if (!symbol->full_font_path)
@@ -813,7 +832,7 @@ int msDrawMarkerSymbol(symbolSetObj *symbolset,imageObj *image, pointObj *p, sty
         break;
         case (MS_SYMBOL_PIXMAP): {
           if (!symbol->pixmap_buffer) {
-            if (MS_SUCCESS != msPreloadImageSymbol(renderer, symbol))
+            if (MS_SUCCESS != msPreloadImageSymbol(image->format, symbol))
               return MS_FAILURE;
           }
         }
diff --git a/mapscale.c b/mapscale.c
index 3f1a1fcee8..b2e9017d65 100644
--- a/mapscale.c
+++ b/mapscale.c
@@ -344,9 +344,14 @@ int msEmbedScalebar(mapObj *map, imageObj *img)
   char* imageType = NULL;
 
   index = msGetSymbolIndex(&(map->symbolset), "scalebar", MS_FALSE);
-  if(index != -1)
-    msRemoveSymbol(&(map->symbolset), index); /* remove cached symbol in case the function is called multiple
+  if(index != -1) {
+    embededSymbol = msRemoveSymbol(&(map->symbolset), index); /* remove cached symbol in case the function is called multiple
                       times with different zoom levels */
+    if (embededSymbol && embededSymbol->refcount == 0) {
+      msFreeSymbol(embededSymbol); /* clean up */
+      msFree(embededSymbol);
+    }
+  }
 
   if((embededSymbol=msGrowSymbolSet(&map->symbolset)) == NULL)
     return MS_FAILURE;
diff --git a/mapscript/php/mapscript_i.c b/mapscript/php/mapscript_i.c
index 7aea08248c..933c485be5 100644
--- a/mapscript/php/mapscript_i.c
+++ b/mapscript/php/mapscript_i.c
@@ -2045,7 +2045,7 @@ imageObj *symbolObj_getImage(symbolObj *self, outputFormatObj *input_format)
   }
 
   renderer = format->vtable;
-  msPreloadImageSymbol(renderer, self);
+  msPreloadImageSymbol(format, self);
   if (self->pixmap_buffer) {
     image = msImageCreate(self->pixmap_buffer->width, self->pixmap_buffer->height, format, NULL, NULL,
                           MS_DEFAULT_RESOLUTION, MS_DEFAULT_RESOLUTION, NULL);
diff --git a/mapscript/swiginc/symbol.i b/mapscript/swiginc/symbol.i
index de4862162b..9eef51843c 100644
--- a/mapscript/swiginc/symbol.i
+++ b/mapscript/swiginc/symbol.i
@@ -130,7 +130,7 @@
         }
 
         renderer = format->vtable;
-        msPreloadImageSymbol(renderer, self);
+        msPreloadImageSymbol(format, self);
         if (self->pixmap_buffer) 
         {
             image = msImageCreate(self->pixmap_buffer->width, self->pixmap_buffer->height, format, NULL, NULL,
diff --git a/mapserver.h b/mapserver.h
index 45cfac457b..8097642e42 100644
--- a/mapserver.h
+++ b/mapserver.h
@@ -106,6 +106,7 @@ typedef uint32_t        ms_uint32;
 /*forward declaration of rendering object*/
 typedef struct rendererVTableObj rendererVTableObj;
 typedef struct tileCacheObj tileCacheObj;
+typedef struct outputFormatObj outputFormatObj;
 
 
 /* ms_bitarray is used by the bit mask in mapbit.c */
@@ -752,7 +753,7 @@ extern "C" {
   /*      see mapoutput.c for most related code.                          */
   /************************************************************************/
 
-  typedef struct {
+  typedef struct outputFormatObj {
     char *name;
     char *mimetype;
     char *driver;
@@ -769,7 +770,7 @@ extern "C" {
     rendererVTableObj *vtable;
     void *device; /* for supporting direct rendering onto a device context */
 #endif
-  } outputFormatObj;
+  };
 
   /* The following is used for "don't care" values in transparent, interlace and
      imagequality values. */
@@ -2067,7 +2068,7 @@ extern "C" {
   MS_DLL_EXPORT symbolObj *msRemoveSymbol(symbolSetObj *symbolset, int index);
   MS_DLL_EXPORT int msSaveSymbolSet(symbolSetObj *symbolset, const char *filename);
   MS_DLL_EXPORT int msLoadImageSymbol(symbolObj *symbol, const char *filename);
-  MS_DLL_EXPORT int msPreloadImageSymbol(rendererVTableObj *renderer, symbolObj *symbol);
+  MS_DLL_EXPORT int msPreloadImageSymbol(outputFormatObj *format, symbolObj *symbol);
   MS_DLL_EXPORT int msPreloadSVGSymbol(symbolObj *symbol);
   MS_DLL_EXPORT symbolObj *msRotateSymbol(symbolObj *symbol, double angle);
 
diff --git a/mapsymbol.c b/mapsymbol.c
index 0d0ddb5815..bb17d8c74b 100644
--- a/mapsymbol.c
+++ b/mapsymbol.c
@@ -111,7 +111,7 @@ void initSymbol(symbolObj *s)
   s->sizey = 1;
   s->filled = MS_FALSE;
   s->numpoints=0;
-  s->renderer=NULL;
+  s->format=NULL;
   s->renderer_cache = NULL;
   s->pixmap_buffer=NULL;
   s->imagepath = NULL;
@@ -136,8 +136,12 @@ int msFreeSymbol(symbolObj *s)
   }
 
   if(s->name) free(s->name);
-  if(s->renderer!=NULL) {
-    s->renderer->freeSymbol(s);
+  if(s->format!=NULL) {
+    if (s->format->vtable!=NULL)
+      s->format->vtable->freeSymbol(s);
+    /* clean up output format */
+    if( --s->format->refcount < 1 )
+      msFreeOutputFormat( s->format );
   }
   if(s->pixmap_buffer) {
     msFreeRasterBuffer(s->pixmap_buffer);
@@ -624,7 +628,7 @@ int msGetMarkerSize(symbolSetObj *symbolset, styleObj *style, double *width, dou
 
   symbol = symbolset->symbol[style->symbol];
   if (symbol->type == MS_SYMBOL_PIXMAP && !symbol->pixmap_buffer) {
-    if (MS_SUCCESS != msPreloadImageSymbol(MS_MAP_RENDERER(symbolset->map), symbol))
+    if (MS_SUCCESS != msPreloadImageSymbol(symbolset->map->outputformat, symbol))
       return MS_FAILURE;
   }
   if(symbol->type == MS_SYMBOL_SVG && !symbol->renderer_cache) {
@@ -822,22 +826,28 @@ int msLoadImageSymbol(symbolObj *symbol, const char *filename)
   return MS_SUCCESS;
 }
 
-int msPreloadImageSymbol(rendererVTableObj *renderer, symbolObj *symbol)
+int msPreloadImageSymbol(outputFormatObj *format, symbolObj *symbol)
 {
-  if(symbol->pixmap_buffer && symbol->renderer == renderer)
+  if(symbol->pixmap_buffer && symbol->format == format)
     return MS_SUCCESS;
   if(symbol->pixmap_buffer) { /* other renderer was used, start again */
     msFreeRasterBuffer(symbol->pixmap_buffer);
   } else {
     symbol->pixmap_buffer = (rasterBufferObj*)calloc(1,sizeof(rasterBufferObj));
   }
-  if(MS_SUCCESS != renderer->loadImageFromFile(symbol->full_pixmap_path, symbol->pixmap_buffer)) {
+  if(MS_SUCCESS != format->vtable->loadImageFromFile(symbol->full_pixmap_path, symbol->pixmap_buffer)) {
     /* Free pixmap_buffer already allocated */
     free(symbol->pixmap_buffer);
     symbol->pixmap_buffer = NULL;
     return MS_FAILURE;
   }
-  symbol->renderer = renderer;
+  if (symbol->format != format) {
+    /* clean up previous output format */
+    if( symbol->format && --symbol->format->refcount < 1 )
+      msFreeOutputFormat( symbol->format );
+    symbol->format = format;
+    symbol->format->refcount++;
+  }
   symbol->sizex = symbol->pixmap_buffer->width;
   symbol->sizey = symbol->pixmap_buffer->height;
   return MS_SUCCESS;
diff --git a/mapsymbol.h b/mapsymbol.h
index 45ed2965c3..f7baf77aa6 100644
--- a/mapsymbol.h
+++ b/mapsymbol.h
@@ -191,7 +191,7 @@ typedef struct {
   ** MS_SYMBOL_PIXMAP options
   */
 #ifndef SWIG
-  rendererVTableObj *renderer;
+  outputFormatObj *format;
   rasterBufferObj *pixmap_buffer;
   void *renderer_cache;
   char *full_font_path;