diff --git a/holoviews/plotting/bokeh/element.py b/holoviews/plotting/bokeh/element.py
index 97c10bc1d9..2a96f68d38 100644
--- a/holoviews/plotting/bokeh/element.py
+++ b/holoviews/plotting/bokeh/element.py
@@ -575,10 +575,20 @@ def current_handles(self):
         handles = []
         if self.static and not self.dynamic:
             return handles
+
+        previous_id = self.handles.get('previous_id', None)
+        current_id = id(self.current_frame)
         for handle in self._update_handles:
+            if (handle == 'source' and self.dynamic and
+                current_id == previous_id):
+                continue
             if handle in self.handles:
                 handles.append(self.handles[handle])
 
+        # Cache frame object id to skip updating if unchanged
+        if self.dynamic:
+            self.handles['previous_id'] = current_id
+
         if self.overlaid:
             return handles
 
diff --git a/holoviews/plotting/bokeh/util.py b/holoviews/plotting/bokeh/util.py
index 432a8d64f3..f8e583ae9a 100644
--- a/holoviews/plotting/bokeh/util.py
+++ b/holoviews/plotting/bokeh/util.py
@@ -45,7 +45,7 @@
                   'FixedTicker', 'FuncTickFormatter', 'LogTickFormatter']
 
 # List of attributes that can safely be dropped from the references
-IGNORED_ATTRIBUTES = ['data', 'palette']
+IGNORED_ATTRIBUTES = ['data', 'palette', 'image', 'x', 'y']
 
 # Model priority order to ensure some types are updated before others
 MODEL_PRIORITY = ['Range1d', 'Title', 'Image', 'LinearColorMapper',
@@ -189,7 +189,7 @@ def get_ids(obj):
         ids = [get_ids(o) for o in obj]
     elif isinstance(obj, dict):
         ids = [(v,) if k == 'id' else get_ids(v)
-               for k, v in obj.items()]
+               for k, v in obj.items() if not k in IGNORED_ATTRIBUTES]
     return list(itertools.chain(*ids))
 
 
@@ -239,7 +239,8 @@ def compute_static_patch(document, models):
             update_types[obj['type']].append(key)
     events = [delete_refs(e, IGNORED_MODELS)
               for _, e in sorted(events, key=lambda x: x[0])]
-    events = [e for e in events if all(i in requested_updates for i in get_ids(e))]
+    events = [e for e in events if all(i in requested_updates for i in get_ids(e))
+              if 'new' in e]
     value_refs = {ref_id: delete_refs(val, IGNORED_MODELS, IGNORED_ATTRIBUTES)
                   for ref_id, val in value_refs.items()}
     references = [val for val in value_refs.values()