update

tensorpack/callbacks/prof.py

@@ -146,12 +146,12 @@ def worker(evt, rst_queue, stop_evt, devices):
                         if evt.is_set():    # stop epoch
                             if stop_evt.is_set():   # or on exit
                                 return
-                            evt.clear()
                             if cnt > 1:
                                 # Ignore the last datapoint. Usually is zero, makes us underestimate the util.
                                 stats -= data
                                 cnt -= 1
                             rst_queue.put(stats / cnt)
+                            evt.clear()
                             break
         except Exception:
             logger.exception("Exception in GPUUtilizationTracker.worker")

tensorpack/tfutils/varreplace.py

@@ -4,6 +4,7 @@
 
 from contextlib import contextmanager
 
+from ..utils import logger
 from ..compat import tfv1 as tf
 from .common import get_tf_version_tuple
 
@@ -113,7 +114,15 @@ def custom_getter(getter, *args, **kwargs):
         # do not perform unnecessary changes if it's not originally trainable
         # otherwise the variable may get added to MODEL_VARIABLES twice
         if trainable and skip_collection:
-            tf.add_to_collection(tf.GraphKeys.MODEL_VARIABLES, v)
+            if isinstance(v, tf.Variable):
+                tf.add_to_collection(tf.GraphKeys.MODEL_VARIABLES, v)
+            else:
+                logger.warning("""
+[freeze_variables] variable getter did not return a Variable, but '{}' instead, likely due to
+another custom getter. freeze_variables() work only if the other custom getter respects the
+`trainable` argument and don't put variables with `trainable=False` into TRAINABLE_VARIABLES
+collection. Please double check if this is true for the custom getter.
+""".format(str(v)).replace("\n", ""))
         if trainable and stop_gradient:
             v = tf.stop_gradient(v, name='freezed_' + name)
         return v