{"filename":"tensorflow/job.go:94","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob pets-train-dist is created.","time":"2019-09-25T17:58:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:81","job":"kubeflow.pets-train-dist","level":"info","msg":"Need to create new pod: chief-0","replica-type":"chief","time":"2019-09-25T17:58:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:179","job":"kubeflow.pets-train-dist","level":"warning","msg":"Restart policy in pod template will be overwritten by restart policy in replica spec","replica-type":"chief","time":"2019-09-25T17:58:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Warning' reason: 'SettedPodTemplateRestartPolicy' Restart policy in pod template will be overwritten by restart policy in replica spec","time":"2019-09-25T17:58:19Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=0, failed=0","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/service.go:58","job":"kubeflow.pets-train-dist","level":"info","msg":"need to create new service: chief-0","replica-type":"chief","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreatePod' Created pod: pets-train-dist-chief-0","time":"2019-09-25T17:58:20Z"}
{"filename":"control/service_control.go:115","level":"info","msg":"Controller pets-train-dist created service pets-train-dist-chief-0","time":"2019-09-25T17:58:20Z"}
{"filename":"tensorflow/pod.go:81","job":"kubeflow.pets-train-dist","level":"info","msg":"Need to create new pod: evaluator-0","replica-type":"evaluator","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:179","job":"kubeflow.pets-train-dist","level":"warning","msg":"Restart policy in pod template will be overwritten by restart policy in replica spec","replica-type":"evaluator","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreateService' Created service: pets-train-dist-chief-0","time":"2019-09-25T17:58:20Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Warning' reason: 'SettedPodTemplateRestartPolicy' Restart policy in pod template will be overwritten by restart policy in replica spec","time":"2019-09-25T17:58:20Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=0, failed=0","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/service.go:58","job":"kubeflow.pets-train-dist","level":"info","msg":"need to create new service: evaluator-0","replica-type":"evaluator","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreatePod' Created pod: pets-train-dist-evaluator-0","time":"2019-09-25T17:58:20Z"}
{"filename":"control/service_control.go:115","level":"info","msg":"Controller pets-train-dist created service pets-train-dist-evaluator-0","time":"2019-09-25T17:58:20Z"}
{"filename":"tensorflow/pod.go:81","job":"kubeflow.pets-train-dist","level":"info","msg":"Need to create new pod: ps-0","replica-type":"ps","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:179","job":"kubeflow.pets-train-dist","level":"warning","msg":"Restart policy in pod template will be overwritten by restart policy in replica spec","replica-type":"ps","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreateService' Created service: pets-train-dist-evaluator-0","time":"2019-09-25T17:58:20Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Warning' reason: 'SettedPodTemplateRestartPolicy' Restart policy in pod template will be overwritten by restart policy in replica spec","time":"2019-09-25T17:58:20Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=0, failed=0","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/service.go:58","job":"kubeflow.pets-train-dist","level":"info","msg":"need to create new service: ps-0","replica-type":"ps","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreatePod' Created pod: pets-train-dist-ps-0","time":"2019-09-25T17:58:20Z"}
{"filename":"control/service_control.go:115","level":"info","msg":"Controller pets-train-dist created service pets-train-dist-ps-0","time":"2019-09-25T17:58:20Z"}
{"filename":"tensorflow/pod.go:81","job":"kubeflow.pets-train-dist","level":"info","msg":"Need to create new pod: worker-0","replica-type":"worker","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:179","job":"kubeflow.pets-train-dist","level":"warning","msg":"Restart policy in pod template will be overwritten by restart policy in replica spec","replica-type":"worker","time":"2019-09-25T17:58:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreateService' Created service: pets-train-dist-ps-0","time":"2019-09-25T17:58:20Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Warning' reason: 'SettedPodTemplateRestartPolicy' Restart policy in pod template will be overwritten by restart policy in replica spec","time":"2019-09-25T17:58:20Z"}
{"filename":"tensorflow/pod.go:81","job":"kubeflow.pets-train-dist","level":"info","msg":"Need to create new pod: worker-1","replica-type":"worker","time":"2019-09-25T17:58:21Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreatePod' Created pod: pets-train-dist-worker-0","time":"2019-09-25T17:58:21Z"}
{"filename":"tensorflow/pod.go:179","job":"kubeflow.pets-train-dist","level":"warning","msg":"Restart policy in pod template will be overwritten by restart policy in replica spec","replica-type":"worker","time":"2019-09-25T17:58:21Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Warning' reason: 'SettedPodTemplateRestartPolicy' Restart policy in pod template will be overwritten by restart policy in replica spec","time":"2019-09-25T17:58:21Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T17:58:21Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/service.go:58","job":"kubeflow.pets-train-dist","level":"info","msg":"need to create new service: worker-0","replica-type":"worker","time":"2019-09-25T17:58:21Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreatePod' Created pod: pets-train-dist-worker-1","time":"2019-09-25T17:58:21Z"}
{"filename":"control/service_control.go:115","level":"info","msg":"Controller pets-train-dist created service pets-train-dist-worker-0","time":"2019-09-25T17:58:21Z"}
{"filename":"tensorflow/service.go:58","job":"kubeflow.pets-train-dist","level":"info","msg":"need to create new service: worker-1","replica-type":"worker","time":"2019-09-25T17:58:21Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreateService' Created service: pets-train-dist-worker-0","time":"2019-09-25T17:58:21Z"}
{"filename":"control/service_control.go:115","level":"info","msg":"Controller pets-train-dist created service pets-train-dist-worker-1","time":"2019-09-25T17:58:22Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"43151\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreateService' Created service: pets-train-dist-worker-1","time":"2019-09-25T17:58:22Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (2.256621871s)","time":"2019-09-25T17:58:22Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T17:58:22Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=0, failed=0","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=0, failed=0","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=0, failed=0","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (697.447µs)","time":"2019-09-25T17:58:22Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=0, failed=0","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=0, failed=0","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=0, failed=0","time":"2019-09-25T17:58:22Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (634.137µs)","time":"2019-09-25T17:58:22Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=0, failed=0","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=0, failed=0","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (11.745064ms)","time":"2019-09-25T17:58:25Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T17:58:25Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=0, failed=0","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=0, failed=0","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T17:58:25Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (793.895µs)","time":"2019-09-25T17:58:25Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=0, failed=0","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (11.709728ms)","time":"2019-09-25T17:58:32Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T17:58:32Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=0, failed=0","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:32Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.673787ms)","time":"2019-09-25T17:58:32Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (14.616416ms)","time":"2019-09-25T17:58:33Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T17:58:33Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T17:58:33Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (884.398µs)","time":"2019-09-25T17:58:33Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (15.289758ms)","time":"2019-09-25T17:58:34Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T17:58:34Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (999.745µs)","time":"2019-09-25T17:58:34Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T17:58:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T17:58:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:58:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T17:58:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:58:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:58:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.07306ms)","time":"2019-09-25T17:58:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T17:58:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T17:58:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T17:58:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T17:58:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:58:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T17:58:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T17:58:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:58:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T17:58:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T17:58:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T17:58:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T17:58:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T17:58:44Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.115996ms)","time":"2019-09-25T17:58:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T17:58:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T17:59:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T17:59:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:59:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:59:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:59:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T17:59:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:59:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (888.224µs)","time":"2019-09-25T17:59:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T17:59:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T17:59:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T17:59:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T17:59:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T17:59:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T17:59:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T17:59:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:59:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T17:59:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T17:59:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T17:59:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:59:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T17:59:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T17:59:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.38423ms)","time":"2019-09-25T17:59:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:59:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:59:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:59:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:59:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T17:59:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (898.147µs)","time":"2019-09-25T17:59:29Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T17:59:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T17:59:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:59:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:59:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:59:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:59:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T17:59:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.037125ms)","time":"2019-09-25T17:59:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T17:59:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T17:59:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T17:59:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T17:59:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T17:59:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T17:59:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:59:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T17:59:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T17:59:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T17:59:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T17:59:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T17:59:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T17:59:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:59:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.519536ms)","time":"2019-09-25T17:59:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:59:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:59:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:59:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:59:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T17:59:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (978.939µs)","time":"2019-09-25T17:59:49Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:59:50Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:59:50Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:59:50Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:59:50Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T17:59:50Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.036517ms)","time":"2019-09-25T17:59:50Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T17:59:55Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T17:59:55Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T17:59:55Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T17:59:55Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T17:59:55Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (894.75µs)","time":"2019-09-25T17:59:55Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:00:11Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:00:11Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:00:11Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:00:11Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:00:11Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.341595ms)","time":"2019-09-25T18:00:11Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:00:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:00:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:00:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:00:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:00:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:00:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:00:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.242124ms)","time":"2019-09-25T18:00:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:00:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:00:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:00:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:00:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:00:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:00:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:00:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:00:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:00:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:00:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:00:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:00:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:00:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.433358ms)","time":"2019-09-25T18:00:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:00:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:00:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:00:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:00:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:00:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:00:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:00:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:00:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.224952ms)","time":"2019-09-25T18:00:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:00:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:00:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:00:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:00:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:00:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:00:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:00:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:00:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:00:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:00:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:00:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:00:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:00:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:00:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.467622ms)","time":"2019-09-25T18:00:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:01:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:01:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:01:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:01:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:01:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:01:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:01:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.109251ms)","time":"2019-09-25T18:01:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:01:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:01:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:01:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:01:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:01:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:01:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:01:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:01:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:01:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:01:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:01:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:01:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:01:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.494629ms)","time":"2019-09-25T18:01:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:01:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:01:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:01:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:01:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:01:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:01:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (898.52µs)","time":"2019-09-25T18:01:29Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.303368ms)","time":"2019-09-25T18:01:30Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:01:30Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (985.532µs)","time":"2019-09-25T18:01:30Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:01:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:01:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:01:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:01:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:01:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:01:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:01:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:01:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:01:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:01:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:01:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:01:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:01:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:01:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:01:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:01:44Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.604904ms)","time":"2019-09-25T18:01:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:01:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:01:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:01:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:01:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:01:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (778.059µs)","time":"2019-09-25T18:01:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:02:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:02:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:02:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:02:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:02:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:02:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:02:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.117045ms)","time":"2019-09-25T18:02:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:02:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:02:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:02:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:02:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:02:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:02:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:02:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:02:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:02:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:02:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:02:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:02:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:02:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.476956ms)","time":"2019-09-25T18:02:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:02:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:02:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:02:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:02:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:02:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:02:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:02:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:02:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.277117ms)","time":"2019-09-25T18:02:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:02:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:02:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:02:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:02:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:02:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:02:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:02:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:02:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:02:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:02:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:02:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.509776ms)","time":"2019-09-25T18:02:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:02:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:02:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:02:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:03:04Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (21.663399ms)","time":"2019-09-25T18:03:04Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:03:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.261772ms)","time":"2019-09-25T18:03:04Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:03:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:03:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:03:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:03:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:03:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:03:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:03:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:03:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:03:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:03:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:03:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:03:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:03:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.683478ms)","time":"2019-09-25T18:03:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:03:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:03:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:03:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:03:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:03:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:03:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:03:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:03:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (874.45µs)","time":"2019-09-25T18:03:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:03:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:03:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:03:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:03:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:03:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:03:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:03:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:03:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:03:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:03:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:03:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:03:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:03:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:03:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:03:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:03:44Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.621584ms)","time":"2019-09-25T18:03:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:03:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:03:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:03:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:03:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:03:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.161381ms)","time":"2019-09-25T18:03:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:04:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:04:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:04:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:04:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:04:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:04:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:04:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (930.932µs)","time":"2019-09-25T18:04:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:04:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:04:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:04:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:04:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:04:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:04:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:04:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:04:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:04:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:04:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:04:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:04:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.325087ms)","time":"2019-09-25T18:04:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:04:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:04:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:04:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:04:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:04:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:04:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:04:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:04:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:04:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:04:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:04:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:04:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:04:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:04:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:04:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:04:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:04:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.86432ms)","time":"2019-09-25T18:04:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:04:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:04:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:04:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:04:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:04:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:04:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.870183ms)","time":"2019-09-25T18:04:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:05:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:05:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:05:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:05:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:05:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:05:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:05:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.198488ms)","time":"2019-09-25T18:05:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:05:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:05:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:05:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:05:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:05:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:05:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:05:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:05:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:05:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:05:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:05:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:05:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:05:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:05:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.601268ms)","time":"2019-09-25T18:05:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:05:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:05:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:05:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:05:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:05:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:05:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:05:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (994.436µs)","time":"2019-09-25T18:05:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:05:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:05:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:05:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:05:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:05:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:05:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:05:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:05:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:05:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:05:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:05:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:05:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:05:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:05:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.630803ms)","time":"2019-09-25T18:05:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:06:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:06:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:06:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:06:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:06:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:06:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:06:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:06:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:06:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:06:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:06:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:06:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:06:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:06:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.650112ms)","time":"2019-09-25T18:06:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:06:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:06:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:06:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:06:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:06:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:06:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:06:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.063857ms)","time":"2019-09-25T18:06:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:06:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:06:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:06:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:06:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:06:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:06:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:06:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.320599ms)","time":"2019-09-25T18:06:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:06:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:06:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:06:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:06:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:06:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:06:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:06:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:06:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:06:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:06:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:06:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:06:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (2.111363ms)","time":"2019-09-25T18:06:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:06:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:06:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:07:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:07:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:07:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:07:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:07:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:07:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:07:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.437017ms)","time":"2019-09-25T18:07:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:07:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:07:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:07:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:07:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:07:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:07:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:07:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:07:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:07:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:07:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:07:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:07:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:07:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.933715ms)","time":"2019-09-25T18:07:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:07:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:07:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:07:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:07:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:07:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:07:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:07:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:07:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.207987ms)","time":"2019-09-25T18:07:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:07:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:07:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:07:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:07:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:07:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:07:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:07:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:07:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:07:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:07:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:07:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:07:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:07:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.593441ms)","time":"2019-09-25T18:07:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:07:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:08:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:08:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:08:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:08:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:08:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:08:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:08:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.164191ms)","time":"2019-09-25T18:08:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:08:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:08:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:08:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:08:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:08:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:08:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:08:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:08:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:08:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:08:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:08:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:08:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:08:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:08:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (3.932162ms)","time":"2019-09-25T18:08:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:08:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:08:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:08:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:08:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:08:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:08:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:08:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.19847ms)","time":"2019-09-25T18:08:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:08:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:08:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:08:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:08:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:08:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:08:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:08:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:08:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:08:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:08:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:08:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:08:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:08:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.619388ms)","time":"2019-09-25T18:08:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:08:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:09:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:09:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:09:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:09:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:09:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:09:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:09:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:09:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:09:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:09:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:09:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:09:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:09:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:09:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:09:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.920669ms)","time":"2019-09-25T18:09:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:09:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:09:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:09:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:09:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:09:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:09:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.114671ms)","time":"2019-09-25T18:09:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:09:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:09:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:09:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:09:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:09:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:09:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:09:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.113627ms)","time":"2019-09-25T18:09:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:09:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:09:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:09:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:09:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:09:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:09:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:09:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:09:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:09:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:09:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:09:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:09:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:09:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (3.566524ms)","time":"2019-09-25T18:09:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:09:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:10:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:10:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:10:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:10:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:10:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:10:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:10:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:10:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:10:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:10:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:10:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:10:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:10:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:10:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:10:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.595501ms)","time":"2019-09-25T18:10:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:10:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:10:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:10:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:10:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:10:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:10:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (985.606µs)","time":"2019-09-25T18:10:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:10:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:10:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:10:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:10:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:10:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:10:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:10:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.896627ms)","time":"2019-09-25T18:10:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:10:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:10:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:10:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:10:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:10:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:10:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:10:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:10:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:10:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:10:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:10:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.291983ms)","time":"2019-09-25T18:10:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:10:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:10:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:10:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:11:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:11:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:11:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:11:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:11:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:11:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:11:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.509363ms)","time":"2019-09-25T18:11:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:11:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:11:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:11:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:11:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:11:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:11:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:11:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:11:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:11:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:11:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:11:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.435995ms)","time":"2019-09-25T18:11:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:11:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:11:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:11:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:11:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:11:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:11:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:11:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:11:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:11:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:11:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (993.404µs)","time":"2019-09-25T18:11:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:11:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:11:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:11:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:11:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:11:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:11:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:11:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:11:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:11:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:11:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:11:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.196193ms)","time":"2019-09-25T18:11:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:11:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:11:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:11:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:12:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:12:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:12:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:12:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:12:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:12:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:12:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.046335ms)","time":"2019-09-25T18:12:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:12:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:12:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:12:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:12:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:12:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:12:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:12:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:12:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:12:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:12:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:12:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:12:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:12:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:12:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.346255ms)","time":"2019-09-25T18:12:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:12:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:12:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:12:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:12:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:12:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:12:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:12:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:12:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:12:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:12:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:12:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:12:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:12:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.695949ms)","time":"2019-09-25T18:12:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:12:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:12:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:12:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:12:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:12:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:12:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:12:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:12:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.673525ms)","time":"2019-09-25T18:12:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:13:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:13:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:13:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:13:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:13:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:13:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:13:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:13:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:13:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:13:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:13:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:13:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:13:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:13:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:13:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (2.035721ms)","time":"2019-09-25T18:13:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:13:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:13:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:13:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:13:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:13:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:13:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.317747ms)","time":"2019-09-25T18:13:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:13:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:13:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:13:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:13:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:13:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:13:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:13:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (2.848375ms)","time":"2019-09-25T18:13:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:13:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:13:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:13:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:13:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:13:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:13:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:13:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:13:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:13:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:13:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:13:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (2.405344ms)","time":"2019-09-25T18:13:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:13:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:13:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:13:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:00Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=1, running=1, failed=0","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"44940\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:14:00Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (20.381793ms)","time":"2019-09-25T18:14:00Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:14:00Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:00Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=1, running=1, failed=0","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:14:00Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.308996ms)","time":"2019-09-25T18:14:00Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48780\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:14:00Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:02Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:02Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48780\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:14:02Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48780\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:14:02Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (21.586095ms)","time":"2019-09-25T18:14:02Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:14:02Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:02Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:02Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:14:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.707153ms)","time":"2019-09-25T18:14:02Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:14:02Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:14:02Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:14:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:14:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:14:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:14:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:14:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:14:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:14:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.36531ms)","time":"2019-09-25T18:14:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:14:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:14:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:14:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:14:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:14:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:14:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:14:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:14:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:14:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:14:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:14:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.307125ms)","time":"2019-09-25T18:14:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:14:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:14:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:14:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:14:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:14:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:14:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:14:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:14:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:14:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:14:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:14:44Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.433516ms)","time":"2019-09-25T18:14:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:14:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:14:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:14:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:14:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:14:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:14:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:14:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:14:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:14:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.315484ms)","time":"2019-09-25T18:14:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:15:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:15:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:15:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:15:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:15:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:15:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:15:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:15:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:15:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (2.411531ms)","time":"2019-09-25T18:15:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:15:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:15:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:15:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:15:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:15:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:15:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:15:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:15:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:15:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:15:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:15:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:15:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.65312ms)","time":"2019-09-25T18:15:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:15:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:15:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:15:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:15:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:15:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:15:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:15:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:15:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:15:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:15:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:15:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:15:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:15:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:15:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.688476ms)","time":"2019-09-25T18:15:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:15:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:15:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:15:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:15:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:15:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:15:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:15:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:15:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.115105ms)","time":"2019-09-25T18:15:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:15:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:15:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:16:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:16:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:16:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:16:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:16:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:16:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:16:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:16:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:16:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:16:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:16:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.227006ms)","time":"2019-09-25T18:16:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:16:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:16:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:16:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:16:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:16:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:16:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:16:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:16:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:16:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:16:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:16:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.449826ms)","time":"2019-09-25T18:16:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:16:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:16:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:16:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:16:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:16:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:16:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:16:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:16:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:16:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.192821ms)","time":"2019-09-25T18:16:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:16:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:16:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:16:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:16:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:16:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:16:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:16:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:16:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:16:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:16:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:16:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:16:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.568047ms)","time":"2019-09-25T18:16:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:16:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:16:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:17:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:17:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:17:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:17:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:17:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:17:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:17:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:17:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:17:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:17:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:17:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.501688ms)","time":"2019-09-25T18:17:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:17:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:17:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:17:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:17:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:17:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:17:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:17:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:17:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:17:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:17:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:17:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.150843ms)","time":"2019-09-25T18:17:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:17:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:17:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:17:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:17:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:17:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:17:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:17:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:17:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:17:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:17:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:17:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:17:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.539473ms)","time":"2019-09-25T18:17:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:17:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:17:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:17:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:17:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:17:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:17:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:17:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:17:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:17:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.119561ms)","time":"2019-09-25T18:17:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:17:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:17:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:18:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:18:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:18:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:18:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:18:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:18:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:18:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:18:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:18:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:18:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:18:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.293447ms)","time":"2019-09-25T18:18:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:18:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:18:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:18:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:18:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:18:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:18:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:18:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:18:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:18:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:18:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.3637ms)","time":"2019-09-25T18:18:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:18:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:18:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:18:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:18:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:18:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:18:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:18:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:18:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:18:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:18:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:18:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:18:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.438946ms)","time":"2019-09-25T18:18:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:18:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:18:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:18:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:18:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:18:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:18:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:18:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:18:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:18:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.322436ms)","time":"2019-09-25T18:18:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:18:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:18:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:18:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:19:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:19:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:19:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:19:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:19:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:19:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:19:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:19:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:19:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:19:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:19:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.326513ms)","time":"2019-09-25T18:19:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:19:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:19:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:19:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:19:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:19:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:19:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:19:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:19:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:19:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:19:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:19:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.393474ms)","time":"2019-09-25T18:19:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:19:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:19:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:19:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:19:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:19:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:19:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:19:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:19:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:19:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:19:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:19:44Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.35497ms)","time":"2019-09-25T18:19:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:19:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:19:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:19:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:19:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:19:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:19:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:19:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:19:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:19:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:19:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:19:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.677983ms)","time":"2019-09-25T18:19:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:19:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:20:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:20:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:20:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:20:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:20:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:20:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:20:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:20:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:20:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.413987ms)","time":"2019-09-25T18:20:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:20:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:20:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:20:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:20:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:20:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:20:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:20:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:20:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:20:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:20:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:20:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:20:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.497862ms)","time":"2019-09-25T18:20:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:20:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:20:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:20:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:20:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:20:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:20:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:20:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:20:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:20:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:20:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.258415ms)","time":"2019-09-25T18:20:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:20:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:20:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:20:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:20:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:20:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:20:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:20:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:20:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:20:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:20:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:20:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:20:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:20:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:20:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.680815ms)","time":"2019-09-25T18:20:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:21:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:21:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:21:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:21:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:21:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:21:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:21:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:21:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:21:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.484699ms)","time":"2019-09-25T18:21:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:21:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:21:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:21:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:21:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:21:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:21:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:21:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:21:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:21:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:21:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:21:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:21:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:21:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.464527ms)","time":"2019-09-25T18:21:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:21:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:21:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:21:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:21:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:21:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:21:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:21:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:21:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:21:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.243736ms)","time":"2019-09-25T18:21:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:21:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:21:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:21:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:21:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:21:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:21:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:21:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:21:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:21:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:21:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:21:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:21:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:21:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:21:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.153939ms)","time":"2019-09-25T18:21:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:22:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:22:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:22:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:22:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:22:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:22:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:22:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:22:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:22:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:22:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:22:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.400997ms)","time":"2019-09-25T18:22:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:22:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:22:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:22:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:22:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:22:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:22:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:22:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:22:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.132678ms)","time":"2019-09-25T18:22:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:22:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:22:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:22:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:22:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:22:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:22:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:22:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:22:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:22:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:22:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:22:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:22:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.216926ms)","time":"2019-09-25T18:22:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:22:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:22:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:22:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:22:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:22:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:22:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:22:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:22:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:22:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:22:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:22:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:22:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:22:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.465596ms)","time":"2019-09-25T18:22:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:22:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:23:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:23:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:23:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:23:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:23:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:23:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:23:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:23:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:23:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:23:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:23:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.277718ms)","time":"2019-09-25T18:23:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:23:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:23:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:23:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:23:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:23:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:23:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:23:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:23:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.185174ms)","time":"2019-09-25T18:23:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:23:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:23:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:23:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:23:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:23:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:23:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:23:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:23:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:23:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:23:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:23:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:23:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:23:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:23:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:23:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.809503ms)","time":"2019-09-25T18:23:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:23:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:23:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:23:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:23:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:23:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:23:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:23:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:23:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.068844ms)","time":"2019-09-25T18:23:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:23:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:23:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:23:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:24:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:24:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:24:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:24:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:24:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:24:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:24:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:24:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:24:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:24:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:24:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.42487ms)","time":"2019-09-25T18:24:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:24:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:24:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:24:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:24:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:24:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:24:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:24:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:24:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:24:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:24:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:24:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.192756ms)","time":"2019-09-25T18:24:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:24:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:24:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:24:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:24:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:24:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:24:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:24:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:24:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:24:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:24:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:24:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.696904ms)","time":"2019-09-25T18:24:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:24:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:24:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:24:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:24:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:24:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:24:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:24:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:24:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:24:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.110778ms)","time":"2019-09-25T18:24:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:24:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:24:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:24:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:25:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:25:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:25:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:25:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:25:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.273526ms)","time":"2019-09-25T18:25:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:25:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:25:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:25:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:25:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.162279ms)","time":"2019-09-25T18:25:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:25:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:25:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:25:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time 2019-09-25 18:25:19 +0000 UTC","time":"2019-09-25T18:25:19Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:19Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.462114ms)","time":"2019-09-25T18:25:19Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:25:19Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:19Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/pod.go:81","job":"kubeflow.lice-train-dist","level":"info","msg":"Need to create new pod: worker-0","replica-type":"worker","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:179","job":"kubeflow.lice-train-dist","level":"warning","msg":"Restart policy in pod template will be overwritten by restart policy in replica spec","replica-type":"worker","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Warning' reason: 'SettedPodTemplateRestartPolicy' Restart policy in pod template will be overwritten by restart policy in replica spec","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreatePod' Created pod: lice-train-dist-worker-0","time":"2019-09-25T18:25:19Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:19Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"41643\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (517.405279ms)","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:81","job":"kubeflow.pets-train-dist","level":"info","msg":"Need to create new pod: worker-0","replica-type":"worker","time":"2019-09-25T18:25:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:179","job":"kubeflow.pets-train-dist","level":"warning","msg":"Restart policy in pod template will be overwritten by restart policy in replica spec","replica-type":"worker","time":"2019-09-25T18:25:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Warning' reason: 'SettedPodTemplateRestartPolicy' Restart policy in pod template will be overwritten by restart policy in replica spec","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/pod.go:81","job":"kubeflow.pets-train-dist","level":"info","msg":"Need to create new pod: worker-1","replica-type":"worker","time":"2019-09-25T18:25:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:179","job":"kubeflow.pets-train-dist","level":"warning","msg":"Restart policy in pod template will be overwritten by restart policy in replica spec","replica-type":"worker","time":"2019-09-25T18:25:19Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreatePod' Created pod: pets-train-dist-worker-0","time":"2019-09-25T18:25:19Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Warning' reason: 'SettedPodTemplateRestartPolicy' Restart policy in pod template will be overwritten by restart policy in replica spec","time":"2019-09-25T18:25:19Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"48800\", FieldPath:\"\"}): type: 'Normal' reason: 'SuccessfulCreatePod' Created pod: pets-train-dist-worker-1","time":"2019-09-25T18:25:20Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (477.08571ms)","time":"2019-09-25T18:25:20Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:25:20Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:20Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:20Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:25:20Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:25:20Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:20Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:25:20Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.108657ms)","time":"2019-09-25T18:25:20Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:25:20Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:25:20Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:25:20Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (11.47208ms)","time":"2019-09-25T18:25:20Z"}
E0925 18:25:20.025967       1 controller.go:269] error syncing tfjob: Operation cannot be fulfilled on tfjobs.kubeflow.org "pets-train-dist": the object has been modified; please apply your changes to the latest version and try again
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (860.333µs)","time":"2019-09-25T18:25:20Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:20Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (880.943µs)","time":"2019-09-25T18:25:20Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:25:26Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:26Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:26Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:26Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:25:26Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:26Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:25:26Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:25:26Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:25:26Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:25:26Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:26Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.424415ms)","time":"2019-09-25T18:25:26Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:26Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:25:26Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:26Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:26Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:26Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.128812ms)","time":"2019-09-25T18:25:26Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:25:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (839.487µs)","time":"2019-09-25T18:25:27Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:25:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:34Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:34Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:25:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:25:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:25:34Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:25:34Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:25:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.372988ms)","time":"2019-09-25T18:25:34Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.041445ms)","time":"2019-09-25T18:25:34Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:25:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (845.029µs)","time":"2019-09-25T18:25:34Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:25:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:25:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:25:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:25:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:25:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (970.413µs)","time":"2019-09-25T18:25:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:25:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:25:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:25:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:25:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:25:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:25:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:25:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:25:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.017277ms)","time":"2019-09-25T18:25:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:25:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:25:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:26:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:26:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.003292ms)","time":"2019-09-25T18:26:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:26:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:26:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:26:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:26:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:26:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:26:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:26:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.213533ms)","time":"2019-09-25T18:26:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (920.202µs)","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (940.029µs)","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.215462ms)","time":"2019-09-25T18:26:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.134966ms)","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:26:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.618641ms)","time":"2019-09-25T18:26:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:26:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:26:45Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:45Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:45Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:45Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:26:45Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:26:45Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:45Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:26:45Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:26:45Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.463077ms)","time":"2019-09-25T18:26:45Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:26:45Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:26:45Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (956.638µs)","time":"2019-09-25T18:26:45Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:45Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.970899ms)","time":"2019-09-25T18:26:45Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (950.942µs)","time":"2019-09-25T18:26:48Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (829.713µs)","time":"2019-09-25T18:26:48Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:26:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:48Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:48Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:26:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:26:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:26:48Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:26:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:26:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:26:48Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.273892ms)","time":"2019-09-25T18:26:48Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (928.751µs)","time":"2019-09-25T18:26:49Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:49Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (978.678µs)","time":"2019-09-25T18:26:49Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:26:49Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:49Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:49Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:26:49Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:26:49Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:49Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:26:49Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:26:49Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:49Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:26:49Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:26:49Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.211377ms)","time":"2019-09-25T18:26:49Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.025918ms)","time":"2019-09-25T18:26:57Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:26:57Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:57Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:26:57Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:26:57Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:26:57Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:57Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:26:57Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:26:57Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:26:57Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:26:57Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:57Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.498766ms)","time":"2019-09-25T18:26:57Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:26:57Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (864.035µs)","time":"2019-09-25T18:26:57Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (940.821µs)","time":"2019-09-25T18:27:02Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:02Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:02Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:02Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:02Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:02Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:02Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:02Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:02Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:02Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:02Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.486471ms)","time":"2019-09-25T18:27:02Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:02Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:02Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (889.793µs)","time":"2019-09-25T18:27:02Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (784.284µs)","time":"2019-09-25T18:27:04Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:04Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:04Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:04Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:04Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:04Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:04Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:04Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:04Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:04Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:04Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:04Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.293496ms)","time":"2019-09-25T18:27:04Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:04Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (910.326µs)","time":"2019-09-25T18:27:04Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.146158ms)","time":"2019-09-25T18:27:06Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:06Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (926.15µs)","time":"2019-09-25T18:27:06Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:06Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:06Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:06Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:06Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:06Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:06Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:06Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:06Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:06Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.261353ms)","time":"2019-09-25T18:27:06Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:06Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:06Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:27:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:27:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.095381ms)","time":"2019-09-25T18:27:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.219894ms)","time":"2019-09-25T18:27:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (960.164µs)","time":"2019-09-25T18:27:27Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:27Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:27Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:27Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:27Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:27Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:27Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:27Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:27Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:27Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.121744ms)","time":"2019-09-25T18:27:27Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:27Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:27Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:27Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (954.267µs)","time":"2019-09-25T18:27:27Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (870.277µs)","time":"2019-09-25T18:27:28Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (877.238µs)","time":"2019-09-25T18:27:28Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:28Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:28Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:28Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:28Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.550908ms)","time":"2019-09-25T18:27:28Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:29Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:29Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:29Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:29Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:29Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:29Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:29Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:29Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:29Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:29Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:29Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.555321ms)","time":"2019-09-25T18:27:29Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.056025ms)","time":"2019-09-25T18:27:29Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:29Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (946.93µs)","time":"2019-09-25T18:27:29Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.671177ms)","time":"2019-09-25T18:27:35Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:35Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:35Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:35Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:35Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:35Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:35Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:35Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:35Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:35Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:35Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:35Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.455567ms)","time":"2019-09-25T18:27:35Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:35Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (878.152µs)","time":"2019-09-25T18:27:35Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (914.608µs)","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (902.148µs)","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.223971ms)","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.119275ms)","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.202704ms)","time":"2019-09-25T18:27:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (839.154µs)","time":"2019-09-25T18:27:48Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:27:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:48Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:27:48Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:27:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:27:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:27:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:27:48Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.167759ms)","time":"2019-09-25T18:27:48Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:27:48Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:27:48Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:27:48Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.12568ms)","time":"2019-09-25T18:27:48Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:28:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:28:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:28:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:28:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:28:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:28:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:28:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:28:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:28:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.066253ms)","time":"2019-09-25T18:28:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:28:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:28:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:28:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:28:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:28:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (698.563µs)","time":"2019-09-25T18:28:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.04668ms)","time":"2019-09-25T18:28:28Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (870.988µs)","time":"2019-09-25T18:28:28Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:28:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:28:28Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:28:28Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:28:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:28:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:28:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:28:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:28:28Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:28:28Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.209796ms)","time":"2019-09-25T18:28:28Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:28:28Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.073079ms)","time":"2019-09-25T18:28:28Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:28:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:28:34Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:28:34Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:28:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:28:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:28:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:28:34Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:28:34Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:28:34Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.65084ms)","time":"2019-09-25T18:28:34Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.071527ms)","time":"2019-09-25T18:28:34Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:34Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (928.857µs)","time":"2019-09-25T18:28:34Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:28:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:28:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:28:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:28:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:28:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.037372ms)","time":"2019-09-25T18:28:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:28:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:28:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:28:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:28:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:28:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:28:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:28:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:28:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:28:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.274154ms)","time":"2019-09-25T18:28:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:28:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:28:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:29:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:29:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:29:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:29:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:29:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:29:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:29:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (790.748µs)","time":"2019-09-25T18:29:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:29:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:29:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:29:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:29:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:29:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:29:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:29:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:29:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:29:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:29:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:29:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.040395ms)","time":"2019-09-25T18:29:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:29:18Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:29:18Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:29:18Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:29:18Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.252697ms)","time":"2019-09-25T18:29:18Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:29:18Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:29:18Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:29:18Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:29:18Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:29:18Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.176583ms)","time":"2019-09-25T18:29:18Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:29:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:29:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:29:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:29:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:29:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:29:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:29:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:29:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:29:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:29:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:29:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:29:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:29:44Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.211461ms)","time":"2019-09-25T18:29:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:29:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:29:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:29:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:29:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=0, failed=0","time":"2019-09-25T18:29:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (833.275µs)","time":"2019-09-25T18:29:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (25.664562ms)","time":"2019-09-25T18:30:03Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:30:03Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=1, failed=0","time":"2019-09-25T18:30:03Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.109599ms)","time":"2019-09-25T18:30:03Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:30:07Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (15.631019ms)","time":"2019-09-25T18:30:07Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:30:07Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (3.09449ms)","time":"2019-09-25T18:30:07Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=2, running=2, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (985.141µs)","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:30:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:30:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:30:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:30:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.17188ms)","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"54874\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (41.545669ms)","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"54923\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.270271ms)","time":"2019-09-25T18:30:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:23Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:23Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"54923\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:30:23Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"54923\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:30:23Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (18.582483ms)","time":"2019-09-25T18:30:23Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:30:23Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:23Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:23Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=1, running=1, failed=0","time":"2019-09-25T18:30:23Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (851.665µs)","time":"2019-09-25T18:30:23Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"54976\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:30:23Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"54976\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:30:23Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:40Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:40Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:40Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"54976\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:30:40Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"54976\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:30:40Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"54976\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:30:40Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (17.474452ms)","time":"2019-09-25T18:30:40Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:30:40Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:40Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:40Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:40Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:30:40Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:30:40Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (2.164627ms)","time":"2019-09-25T18:30:40Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:30:40Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:30:40Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:30:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:30:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:30:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:30:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:30:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:30:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:30:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.336618ms)","time":"2019-09-25T18:30:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:30:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:30:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:30:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:30:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:30:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=0, failed=0","time":"2019-09-25T18:30:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.503644ms)","time":"2019-09-25T18:30:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:30:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:30:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:50Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:50Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=1, failed=0","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:30:50Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"52818\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:30:50Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (14.248703ms)","time":"2019-09-25T18:30:50Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:30:50Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:50Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:30:50Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=1, running=1, failed=0","time":"2019-09-25T18:30:50Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55148\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:30:50Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55148\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:30:50Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.200965ms)","time":"2019-09-25T18:30:50Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:01Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:01Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:01Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55148\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:31:01Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55148\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:31:01Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55148\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:31:01Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (48.893394ms)","time":"2019-09-25T18:31:01Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:31:01Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:01Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:01Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:01Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:31:01Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:31:01Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.731936ms)","time":"2019-09-25T18:31:01Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:31:01Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:31:01Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:31:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:31:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:31:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:31:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:31:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:31:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:31:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:31:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.444315ms)","time":"2019-09-25T18:31:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:31:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:31:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:31:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:31:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:31:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:31:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:31:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:31:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:31:14Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:31:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.297412ms)","time":"2019-09-25T18:31:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:31:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:31:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:31:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:31:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:31:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:31:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:31:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:31:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:31:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:31:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:31:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:31:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.530406ms)","time":"2019-09-25T18:31:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:31:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:31:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:31:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:31:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:31:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:31:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:31:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:31:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:31:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:31:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.509663ms)","time":"2019-09-25T18:31:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:31:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:31:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:32:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:32:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:32:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:32:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:32:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:32:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:32:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:32:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:32:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:32:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.568782ms)","time":"2019-09-25T18:32:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:32:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:32:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:32:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:32:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:32:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:32:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:32:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:32:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:32:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:32:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:32:14Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:32:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:32:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:32:14Z"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.361163ms)","time":"2019-09-25T18:32:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:32:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:32:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:32:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:32:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:32:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:32:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:32:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:32:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:32:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:32:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:32:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:32:44Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.pets-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/pets-train-dist\" (1.586015ms)","time":"2019-09-25T18:32:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:32:44Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.lice-train-dist","level":"info","msg":"Reconcile TFJobs lice-train-dist","time":"2019-09-25T18:32:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:44Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/lice-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:32:44Z"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:32:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:32:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:32:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:32:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.lice-train-dist","level":"info","msg":"Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:32:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:32:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:32:44Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.lice-train-dist","level":"info","msg":"TFJob=lice-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:32:44Z","uid":"b00975b2-dfb9-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/controller.go:293","job":"kubeflow.lice-train-dist","level":"info","msg":"Finished syncing tfjob \"kubeflow/lice-train-dist\" (1.189429ms)","time":"2019-09-25T18:32:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:32:44Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"lice-train-dist\", UID:\"b00975b2-dfb9-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55218\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.lice-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:32:44Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: pets-train-dist","time":"2019-09-25T18:33:14Z"}
{"filename":"tensorflow/job.go:130","level":"info","msg":"Updating tfjob: lice-train-dist","time":"2019-09-25T18:33:14Z"}
{"filename":"tensorflow/controller.go:342","job":"kubeflow.pets-train-dist","level":"info","msg":"Reconcile TFJobs pets-train-dist","time":"2019-09-25T18:33:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-evaluator-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:33:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-1 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:33:14Z"}
{"filename":"k8sutil/k8sutil.go:101","level":"info","msg":"Ignoring inactive pod kubeflow/pets-train-dist-worker-0 in state Succeeded, deletion time \u003cnil\u003e","time":"2019-09-25T18:33:14Z"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=PS expected=1, running=1, failed=0","time":"2019-09-25T18:33:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","replica-type":"worker","time":"2019-09-25T18:33:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","replica-type":"worker","time":"2019-09-25T18:33:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Worker expected=0, running=0, failed=0","time":"2019-09-25T18:33:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Chief expected=1, running=1, failed=0","time":"2019-09-25T18:33:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/pod.go:107","job":"kubeflow.pets-train-dist","level":"info","msg":"Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","replica-type":"evaluator","time":"2019-09-25T18:33:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"tensorflow/status.go:74","job":"kubeflow.pets-train-dist","level":"info","msg":"TFJob=pets-train-dist, ReplicaType=Evaluator expected=0, running=0, failed=0","time":"2019-09-25T18:33:14Z","uid":"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-0 exited with code 0","time":"2019-09-25T18:33:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-worker-1 exited with code 0","time":"2019-09-25T18:33:14Z"}
{"filename":"record/event.go:221","level":"info","msg":"Event(v1.ObjectReference{Kind:\"TFJob\", Namespace:\"kubeflow\", Name:\"pets-train-dist\", UID:\"0ef8f4a9-dfbe-11e9-83c4-42010a8a0161\", APIVersion:\"kubeflow.org/v1\", ResourceVersion:\"55086\", FieldPath:\"\"}): type: 'Normal' reason: 'ExitedWithCode' Pod: kubeflow.pets-train-dist-evaluator-0 exited with code 0","time":"2019-09-25T18:33:14Z"}