-
Notifications
You must be signed in to change notification settings - Fork 74k
/
executor.cc
2071 lines (1836 loc) · 75.5 KB
/
executor.cc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "tensorflow/core/common_runtime/executor.h"
#include <atomic>
#include <deque>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
#include "tensorflow/core/common_runtime/costmodel_manager.h"
#include "tensorflow/core/common_runtime/pending_counts.h"
#include "tensorflow/core/common_runtime/step_stats_collector.h"
#include "tensorflow/core/framework/allocation_description.pb.h"
#include "tensorflow/core/framework/allocator.h"
#include "tensorflow/core/framework/cancellation.h"
#include "tensorflow/core/framework/control_flow.h"
#include "tensorflow/core/framework/device_attributes.pb.h"
#include "tensorflow/core/framework/graph.pb.h"
#include "tensorflow/core/framework/log_memory.h"
#include "tensorflow/core/framework/node_def_util.h"
#include "tensorflow/core/framework/op_kernel.h"
#include "tensorflow/core/framework/op_segment.h"
#include "tensorflow/core/framework/step_stats.pb.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/framework/tensor_reference.h"
#include "tensorflow/core/framework/types.h"
#include "tensorflow/core/framework/types.pb.h"
#include "tensorflow/core/graph/edgeset.h"
#include "tensorflow/core/lib/core/errors.h"
#include "tensorflow/core/lib/core/notification.h"
#include "tensorflow/core/lib/core/stringpiece.h"
#include "tensorflow/core/lib/gtl/inlined_vector.h"
#include "tensorflow/core/lib/gtl/manual_constructor.h"
#include "tensorflow/core/lib/gtl/stl_util.h"
#include "tensorflow/core/lib/hash/hash.h"
#include "tensorflow/core/lib/strings/str_util.h"
#include "tensorflow/core/lib/strings/stringprintf.h"
#include "tensorflow/core/platform/logging.h"
#include "tensorflow/core/platform/macros.h"
#include "tensorflow/core/platform/mutex.h"
#include "tensorflow/core/platform/thread_annotations.h"
#include "tensorflow/core/platform/tracing.h"
#include "tensorflow/core/platform/types.h"
#include "tensorflow/core/util/tensor_slice_reader_cache.h"
namespace tensorflow {
namespace {
// 1-D, 0 element tensor.
static const Tensor* const kEmptyTensor = new Tensor;
bool IsInitializationOp(const Node* node) {
return node->op_def().allows_uninitialized_input();
}
// Sets the timeline_label field of *node_stats, using data from *node.
// Returns true iff the node is a transfer node.
// TODO(tucker): merge with the DetailText function in session.cc
// in a common location.
bool SetTimelineLabel(const Node* node, NodeExecStats* node_stats) {
bool is_transfer_node = false;
string memory;
for (auto& all : node_stats->memory()) {
int64 tot = all.total_bytes();
if (tot >= 0.1 * 1048576.0) {
int64 peak = all.peak_bytes();
if (peak > 0) {
memory =
strings::StrCat(memory, "[", all.allocator_name(),
strings::Printf(" %.1fMB %.1fMB] ", tot / 1048576.0,
peak / 1048576.0));
} else {
memory = strings::StrCat(memory, "[", all.allocator_name(),
strings::Printf(" %.1fMB] ", tot / 1048576.0));
}
}
}
const NodeDef& def = node->def();
string text = "";
if (IsSend(node)) {
string tensor_name;
TF_CHECK_OK(GetNodeAttr(def, "tensor_name", &tensor_name));
string recv_device;
TF_CHECK_OK(GetNodeAttr(def, "recv_device", &recv_device));
text = strings::StrCat(memory, def.name(), " = ", def.op(), "(",
tensor_name, " @", recv_device);
is_transfer_node = true;
} else if (IsRecv(node)) {
string tensor_name;
TF_CHECK_OK(GetNodeAttr(def, "tensor_name", &tensor_name));
string send_device;
TF_CHECK_OK(GetNodeAttr(def, "send_device", &send_device));
text = strings::StrCat(memory, def.name(), " = ", def.op(), "(",
tensor_name, " @", send_device);
is_transfer_node = true;
} else {
text = strings::StrCat(
memory, def.name(), " = ", def.op(), "(",
str_util::Join(
std::vector<StringPiece>(def.input().begin(), def.input().end()),
", "),
")");
}
node_stats->set_timeline_label(text);
return is_transfer_node;
}
// Helper routines for collecting step stats.
namespace nodestats {
inline int64 NowInUsec() { return Env::Default()->NowMicros(); }
void SetScheduled(NodeExecStats* nt, int64 t) { nt->set_scheduled_micros(t); }
void SetAllStart(NodeExecStats* nt) { nt->set_all_start_micros(NowInUsec()); }
void SetOpStart(NodeExecStats* nt) {
DCHECK_NE(nt->all_start_micros(), 0);
nt->set_op_start_rel_micros(NowInUsec() - nt->all_start_micros());
}
void SetOpEnd(NodeExecStats* nt) {
DCHECK_NE(nt->all_start_micros(), 0);
nt->set_op_end_rel_micros(NowInUsec() - nt->all_start_micros());
}
void SetAllEnd(NodeExecStats* nt) {
DCHECK_NE(nt->all_start_micros(), 0);
nt->set_all_end_rel_micros(NowInUsec() - nt->all_start_micros());
}
void SetOutput(NodeExecStats* nt, int slot, const Tensor* v) {
DCHECK(v);
NodeOutput* no = nt->add_output();
no->set_slot(slot);
v->FillDescription(no->mutable_tensor_description());
}
void SetMemory(NodeExecStats* nt, OpKernelContext* ctx) {
for (const auto& allocator_pair : ctx->wrapped_allocators()) {
AllocatorMemoryUsed* memory = nt->add_memory();
// retrieving the sizes from the wrapped allocator removes the
// executor's reference to it, so allocator_pair.second must not
// be dereferenced again after this statement
auto sizes = allocator_pair.second->GetSizesAndUnRef();
memory->set_allocator_name(allocator_pair.first->Name());
int tb = sizes.first;
memory->set_total_bytes(tb);
if (allocator_pair.first->TracksAllocationSizes()) {
memory->set_peak_bytes(sizes.second);
}
}
}
void SetReferencedTensors(NodeExecStats* nt,
const TensorReferenceVector& tensors) {
// be careful not to increment the reference count on any tensor
// while recording the information
for (size_t i = 0; i < tensors.size(); ++i) {
AllocationDescription* description = nt->add_referenced_tensor();
tensors.at(i).FillDescription(description);
}
}
} // namespace nodestats
struct NodeItem {
// A graph node.
const Node* node = nullptr;
// The kernel for this node.
OpKernel* kernel = nullptr;
bool kernel_is_expensive = false; // True iff kernel->IsExpensive()
bool kernel_is_async = false; // True iff kernel->AsAsync() != nullptr
bool is_merge = false; // True iff IsMerge(node)
// Cached values of node->num_inputs() and node->num_outputs(), to
// avoid levels of indirection.
int num_inputs;
int num_outputs;
// ExecutorImpl::tensors_[input_start] is the 1st positional input
// for this node.
int input_start = 0;
// ExecutorImpl::output_attrs_[output_attr_start] is the 1st
// positional attribute for the 0th output of this node.
int output_attr_start = 0;
DataType input_type(int i) const {
DCHECK_LT(i, num_inputs);
return (i < 4) ? inlined_input_type[i] : node->input_type(i);
}
DataType output_type(int i) const {
DCHECK_LT(i, num_outputs);
return (i < 4) ? inlined_output_type[i] : node->output_type(i);
}
// Cache first 4 input and output types to reduce levels of indirection
DataType inlined_input_type[4];
DataType inlined_output_type[4];
};
typedef gtl::InlinedVector<TensorValue, 4> TensorValueVec;
typedef gtl::InlinedVector<DeviceContext*, 4> DeviceContextVec;
typedef gtl::InlinedVector<AllocatorAttributes, 4> AllocatorAttributeVec;
class ExecutorImpl : public Executor {
public:
ExecutorImpl(const LocalExecutorParams& p, const Graph* g)
: params_(p), graph_(g), initial_pending_counts_(graph_->num_node_ids()) {
CHECK(p.create_kernel != nullptr);
CHECK(p.delete_kernel != nullptr);
}
~ExecutorImpl() override {
for (int i = 0; i < graph_->num_node_ids(); i++) {
params_.delete_kernel(nodes_[i].kernel);
}
delete[] nodes_;
delete graph_;
}
Status Initialize();
// Infer memory allocation attributes of a node n's output,
// based on its use node dst. Note that dst might not be directly
// connected to n by a single edge, but might be a downstream
// consumer of n's output by reference. *attr is updated with any
// necessary attributes.
Status InferAllocAttr(const Node* n, const Node* dst,
const DeviceNameUtils::ParsedName& local_dev_name,
AllocatorAttributes* attr);
// Process all Nodes in the current graph, attempting to infer the
// memory allocation attributes to be used wherever they may allocate
// a tensor buffer.
Status SetAllocAttrs();
void RunAsync(const Args& args, DoneCallback done) override;
private:
friend class ExecutorState;
static void InitializePending(const Graph* graph, PendingCounts* counts);
// Owned.
LocalExecutorParams params_;
const Graph* graph_;
NodeItem* nodes_ = nullptr; // array of size "graph_.num_node_ids()"
int total_input_tensors_ = 0; // == sum(nodes_[*].num_inputs())
int total_output_tensors_ = 0; // == sum(nodes_[*].num_outputs())
// A cached value of params_
bool device_record_tensor_accesses_ = false;
// Root nodes (with no in edges) that should form the initial ready queue
std::vector<const Node*> root_nodes_;
PendingCounts initial_pending_counts_;
// The number of inputs for each frame in this graph. This is static
// information of the graph.
std::unordered_map<string, int> frame_input_count_;
std::vector<AllocatorAttributes> output_attrs_;
TF_DISALLOW_COPY_AND_ASSIGN(ExecutorImpl);
};
Status ExecutorImpl::Initialize() {
const int num_nodes = graph_->num_node_ids();
delete[] nodes_;
nodes_ = new NodeItem[num_nodes];
Status s;
total_input_tensors_ = 0;
total_output_tensors_ = 0;
InitializePending(graph_, &initial_pending_counts_);
// Cache this value so we make this virtual function call once, rather
// that O(# steps * # nodes per step) times.
device_record_tensor_accesses_ =
params_.device->RequiresRecordingAccessedTensors();
// Preprocess every node in the graph to create an instance of op
// kernel for each node;
for (const Node* n : graph_->nodes()) {
const int id = n->id();
// See if this node is a root node, and if so, add to root_nodes_
const int num_in_edges = n->in_edges().size();
if (num_in_edges == 0) {
root_nodes_.push_back(n);
}
NodeItem* item = &nodes_[id];
item->node = n;
item->num_inputs = n->num_inputs();
item->num_outputs = n->num_outputs();
for (int i = 0; i < std::min(4, item->num_inputs); i++) {
item->inlined_input_type[i] = n->input_type(i);
}
for (int i = 0; i < std::min(4, item->num_outputs); i++) {
item->inlined_output_type[i] = n->output_type(i);
}
item->input_start = total_input_tensors_;
total_input_tensors_ += n->num_inputs();
item->output_attr_start = total_output_tensors_;
total_output_tensors_ += n->num_outputs();
s = params_.create_kernel(n->def(), &item->kernel);
if (!s.ok()) {
item->kernel = nullptr;
s = AttachDef(s, n->def());
LOG(ERROR) << "Executor failed to create kernel. " << s;
break;
}
CHECK(item->kernel);
item->kernel_is_expensive = item->kernel->IsExpensive();
item->kernel_is_async = (item->kernel->AsAsync() != nullptr);
item->is_merge = IsMerge(n);
// Initialize static information about the frames in the graph.
if (IsEnter(n)) {
string frame_name;
s = GetNodeAttr(n->def(), "frame_name", &frame_name);
if (!s.ok()) return s;
++frame_input_count_[frame_name];
}
}
if (!s.ok()) return s;
return SetAllocAttrs();
}
Status ExecutorImpl::SetAllocAttrs() {
Status s;
Device* device = params_.device;
DeviceNameUtils::ParsedName local_dev_name = device->parsed_name();
output_attrs_.resize(total_output_tensors_);
for (const Node* n : graph_->nodes()) {
NodeItem* item = &nodes_[n->id()];
const int base_index = item->output_attr_start;
// Examine the out edges of each node looking for special use
// cases that may affect memory allocation attributes.
for (auto e : n->out_edges()) {
const int index = e->src_output();
AllocatorAttributes attr;
s = InferAllocAttr(n, e->dst(), local_dev_name, &attr);
if (!s.ok()) return s;
if (attr.value != 0) {
if (!e->IsControlEdge()) {
output_attrs_[base_index + index].Merge(attr);
}
}
}
for (int out = 0; out < n->num_outputs(); out++) {
OpKernel* op_kernel = item->kernel;
DCHECK_LT(out, op_kernel->output_memory_types().size());
bool on_host = op_kernel->output_memory_types()[out] == HOST_MEMORY;
AllocatorAttributes h;
h.set_on_host(on_host);
output_attrs_[base_index + out].Merge(h);
}
}
return s;
}
Status ExecutorImpl::InferAllocAttr(
const Node* n, const Node* dst,
const DeviceNameUtils::ParsedName& local_dev_name,
AllocatorAttributes* attr) {
Status s;
// Note that it's possible for *n to be a Recv and *dst to be a Send,
// so these two cases are not mutually exclusive.
if (IsRecv(n)) {
string src_name;
s = GetNodeAttr(n->def(), "send_device", &src_name);
if (!s.ok()) return s;
DeviceNameUtils::ParsedName parsed_src_name;
if (!DeviceNameUtils::ParseFullName(src_name, &parsed_src_name)) {
s = errors::Internal("Bad send_device attr '", src_name, "' in node ",
n->name());
return s;
}
if (!DeviceNameUtils::IsSameAddressSpace(parsed_src_name, local_dev_name)) {
// Value is going to be the sink of an RPC.
attr->set_nic_compatible(true);
VLOG(2) << "node " << n->name() << " is the sink of an RPC in";
} else if ((local_dev_name.type == "CPU" || n->IsHostRecv()) &&
parsed_src_name.type == "GPU") {
// Value is going to be the sink of a local DMA from GPU to CPU.
attr->set_gpu_compatible(true);
VLOG(2) << "node " << n->name() << " is the sink of a gpu->cpu copy";
} else {
VLOG(2) << "default alloc case local type " << local_dev_name.type
<< " remote type " << parsed_src_name.type;
}
}
if (IsSend(dst)) {
string dst_name;
s = GetNodeAttr(dst->def(), "recv_device", &dst_name);
if (!s.ok()) return s;
DeviceNameUtils::ParsedName parsed_dst_name;
if (!DeviceNameUtils::ParseFullName(dst_name, &parsed_dst_name)) {
s = errors::Internal("Bad recv_device attr '", dst_name, "' in node ",
n->name());
return s;
}
if (!DeviceNameUtils::IsSameAddressSpace(parsed_dst_name, local_dev_name)) {
// Value is going to be the source of an RPC.
attr->set_nic_compatible(true);
VLOG(2) << "node " << n->name() << " is the source of an RPC out";
} else if ((local_dev_name.type == "CPU" || dst->IsHostSend()) &&
parsed_dst_name.type == "GPU") {
// Value is going to be the source of a local DMA from CPU to GPU.
// Note that this does not cover the case where the allocation of the
// output tensor is not generated by the src: n.
attr->set_gpu_compatible(true);
VLOG(2) << "node " << n->name() << " is the source of a cpu->gpu copy";
} else {
VLOG(2) << "default alloc case local type " << local_dev_name.type
<< " remote type " << parsed_dst_name.type;
}
} else if (dst->type_string() == "ToFloat") {
for (auto e : dst->out_edges()) {
s = InferAllocAttr(n, e->dst(), local_dev_name, attr);
if (!s.ok()) return s;
}
}
return s;
}
// The state associated with one invocation of ExecutorImpl::Run.
// ExecutorState dispatches nodes when they become ready and keeps
// track of how many predecessors of a node have not done (pending_).
class ExecutorState {
public:
ExecutorState(const Executor::Args& args, ExecutorImpl* impl);
~ExecutorState();
void RunAsync(Executor::DoneCallback done);
private:
// Either a tensor pointer (pass-by-reference) or a tensor (pass-by-value).
// TODO(yuanbyu): A better way to do "has_value"?
struct Entry {
Entry() {}
Entry(const Entry& other)
: ref(other.ref),
ref_mu(other.ref_mu),
has_value(other.has_value),
val_field_is_set(other.val_field_is_set),
alloc_attr(other.alloc_attr),
device_context(other.device_context) {
if (val_field_is_set) {
val.Init(*other.val);
}
}
~Entry() {
if (val_field_is_set) val.Destroy();
}
Entry& operator=(const Entry& other) {
if (val_field_is_set) {
val.Destroy();
}
ref = other.ref;
ref_mu = other.ref_mu;
has_value = other.has_value;
val_field_is_set = other.val_field_is_set;
alloc_attr = other.alloc_attr;
device_context = other.device_context;
if (val_field_is_set) {
val.Init(*other.val);
}
return *this;
}
// Clears the <val> field.
void ClearVal() {
if (val_field_is_set) {
val.Destroy();
val_field_is_set = false;
}
}
// A tensor value, if val_field_is_set.
ManualConstructor<Tensor> val;
Tensor* ref = nullptr; // A tensor reference.
mutex* ref_mu = nullptr; // mutex for *ref if ref is not nullptr.
// Whether the value exists, either in <val> or <ref>.
bool has_value = false;
bool val_field_is_set = false;
// The attributes of the allocator that creates the tensor.
AllocatorAttributes alloc_attr;
// Every entry carries an optional DeviceContext containing
// Device-specific information about how the Tensor was produced.
DeviceContext* device_context = nullptr;
};
// Contains a value for [node->id()] for the device context assigned by the
// device at the beginning of a step.
DeviceContextMap device_context_map_;
struct IterationState {
explicit IterationState(const ExecutorImpl* impl)
: input_tensors(new Entry[impl->total_input_tensors_]),
outstanding_ops(0),
outstanding_frame_count(0),
counts_(impl->graph_->num_node_ids()) {
counts_.InitializeFrom(impl->initial_pending_counts_);
}
// The state of an iteration.
// One copy per iteration. For iteration k, i-th node's j-th input is in
// input_tensors[k][impl_->nodes[i].input_start + j]. An entry is either
// a tensor pointer (pass-by-reference) or a tensor (pass-by-value).
//
// NOTE: No need to protect input_tensors[i] by any locks because it
// is resized once. Each element of tensors_ is written once by the
// source node of an edge and is cleared by the destination of the same
// edge. The latter node is never run concurrently with the former node.
Entry* input_tensors;
// The number of outstanding ops for each iteration.
int outstanding_ops;
// The number of outstanding frames for each iteration.
int outstanding_frame_count;
int pending(int id) { return counts_.pending(id); }
int decrement_pending(int id, int v) {
return counts_.decrement_pending(id, v);
}
// Mark a merge node as live
// REQUIRES: Node corresponding to "id" is a merge node
void mark_live(int id) { counts_.mark_live(id); }
// Mark a node to show that processing has started.
void mark_started(int id) { counts_.mark_started(id); }
// Mark a node to show that processing has completed.
void mark_completed(int id) { counts_.mark_completed(id); }
PendingCounts::NodeState node_state(int id) {
return counts_.node_state(id);
}
int dead_count(int id) { return counts_.dead_count(id); }
void increment_dead_count(int id) { counts_.increment_dead_count(id); }
~IterationState() { delete[] input_tensors; }
private:
PendingCounts counts_;
};
struct FrameState {
// A new frame is created for each loop. Execution starts at iteration 0.
// When a value at iteration 0 passes through a NextIteration node,
// iteration 1 is created and starts running. Note that iteration 0 may
// still be running so multiple iterations may run in parallel. The
// frame maintains the state of iterations in several data structures
// such as pending_count and input_tensors. When iteration 0 completes,
// we garbage collect the state of iteration 0.
//
// A frame instance is considered "done" and can be garbage collected
// if all its inputs have entered and all its iterations are "done".
//
// A frame manages the live iterations of an iterative computation.
// Iteration i is considered "done" when there are no outstanding ops,
// frames at iteration i are done, all recvs for this iteration are
// completed, and iteration i-1 is done. For iteration 0, we instead
// wait for there to be no more pending inputs of the frame.
//
// Frames and iterations are garbage collected once they are done.
// The state we need to keep around is highly dependent on the
// parallelism enabled by the scheduler. We may want to have the
// scheduler dynamically control the outstanding number of live
// parallel frames and iterations. To reduce the state space, the
// scheduler might want to schedule ops in inner frames first and
// lower iterations first.
//
// This frame state is mostly initialized lazily on demand so we
// don't introduce unnecessary overhead.
// The name of this frame, which is the concatenation of its parent
// frame name, the iteration of the parent frame when this frame was
// created, and the value of the attr 'frame_name'.
string frame_name;
// The unique id for this frame. Generated by fingerprinting
// frame_name.
uint64 frame_id;
// The iteration id of its parent frame when this frame is created.
// -1 if there is no parent frame. The frame_name/parent_iter pair
// uniquely identifies this FrameState.
int64 parent_iter = -1;
// The FrameState of its parent frame.
FrameState* parent_frame = nullptr;
// The highest iteration number we have reached so far in this frame.
int64 iteration_count = 0;
// The number of inputs this frame is still waiting.
int num_pending_inputs = 0;
// The number of outstanding iterations.
int num_outstanding_iterations = 0;
// The maximum allowed number of parallel iterations.
int max_parallel_iterations = 1;
// The iteration states of this frame.
gtl::InlinedVector<IterationState*, 12> iterations;
// The NextIteration nodes to enter a new iteration. If the number of
// outstanding iterations reaches the limit, we will defer the start of
// the next iteration until the number of outstanding iterations falls
// below the limit.
std::vector<std::pair<const Node*, Entry>> next_iter_roots;
// The values of the loop invariants for this loop. They are added into
// this list as they "enter" the frame. When a loop invariant enters,
// we make it available to all active iterations. When the frame starts
// a new iteration, we make all the current loop invariants available
// to the new iteration.
std::vector<std::pair<const Node*, Entry>> inv_values;
// The list of dead exit nodes for the current highest iteration. We
// will only "execute" the dead exits of the final iteration.
std::vector<const Node*> dead_exits;
IterationState* GetIteration(int64 iter) {
int index = iter % iterations.size();
return iterations[index];
}
void SetIteration(int64 iter, IterationState* state) {
int index = iter % iterations.size();
iterations[index] = state;
}
~FrameState() {
for (size_t i = 0; i < iterations.size(); ++i) {
delete iterations[i];
iterations[i] = nullptr;
}
}
};
// A tagged node: <frame*, iter, node*>.
struct TaggedNode {
const Node* node = nullptr;
FrameState* input_frame = nullptr;
int64 input_iter = -1;
bool is_dead = false;
TaggedNode(const Node* t_node, FrameState* in_frame, int64 in_iter,
bool dead) {
node = t_node;
input_frame = in_frame;
input_iter = in_iter;
is_dead = dead;
}
};
// A drop-in replacement for std::deque<TaggedNode>. We typically don't
// have that many nodes in the ready queue, so we just use a vector and
// don't free up memory from the queue as we consume nodes.
class TaggedNodeReadyQueue {
public:
TaggedNodeReadyQueue() : front_index_(0) {}
void push_back(TaggedNode node) { ready_.push_back(node); }
TaggedNode front() const {
DCHECK_LT(front_index_, ready_.size());
return ready_[front_index_];
}
void pop_front() {
DCHECK_LT(front_index_, ready_.size());
front_index_++;
if ((front_index_ == ready_.size()) || (front_index_ > 16384)) {
if (front_index_ == ready_.size()) {
ready_.clear();
} else {
// Lots of unused entries at beginning of vector: move everything down
// to start of vector.
ready_.erase(ready_.begin(), ready_.begin() + front_index_);
}
front_index_ = 0;
}
}
bool empty() const { return ready_.empty(); }
const TaggedNode* begin() const { return ready_.begin() + front_index_; }
const TaggedNode* end() const { return ready_.end(); }
private:
gtl::InlinedVector<TaggedNode, 16> ready_;
int front_index_;
};
struct AsyncState;
typedef gtl::InlinedVector<TaggedNode, 8> TaggedNodeSeq;
typedef gtl::InlinedVector<Entry, 4> EntryVector;
const bool vlog_; // true if VLOG_IS_ON(1). Used to check vlog cheaply.
// true if LogMemory::IsEnabled(). Used to check memory enabled cheaply.
const bool log_memory_;
int64 step_id_;
// Not owned.
Rendezvous* rendezvous_;
SessionState* session_state_;
TensorStore* tensor_store_;
// Step-local resource manager.
ResourceMgr* step_resource_manager_;
StepStatsCollector* stats_collector_;
// QUESTION: Make it a checkpoint::TensorSliceReaderCacheWrapper
// instead of a pointer? (avoids having to delete).
checkpoint::TensorSliceReaderCacheWrapper* slice_reader_cache_;
FunctionCallFrame* call_frame_;
const ExecutorImpl* impl_;
CancellationManager* cancellation_manager_;
Executor::Args::Runner runner_;
// Owned.
// A flag that is set on error after the frame state has been
// dumped for diagnostic purposes.
bool dumped_on_error_ = false;
// The root frame in which the execution of this step is started.
FrameState* root_frame_;
// Invoked when the execution finishes.
Executor::DoneCallback done_cb_;
std::atomic_int_fast32_t num_outstanding_ops_;
mutex mu_;
Status status_ GUARDED_BY(mu_);
// Mapping from frame name to outstanding frames. A new frame is created
// at some iteration of an active frame. So the unique key for the new
// child frame is composed of the name of the parent frame, the iteration
// number at which the parent frame is creating the new frame, and the
// name of the new frame from nodedef.
std::unordered_map<string, FrameState*> outstanding_frames_ GUARDED_BY(mu_);
// The unique name of a frame.
inline string MakeFrameName(FrameState* frame, int64 iter_id, string name) {
return strings::StrCat(frame->frame_name, ";", iter_id, ";", name);
}
// Find an existing or create a new child frame in the frame 'frame' at
// iteration 'iter'.
void FindOrCreateChildFrame(FrameState* frame, int64 iter, const Node* node,
FrameState** child) EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Increments the iteration id. If this is a new iteration, initialize it.
void IncrementIteration(FrameState* frame, TaggedNodeSeq* ready)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Returns true if the computation in the frame is completed.
bool IsFrameDone(FrameState* frame) EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Returns true if the iteration of the frame is completed.
bool IsIterationDone(FrameState* frame, int64 iter)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Get the output frame/iter of a node. Create new frame/iteration if
// needed. If there are dead roots for the new iteration, we need to
// "execute" them so add them to the ready queue. Returns true if
// we need to check for the completion of output frame/iter.
void FindOrCreateOutputFrameIter(const TaggedNode& tagged_node,
const EntryVector& outputs,
FrameState** frame, int64* iter,
TaggedNodeSeq* ready)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Cleanup frames and iterations
void CleanupFramesIterations(FrameState* frame, int64 iter,
TaggedNodeSeq* ready)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Activate all the deferred NextIteration nodes in a new iteration.
void ActivateNexts(FrameState* frame, int64 iter, TaggedNodeSeq* ready)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Activate all the current loop invariants in a new iteration.
void ActivateLoopInvs(FrameState* frame, int64 iter, TaggedNodeSeq* ready)
EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Add a new loop invariant and make it available to all active iterations.
void AddLoopInv(FrameState* frame, const Node* node, const Entry& value,
TaggedNodeSeq* ready) EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Activate the successors of a node.
void ActivateNode(const Node* node, const bool is_dead, FrameState* frame,
int64 iter, const EntryVector& outputs,
TaggedNodeSeq* ready) EXCLUSIVE_LOCKS_REQUIRED(mu_);
// Process a ready node in current thread.
void Process(TaggedNode node, int64 scheduled_usec);
// Before invoking item->kernel, fills in its "inputs".
Status PrepareInputs(const NodeItem& item, Entry* first_input,
TensorValueVec* inputs,
DeviceContextVec* input_device_contexts,
AllocatorAttributeVec* input_alloc_attrs,
bool* is_input_dead);
// After item->kernel computation is done, processes its outputs.
Status ProcessOutputs(const NodeItem& item, OpKernelContext* ctx,
EntryVector* outputs, NodeExecStats* stats);
// After processing the outputs, propagates the outputs to their dsts.
void PropagateOutputs(const TaggedNode& tagged_node,
const EntryVector& outputs, TaggedNodeSeq* ready);
// "node" just finishes. Takes ownership of "stats". Returns true if
// execution has completed.
bool NodeDone(const Status& s, const Node* node, const TaggedNodeSeq& ready,
NodeExecStats* stats, TaggedNodeReadyQueue* inline_ready);
// Schedule all the expensive nodes in 'ready', and put all the inexpensive
// nodes in 'ready' into 'inline_ready'.
void ScheduleReady(const TaggedNodeSeq& ready,
TaggedNodeReadyQueue* inline_ready);
// Provide debugging output about an outstanding node in the executor.
void DumpCompletedNodeState(const int node_id, const Entry* input_vector);
void DumpPendingNodeState(const int node_id, const Entry* input_vector,
bool show_nodes_with_no_ready_inputs);
void DumpActiveNodeState(const int node_id, const Entry* input_vector);
// Provide debugging output about an outstanding iteration in the executor.
void DumpIterationState(IterationState* iteration);
// Provide debugging output of the state of the executor.
void DumpState();
const Tensor* GetTensorValueForDump(const Entry& input);
// One thread of control finishes.
void Finish();
// A standalone routine for this expression so that we can express
// that we don't want thread safety analysis on this reference (it's
// safe to do without the lock because the iterations array never
// resizes and this particular iteration's array element will not
// be changed out from under us because the iteration is still alive).
Entry* GetInputTensors(FrameState* input_frame,
int64 input_iter) const NO_THREAD_SAFETY_ANALYSIS {
return input_frame->GetIteration(input_iter)->input_tensors;
}
};
ExecutorState::ExecutorState(const Executor::Args& args, ExecutorImpl* impl)
: vlog_(VLOG_IS_ON(1)),
log_memory_(LogMemory::IsEnabled()),
step_id_(args.step_id),
rendezvous_(args.rendezvous),
session_state_(args.session_state),
tensor_store_(args.tensor_store),
step_resource_manager_(args.step_resource_manager),
stats_collector_(args.stats_collector),
slice_reader_cache_(new checkpoint::TensorSliceReaderCacheWrapper),
call_frame_(args.call_frame),
impl_(impl),
cancellation_manager_(args.cancellation_manager),
runner_(args.runner),
num_outstanding_ops_(0) {
// We start the entire execution in iteration 0 of the root frame
// so let us create the root frame and the state for iteration 0.
// Initialize the frame.
root_frame_ = new FrameState;
root_frame_->frame_name = "_root"; // assume to be unique
root_frame_->frame_id = 0; // must be 0
root_frame_->num_pending_inputs = 0;
root_frame_->num_outstanding_iterations = 1;
root_frame_->max_parallel_iterations = 1; // enough for root frame
root_frame_->iterations.resize(root_frame_->max_parallel_iterations);
if (vlog_) VLOG(2) << "Create frame: " << root_frame_->frame_name;
// Initialize the first iteration.
IterationState* iter_state = new IterationState(impl);
root_frame_->iterations[0] = iter_state;
// Initialize the executor state.
outstanding_frames_.insert({root_frame_->frame_name, root_frame_});
}
ExecutorState::~ExecutorState() {
for (auto name_frame : outstanding_frames_) {
delete name_frame.second;
}
for (auto it : device_context_map_) {
it->Unref();
}
delete slice_reader_cache_;
}
void ExecutorImpl::InitializePending(const Graph* graph,
PendingCounts* counts) {
for (int id = 0; id < graph->num_node_ids(); id++) {
counts->set_initial_count(id, 0, 0); // Make sure everything is initialized
}
for (const Node* n : graph->nodes()) {
const int id = n->id();
const int num_in_edges = n->in_edges().size();
int initial_count;
if (IsMerge(n)) {
// merge waits all control inputs so we initialize the pending
// count to be the number of control edges.
int32 num_control_edges = 0;
for (const Edge* edge : n->in_edges()) {
if (edge->IsControlEdge()) {
num_control_edges++;
}
}
// Use bit 0 to indicate if we are waiting for a ready live data input.
initial_count = 1 + (num_control_edges << 1);
} else {
initial_count = num_in_edges;
}
counts->set_initial_count(id, initial_count, num_in_edges);
}
}
void ExecutorState::RunAsync(Executor::DoneCallback done) {
const Graph* graph = impl_->graph_;
TaggedNodeSeq ready;
// Ask the device to fill in the device context map.
Device* device = impl_->params_.device;
Status fill_status = device->FillContextMap(graph, &device_context_map_);
if (!fill_status.ok()) {
done(fill_status);
return;
}
// Initialize the ready queue.
for (const Node* n : impl_->root_nodes_) {
DCHECK_EQ(n->in_edges().size(), 0);
ready.push_back(TaggedNode{n, root_frame_, 0, false});
}
if (ready.empty()) {
done(Status::OK());
} else {
num_outstanding_ops_ = ready.size();
root_frame_->iterations[0]->outstanding_ops = ready.size();
done_cb_ = done;
// Schedule to run all the ready ops in thread pool.
ScheduleReady(ready, nullptr);
}
}
// State kept alive for executing an asynchronous node in another
// thread. NOTE: We need to make a copy of p.input,
// p.input_device_contexts, and p.input_alloc_attrs for asynchronous
// kernels because OpKernelContext methods like input_type(i) needs
// the param points to valid input type vector. It's not an issue for
// sync kernels because these vectors are kept on the stack.
struct ExecutorState::AsyncState {
AsyncState(const OpKernelContext::Params& p, const TaggedNode& _tagged_node,
const NodeItem& _item, Entry* _first_input, NodeExecStats* _stats)
: saved_inputs(*p.inputs),
saved_input_device_contexts(*p.input_device_contexts),
saved_input_alloc_attrs(*p.input_alloc_attrs),