Skip to content

Commit

Permalink
tf.PriorityQueue, Protobuf.TextFormat
Browse files Browse the repository at this point in the history
  • Loading branch information
Oceania2018 committed Sep 23, 2019
1 parent 7a828c0 commit a596dbe
Show file tree
Hide file tree
Showing 11 changed files with 328 additions and 146 deletions.
27 changes: 27 additions & 0 deletions docs/source/Queue.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,33 @@ A FIFOQueue that supports batching variable-sized tensors by padding. A `Padding

A queue implementation that dequeues elements in prioritized order. A `PriorityQueue` has bounded capacity; supports multiple concurrent producers and consumers; and provides exactly-once delivery. A `PriorityQueue` holds a list of up to `capacity` elements. Each element is a fixed-length tuple of tensors whose dtypes are described by `types`, and whose shapes are optionally described by the `shapes` argument.

```csharp
[TestMethod]
public void PriorityQueue()
{
var queue = tf.PriorityQueue(3, tf.@string);
var init = queue.enqueue_many(new[] { 2L, 4L, 3L }, new[] { "p1", "p2", "p3" });
var x = queue.dequeue();

using (var sess = tf.Session())
{
init.run();

// output will 2, 3, 4
var result = sess.run(x);
Assert.AreEqual(result[0].GetInt64(), 2L);

result = sess.run(x);
Assert.AreEqual(result[0].GetInt64(), 3L);

result = sess.run(x);
Assert.AreEqual(result[0].GetInt64(), 4L);
}
}
```



#### RandomShuffleQueue

A queue implementation that dequeues elements in a random order. A `RandomShuffleQueue` has bounded capacity; supports multiple concurrent producers and consumers; and provides exactly-once delivery. A `RandomShuffleQueue` holds a list of up to `capacity` elements. Each element is a fixed-length tuple of tensors whose dtypes are described by `dtypes`, and whose shapes are optionally described by the `shapes` argument.
Expand Down
23 changes: 21 additions & 2 deletions src/TensorFlowNET.Core/APIs/tf.queue.cs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public partial class tensorflow
string shared_name = null,
string name = "padding_fifo_queue")
=> new PaddingFIFOQueue(capacity,
new [] { dtype },
new[] { dtype },
new[] { shape },
shared_name: shared_name,
name: name);
Expand Down Expand Up @@ -86,7 +86,26 @@ public partial class tensorflow
=> new FIFOQueue(capacity,
new[] { dtype },
new[] { shape ?? new TensorShape() },
new[] { name },
shared_name: shared_name,
name: name);

/// <summary>
/// Creates a queue that dequeues elements in a first-in first-out order.
/// </summary>
/// <param name="capacity"></param>
/// <param name="dtype"></param>
/// <param name="shape"></param>
/// <param name="shared_name"></param>
/// <param name="name"></param>
/// <returns></returns>
public PriorityQueue PriorityQueue(int capacity,
TF_DataType dtype,
TensorShape shape = null,
string shared_name = null,
string name = "priority_queue")
=> new PriorityQueue(capacity,
new[] { dtype },
new[] { shape ?? new TensorShape() },
shared_name: shared_name,
name: name);
}
Expand Down
66 changes: 66 additions & 0 deletions src/TensorFlowNET.Core/Operations/Queues/PriorityQueue.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using static Tensorflow.Binding;

namespace Tensorflow.Queues
{
public class PriorityQueue : QueueBase
{
public PriorityQueue(int capacity,
TF_DataType[] dtypes,
TensorShape[] shapes,
string[] names = null,
string shared_name = null,
string name = "priority_queue")
: base(dtypes: dtypes, shapes: shapes, names: names)
{
_queue_ref = gen_data_flow_ops.priority_queue_v2(
component_types: dtypes,
shapes: shapes,
capacity: capacity,
shared_name: shared_name,
name: name);

_name = _queue_ref.op.name.Split('/').Last();

var dtypes1 = dtypes.ToList();
dtypes1.Insert(0, TF_DataType.TF_INT64);
_dtypes = dtypes1.ToArray();

var shapes1 = shapes.ToList();
shapes1.Insert(0, new TensorShape());
_shapes = shapes1.ToArray();
}

public Operation enqueue_many<T>(long[] indexes, T[] vals, string name = null)
{
return tf_with(ops.name_scope(name, $"{_name}_EnqueueMany", vals), scope =>
{
var vals_tensor1 = _check_enqueue_dtypes(indexes);
var vals_tensor2 = _check_enqueue_dtypes(vals);
var tensors = new List<Tensor>();
tensors.AddRange(vals_tensor1);
tensors.AddRange(vals_tensor2);
return gen_data_flow_ops.queue_enqueue_many_v2(_queue_ref, tensors.ToArray(), name: scope);
});
}

public Tensor[] dequeue(string name = null)
{
Tensor[] ret;
if (name == null)
name = $"{_name}_Dequeue";

if (_queue_ref.dtype == TF_DataType.TF_RESOURCE)
ret = gen_data_flow_ops.queue_dequeue_v2(_queue_ref, _dtypes, name: name);
else
ret = gen_data_flow_ops.queue_dequeue(_queue_ref, _dtypes, name: name);

return ret;
}
}
}
10 changes: 4 additions & 6 deletions src/TensorFlowNET.Core/Operations/Queues/QueueBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public Operation enqueue_many<T>(T[] vals, string name = null)
});
}

private Tensor[] _check_enqueue_dtypes(object vals)
protected Tensor[] _check_enqueue_dtypes(object vals)
{
var tensors = new List<Tensor>();

Expand All @@ -56,12 +56,10 @@ private Tensor[] _check_enqueue_dtypes(object vals)
}
break;

case int[] vals1:
tensors.Add(ops.convert_to_tensor(vals1, dtype: _dtypes[0], name: $"component_0"));
break;

default:
throw new NotImplementedException("");
var dtype1 = GetType().Name == "PriorityQueue" ? _dtypes[1] : _dtypes[0];
tensors.Add(ops.convert_to_tensor(vals, dtype: dtype1, name: $"component_0"));
break;
}

return tensors.ToArray();
Expand Down
28 changes: 28 additions & 0 deletions src/TensorFlowNET.Core/Operations/Queues/RandomShuffleQueue.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace Tensorflow.Queues
{
public class RandomShuffleQueue : QueueBase
{
public RandomShuffleQueue(int capacity,
TF_DataType[] dtypes,
TensorShape[] shapes,
string[] names = null,
string shared_name = null,
string name = "randomshuffle_fifo_queue")
: base(dtypes: dtypes, shapes: shapes, names: names)
{
_queue_ref = gen_data_flow_ops.padding_fifo_queue_v2(
component_types: dtypes,
shapes: shapes,
capacity: capacity,
shared_name: shared_name,
name: name);

_name = _queue_ref.op.name.Split('/').Last();
}
}
}
16 changes: 16 additions & 0 deletions src/TensorFlowNET.Core/Operations/gen_data_flow_ops.cs
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,22 @@ public static Tensor dynamic_stitch(Tensor[] indices, Tensor[] data, string name
return _op.output;
}

public static Tensor priority_queue_v2(TF_DataType[] component_types, TensorShape[] shapes,
int capacity = -1, string container = "", string shared_name = "",
string name = null)
{
var _op = _op_def_lib._apply_op_helper("PriorityQueueV2", name, new
{
component_types,
shapes,
capacity,
container,
shared_name
});

return _op.output;
}

public static Operation queue_enqueue(Tensor handle, Tensor[] components, int timeout_ms = -1, string name = null)
{
var _op = _op_def_lib._apply_op_helper("QueueEnqueue", name, new
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,12 @@ public class ModelBuilder
{
ImageResizerBuilder _image_resizer_builder;
FasterRCNNFeatureExtractor _feature_extractor;
AnchorGeneratorBuilder anchor_generator_builder;
AnchorGeneratorBuilder _anchor_generator_builder;

public ModelBuilder()
{
_image_resizer_builder = new ImageResizerBuilder();
_anchor_generator_builder = new AnchorGeneratorBuilder();
}

/// <summary>
Expand Down Expand Up @@ -51,7 +52,7 @@ private FasterRCNNMetaArch _build_faster_rcnn_model(FasterRcnn frcnn_config, boo
inplace_batchnorm_update: frcnn_config.InplaceBatchnormUpdate);

var number_of_stages = frcnn_config.NumberOfStages;
var first_stage_anchor_generator = anchor_generator_builder.build(frcnn_config.FirstStageAnchorGenerator);
var first_stage_anchor_generator = _anchor_generator_builder.build(frcnn_config.FirstStageAnchorGenerator);
var first_stage_atrous_rate = frcnn_config.FirstStageAtrousRate;

return new FasterRCNNMetaArch(new FasterRCNNInitArgs
Expand Down

0 comments on commit a596dbe

Please sign in to comment.