basic interface for shape and layout (PaddlePaddle#15)

* basic interface for shape and layout

* cmake configure and test file

* update interface and add unit test

* update shape construct

* remove redundant std::move on return statement

* fix auto&& on ranged-base for loop to deducetype more clearly

* update proto and unit test

paddle/fluid/compiler/piano/CMakeLists.txt

@@ -1,2 +1,6 @@
 add_subdirectory(backends)
 add_subdirectory(note)
+
+cc_library(piano_data_description SRCS layout.cc shape.cc DEPS string_helper note_proto)
+cc_test(piano_layout_test SRCS layout_test.cc DEPS piano_data_description)
+cc_test(piano_shape_test SRCS shape_test.cc DEPS piano_data_description)

paddle/fluid/compiler/piano/layout.cc

@@ -0,0 +1,50 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/compiler/piano/layout.h"
+#include <algorithm>
+#include <iterator>
+#include "paddle/fluid/string/string_helper.h"
+
+namespace paddle {
+namespace piano {
+
+Layout::Layout(const note::LayoutProto& proto) {
+  minor_to_major_.reserve(proto.minor_to_major_size());
+  minor_to_major_.insert(minor_to_major_.end(), proto.minor_to_major().begin(),
+                         proto.minor_to_major().end());
+}
+
+note::LayoutProto Layout::ToProto() const {
+  note::LayoutProto proto;
+  proto.mutable_minor_to_major()->Reserve(minor_to_major().size());
+  for (const auto& dim : minor_to_major_) {
+    proto.add_minor_to_major(dim);
+  }
+  return proto;
+}
+
+std::string Layout::ToString() const {
+  std::vector<std::string> dim_names;
+  std::transform(minor_to_major().begin(), minor_to_major().end(),
+                 std::back_inserter(dim_names),
+                 [](const auto& dim) { return std::to_string(dim); });
+  return paddle::string::format_string(
+      "{%s}", paddle::string::join_strings(dim_names, ',').c_str());
+}
+
+bool Layout::Valid() const { return !minor_to_major().empty(); }
+
+}  // namespace piano
+}  // namespace paddle

paddle/fluid/compiler/piano/layout.h

@@ -0,0 +1,71 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <cstdint>
+#include <string>
+#include <vector>
+#include "paddle/fluid/compiler/piano/note/note.pb.h"
+
+namespace paddle {
+namespace piano {
+
+// A Layout describes how an array is represented in memory
+class Layout {
+ public:
+  Layout() = default;
+
+  // Construct a layout from a LayoutProto.
+  explicit Layout(const note::LayoutProto& proto);
+
+  // Constructs a layout with the given minor-to-major order.
+  explicit Layout(const std::vector<int64_t>& minor_to_major)
+      : minor_to_major_(minor_to_major.begin(), minor_to_major.end()) {}
+
+  // Returns a LayoutProto representation of the Layout.
+  note::LayoutProto ToProto() const;
+
+  // Returns a human-readable string that represents this layout.
+  std::string ToString() const;
+
+  // Return whether this layout is valid
+  bool Valid() const;
+
+  // The following methods for accessing the data member of a Layout object
+  // stores.
+  //
+  // Methods for accessing the minor-to-major array.
+  std::vector<int64_t>* mutable_minor_to_major() { return &minor_to_major_; }
+  const std::vector<int64_t>& minor_to_major() const { return minor_to_major_; }
+
+  void Clear() { mutable_minor_to_major()->clear(); }
+
+ private:
+  // A map from physical dimension numbers to logical dimension numbers.
+  // The first element is the most minor physical dimension (fastest varying
+  // index) and the last the most major (slowest varying index). The contents of
+  // the vector are the indices of the *logical* dimensions in the shape.
+  //
+  // For example, in shape f32[8,100,100,3]{3,0,2,1}, the logical dimensions
+  // are [8,100,100,3] and minor_to_major_ is {3,0,2,1}.
+  // So, the most minor physical dimension is [8,100,100,3][3], which is size 3.
+  // The second most minor is [8,100,100,3][0], which is size 8.
+  // The third most minor is [8,100,100,3][2], which is size 100.
+  // And the major dim is [8,100,100,3][1], which is size 100.
+  std::vector<int64_t> minor_to_major_;
+};
+
+}  // namespace piano
+}  // namespace paddle

paddle/fluid/compiler/piano/layout_test.cc

@@ -0,0 +1,43 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/compiler/piano/layout.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+namespace paddle {
+namespace piano {
+
+class LayoutTest : public ::testing::Test {
+ protected:
+  const Layout layout_ = Layout({3, 2, 1, 0});
+};
+
+TEST_F(LayoutTest, LayoutTransWithProto) {
+  // Layout::ToProto and construct from a note::LayoutProto
+  auto&& layout_proto = layout_.ToProto();
+  ASSERT_EQ(4, layout_proto.minor_to_major_size());
+  EXPECT_EQ(1, layout_proto.minor_to_major(2));
+
+  Layout layout_from_proto(layout_proto);
+  ASSERT_TRUE(layout_from_proto.Valid());
+  ASSERT_EQ(4U, layout_from_proto.minor_to_major().size());
+}
+
+TEST_F(LayoutTest, LayoutToString) {
+  ASSERT_EQ("{3,2,1,0}", layout_.ToString());
+}
+
+}  // namespace piano
+}  // namespace paddle

paddle/fluid/compiler/piano/note/CMakeLists.txt

@@ -2,4 +2,4 @@ cc_library(note_opcode SRCS opcode.cc DEPS errors enforce)
 cc_test(note_opcode_test SRCS opcode_test.cc DEPS note_opcode)
 
 proto_library(note_proto SRCS note.proto)
-set_target_properties(note_proto PROPERTIES COMPILE_FLAGS "-Wno-extra")
+target_compile_options(note_proto PUBLIC "-Wno-extra")

paddle/fluid/compiler/piano/note/note.proto

@@ -33,6 +33,8 @@ enum ElementType {
   F16 = 10;
   F32 = 11;
   F64 = 12;
+
+  ELEMENT_TYPE_TUPLE = 13;
 }
 
 // Memory layout
@@ -44,13 +46,15 @@ message ShapeProto {
   required ElementType element_type = 1;
   repeated int64 dimensions = 2;
   optional LayoutProto layout = 3;
+  // For tuples only, the shapes of constituent shapes in the tuple sequence.
+  repeated ShapeProto tuple_shapes = 4;
 }
 
 // Shape of params and res
 message SignatureProto {
   repeated ShapeProto parameters = 1;
-  repeated ShapeProto results = 2;
-  repeated string parameter_names = 3;
+  repeated string parameter_names = 2;
+  optional ShapeProto result = 3;
 }
 
 // Attribute value

paddle/fluid/compiler/piano/shape.cc

@@ -0,0 +1,105 @@
+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "paddle/fluid/compiler/piano/shape.h"
+#include <algorithm>
+#include <utility>
+#include "paddle/fluid/compiler/piano/layout.h"
+#include "paddle/fluid/string/string_helper.h"
+
+namespace paddle {
+namespace piano {
+
+Shape::Shape(const note::ShapeProto& proto) {
+  element_type_ = proto.element_type();
+  dimensions_.reserve(proto.dimensions_size());
+  dimensions_.insert(dimensions_.end(), proto.dimensions().begin(),
+                     proto.dimensions().end());
+  if (proto.has_layout()) {
+    layout_ = Layout(proto.layout());
+  }
+}
+
+note::ShapeProto Shape::ToProto() const {
+  note::ShapeProto proto;
+  proto.set_element_type(element_type());
+  proto.mutable_dimensions()->Reserve(dimensions().size());
+  for (const auto& dim : dimensions()) {
+    proto.add_dimensions(dim);
+  }
+  if (has_layout()) {
+    *proto.mutable_layout() = layout().ToProto();
+  }
+
+  return proto;
+}
+
+std::string Shape::ToString() const {
+  auto dtype_name = note::ElementType_Name(element_type());
+  std::transform(dtype_name.begin(), dtype_name.end(), dtype_name.begin(),
+                 [](const auto& c) { return std::tolower(c); });
+
+  std::vector<std::string> dim_names;
+  std::transform(dimensions().begin(), dimensions().end(),
+                 std::back_inserter(dim_names),
+                 [](const auto& dim) { return std::to_string(dim); });
+
+  return paddle::string::format_string(
+      "%s[%s]%s", dtype_name.c_str(),
+      paddle::string::join_strings(dim_names, ',').c_str(),
+      layout().ToString().c_str());
+}
+
+Signature::Signature(const note::SignatureProto& proto) {
+  std::transform(proto.parameters().begin(), proto.parameters().end(),
+                 std::back_inserter(parameters_),
+                 [](const auto& shape_proto) { return Shape(shape_proto); });
+  *mutable_result() = Shape(proto.result());
+  parameter_names_.reserve(proto.parameter_names_size());
+  parameter_names_.insert(parameter_names_.end(),
+                          proto.parameter_names().begin(),
+                          proto.parameter_names().end());
+}
+
+note::SignatureProto Signature::ToProto() const {
+  note::SignatureProto proto;
+  proto.mutable_parameters()->Reserve(parameters().size());
+  for (const auto& shape : parameters()) {
+    *proto.add_parameters() = shape.ToProto();
+  }
+  *proto.mutable_result() = result().ToProto();
+  proto.mutable_parameter_names()->Reserve(parameter_names().size());
+  for (const auto& name : parameter_names()) {
+    proto.add_parameter_names(name);
+  }
+  return proto;
+}
+
+std::string Signature::ToString() const {
+  std::vector<std::string> names;
+  for (decltype(parameters().size()) i = 0; i < parameters().size(); ++i) {
+    auto cur = paddle::string::format_string(
+        "%s: %s", i < parameter_names().size() ? parameter_names().at(i).c_str()
+                                               : "(unknown)",
+        parameters().at(i).ToString().c_str());
+    names.emplace_back(std::move(cur));
+  }
+
+  return paddle::string::format_string(
+      "(%s) -> %s", paddle::string::join_strings(names, ',').c_str(),
+      result().ToString().c_str());
+}
+
+}  // namespace piano
+}  // namespace paddle