From 08e556b41489dc7cfe7028f6d11937ddde01b05a Mon Sep 17 00:00:00 2001 From: Doug Friedman Date: Wed, 3 Apr 2019 07:22:14 -0400 Subject: [PATCH 1/8] [TF-76] add TensorShape conformance to PythonConvertible --- stdlib/public/TensorFlow/TensorShape.swift | 12 ++++++++++++ test/Python/python_runtime.swift | 14 ++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/stdlib/public/TensorFlow/TensorShape.swift b/stdlib/public/TensorFlow/TensorShape.swift index 7209e4d01a275..862f140a2ccbd 100644 --- a/stdlib/public/TensorFlow/TensorShape.swift +++ b/stdlib/public/TensorFlow/TensorShape.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +import Python + // NOTE: it may be possible to edit `TensorShape` to support "labeled tensors". // Dimensions may be either an Int32 or an enum representing a label. @@ -158,3 +160,13 @@ extension TensorShape : Codable { self.init(dimensions) } } + +extension TensorShape: PythonConvertible { + public var pythonObject: PythonObject { + return self.dimensions.pythonObject + } + + public init?(_ pythonObject: PythonObject) { + self.init(Array(pythonObject)!) + } +} diff --git a/test/Python/python_runtime.swift b/test/Python/python_runtime.swift index c44d2d9e60f14..342d5a06f966d 100644 --- a/test/Python/python_runtime.swift +++ b/test/Python/python_runtime.swift @@ -5,6 +5,7 @@ import Python import StdlibUnittest +import struct TensorFlow.TensorShape /// The gc module is an interface to the Python garbage collector. let gc = Python.import("gc") @@ -258,6 +259,8 @@ PythonRuntimeTestSuite.testWithLeakChecking("ConvertibleFromPython") { let five: PythonObject = 5 let half: PythonObject = 0.5 let string: PythonObject = "abc" + let intArray: PythonObject = [2, 3] + let dict: PythonObject = ["abc": 97] expectEqual(-1, Int(minusOne)) expectEqual(-1, Int8(minusOne)) @@ -285,6 +288,10 @@ PythonRuntimeTestSuite.testWithLeakChecking("ConvertibleFromPython") { expectEqual("abc", String(string)) + expectEqual([2, 3], Array(intArray)) + expectEqual(TensorShape(2, 3), TensorShape(intArray)) + expectEqual(["abc": 97], Dictionary(dict)) + expectNil(String(zero)) expectNil(Int(string)) expectNil(Double(string)) @@ -293,6 +300,8 @@ PythonRuntimeTestSuite.testWithLeakChecking("ConvertibleFromPython") { PythonRuntimeTestSuite.testWithLeakChecking("PythonConvertible") { let minusOne: PythonObject = -1 let five: PythonObject = 5 + let intArray: PythonObject = [2, 3] + let dict: PythonObject = ["abc": 7] expectEqual(minusOne, Int(-1).pythonObject) expectEqual(minusOne, Int8(-1).pythonObject) @@ -309,6 +318,11 @@ PythonRuntimeTestSuite.testWithLeakChecking("PythonConvertible") { expectEqual(five, UInt64(5).pythonObject) expectEqual(five, Float(5).pythonObject) expectEqual(five, Double(5).pythonObject) + + expectEqual(intArray, Array([2, 3]).pythonObject) + expectEqual(dict, Dictionary(["abc": 7]).pythonObject) + + expectEqual(intArray, TensorShape(2, 3).pythonObject) } PythonRuntimeTestSuite.testWithLeakChecking("Optional") { From 99205e323247efafb9bdc085894bb6623a2dccdf Mon Sep 17 00:00:00 2001 From: Doug Friedman Date: Wed, 3 Apr 2019 21:39:27 -0400 Subject: [PATCH 2/8] resolve suggestions --- stdlib/public/TensorFlow/TensorShape.swift | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/stdlib/public/TensorFlow/TensorShape.swift b/stdlib/public/TensorFlow/TensorShape.swift index 862f140a2ccbd..9d1543d570219 100644 --- a/stdlib/public/TensorFlow/TensorShape.swift +++ b/stdlib/public/TensorFlow/TensorShape.swift @@ -163,10 +163,13 @@ extension TensorShape : Codable { extension TensorShape: PythonConvertible { public var pythonObject: PythonObject { - return self.dimensions.pythonObject + return dimensions.pythonObject } public init?(_ pythonObject: PythonObject) { - self.init(Array(pythonObject)!) + guard let array = [Int32](pythonObject) else { + return nil + } + self.init(array) } } From 4e84b9267991c2e88342c83e8698914e335a9e44 Mon Sep 17 00:00:00 2001 From: Doug Friedman Date: Fri, 5 Apr 2019 01:01:24 -0400 Subject: [PATCH 3/8] syntax change and add numpy reshape tests --- test/Python/numpy_conversion.swift | 11 +++++++++++ test/Python/python_runtime.swift | 14 +++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/test/Python/numpy_conversion.swift b/test/Python/numpy_conversion.swift index 266f39aab00cc..3e878808d800a 100644 --- a/test/Python/numpy_conversion.swift +++ b/test/Python/numpy_conversion.swift @@ -35,6 +35,17 @@ NumpyConversionTests.test("array-conversion") { let numpyArray2D = np.ones([2, 3]) expectNil(Array(numpy: numpyArray2D)) + let reshaped = np.reshape(numpyArray2D, 6) + if let array = expectNotNil(Array(numpy: reshaped)) { + expectEqual([1.0, 1.0, 1.0, 1.0, 1.0, 1.0], array) + } + + let numpyArray1D = np.ones(28) + let reshaped3D = np.reshape(numpyArray1D, [2, 7, 2]) + expectEqual(Array(reshaped3D.shape), [2, 7, 2]) + let reshaped2D = np.reshape(reshaped3D, [14, 2]) + expectEqual(Array(reshaped2D.shape), [14, 2]) + let numpyArrayStrided = np.array([[1, 2], [1, 2]], dtype: np.int32)[ Python.slice(Python.None), 1] // Assert that the array has a stride, so that we're certainly testing a diff --git a/test/Python/python_runtime.swift b/test/Python/python_runtime.swift index 342d5a06f966d..51f0055d6f24a 100644 --- a/test/Python/python_runtime.swift +++ b/test/Python/python_runtime.swift @@ -67,7 +67,7 @@ PythonRuntimeTestSuite.testWithLeakChecking("PythonList") { } PythonRuntimeTestSuite.testWithLeakChecking("PythonDict") { - let dict: PythonObject = ["a": 1, 1: 0.5] + let dict: PythonObject = ["a" : 1, 1 : 0.5] expectEqual(2, Python.len(dict)) expectEqual(1, dict["a"]) expectEqual(0.5, dict[1]) @@ -211,7 +211,7 @@ PythonRuntimeTestSuite.testWithLeakChecking("Tuple") { let element1: PythonObject = 0 let element2: PythonObject = "abc" let element3: PythonObject = [0, 0] - let element4: PythonObject = ["a": 0, "b": "c"] + let element4: PythonObject = ["a" : 0, "b" : "c"] let pair = PythonObject(tupleOf: element1, element2) let (pair1, pair2) = pair.tuple2 expectEqual(element1, pair1) @@ -260,7 +260,7 @@ PythonRuntimeTestSuite.testWithLeakChecking("ConvertibleFromPython") { let half: PythonObject = 0.5 let string: PythonObject = "abc" let intArray: PythonObject = [2, 3] - let dict: PythonObject = ["abc": 97] + let dict: PythonObject = ["abc" : 97] expectEqual(-1, Int(minusOne)) expectEqual(-1, Int8(minusOne)) @@ -290,7 +290,7 @@ PythonRuntimeTestSuite.testWithLeakChecking("ConvertibleFromPython") { expectEqual([2, 3], Array(intArray)) expectEqual(TensorShape(2, 3), TensorShape(intArray)) - expectEqual(["abc": 97], Dictionary(dict)) + expectEqual(["abc" : 97], Dictionary(dict)) expectNil(String(zero)) expectNil(Int(string)) @@ -301,7 +301,7 @@ PythonRuntimeTestSuite.testWithLeakChecking("PythonConvertible") { let minusOne: PythonObject = -1 let five: PythonObject = 5 let intArray: PythonObject = [2, 3] - let dict: PythonObject = ["abc": 7] + let dict: PythonObject = ["abc" : 7] expectEqual(minusOne, Int(-1).pythonObject) expectEqual(minusOne, Int8(-1).pythonObject) @@ -319,8 +319,8 @@ PythonRuntimeTestSuite.testWithLeakChecking("PythonConvertible") { expectEqual(five, Float(5).pythonObject) expectEqual(five, Double(5).pythonObject) - expectEqual(intArray, Array([2, 3]).pythonObject) - expectEqual(dict, Dictionary(["abc": 7]).pythonObject) + expectEqual(intArray, [2, 3].pythonObject) + expectEqual(dict, ["abc" : 7].pythonObject) expectEqual(intArray, TensorShape(2, 3).pythonObject) } From cd57ddbd3d6831261ab0c2d205cf302c4ab126f4 Mon Sep 17 00:00:00 2001 From: Doug Friedman Date: Fri, 5 Apr 2019 01:15:36 -0400 Subject: [PATCH 4/8] add np.reshape test to ShapedArray tests --- test/TensorFlowRuntime/numpy_conversion.swift | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/TensorFlowRuntime/numpy_conversion.swift b/test/TensorFlowRuntime/numpy_conversion.swift index 7788b08e85ca5..3b2a6526511c7 100644 --- a/test/TensorFlowRuntime/numpy_conversion.swift +++ b/test/TensorFlowRuntime/numpy_conversion.swift @@ -52,6 +52,12 @@ NumpyConversionTests.test("shaped-array-conversion") { array) } + let reshaped = np.reshape(numpyArrayInt32, [2, 3]) + if let array = expectNotNil(ShapedArray(numpy: reshaped)) { + expectEqual(ShapedArray(shape: [2, 3], scalars: [1, 2, 3, 4, 5, 6]), + array) + } + let numpyArrayStrided = np.array([[1, 2], [1, 2]], dtype: np.int32)[ Python.slice(Python.None), 1] // Assert that the array has a stride, so that we're certainly testing a @@ -95,6 +101,12 @@ NumpyConversionTests.test("tensor-conversion") { tensor.array) } + let reshaped = np.reshape(numpyArrayInt32, [2, 3]) + if let tensor = expectNotNil(Tensor(numpy: reshaped)) { + expectEqual(ShapedArray(shape: [2, 3], scalars: [1, 2, 3, 4, 5, 6]), + tensor.array) + } + let numpyArrayStrided = np.array([[1, 2], [1, 2]], dtype: np.int32)[ Python.slice(Python.None), 1] // Assert that the array has a stride, so that we're certainly testing a From 291d9655269c6c43623f318971aa0400f51aea30 Mon Sep 17 00:00:00 2001 From: Doug Friedman Date: Fri, 5 Apr 2019 01:29:00 -0400 Subject: [PATCH 5/8] use TensorShape instead of Array --- test/Python/numpy_conversion.swift | 9 +++++---- test/TensorFlowRuntime/numpy_conversion.swift | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test/Python/numpy_conversion.swift b/test/Python/numpy_conversion.swift index 3e878808d800a..b488db348c422 100644 --- a/test/Python/numpy_conversion.swift +++ b/test/Python/numpy_conversion.swift @@ -4,6 +4,7 @@ // `numpy.ndarray` conversion tests. import Python +import struct TensorFlow.TensorShape import StdlibUnittest var NumpyConversionTests = TestSuite("NumpyConversion") @@ -41,10 +42,10 @@ NumpyConversionTests.test("array-conversion") { } let numpyArray1D = np.ones(28) - let reshaped3D = np.reshape(numpyArray1D, [2, 7, 2]) - expectEqual(Array(reshaped3D.shape), [2, 7, 2]) - let reshaped2D = np.reshape(reshaped3D, [14, 2]) - expectEqual(Array(reshaped2D.shape), [14, 2]) + let reshaped3D = np.reshape(numpyArray1D, [2, 7, 2] as TensorShape) + expectEqual(TensorShape(reshaped3D.shape), [2, 7, 2]) + let reshaped2D = np.reshape(reshaped3D, [14, 2] as TensorShape) + expectEqual(TensorShape(reshaped2D.shape), [14, 2]) let numpyArrayStrided = np.array([[1, 2], [1, 2]], dtype: np.int32)[ Python.slice(Python.None), 1] diff --git a/test/TensorFlowRuntime/numpy_conversion.swift b/test/TensorFlowRuntime/numpy_conversion.swift index 3b2a6526511c7..8ad1f0f0b6d4e 100644 --- a/test/TensorFlowRuntime/numpy_conversion.swift +++ b/test/TensorFlowRuntime/numpy_conversion.swift @@ -52,7 +52,7 @@ NumpyConversionTests.test("shaped-array-conversion") { array) } - let reshaped = np.reshape(numpyArrayInt32, [2, 3]) + let reshaped = np.reshape(numpyArrayInt32, [2, 3] as TensorShape) if let array = expectNotNil(ShapedArray(numpy: reshaped)) { expectEqual(ShapedArray(shape: [2, 3], scalars: [1, 2, 3, 4, 5, 6]), array) @@ -101,7 +101,7 @@ NumpyConversionTests.test("tensor-conversion") { tensor.array) } - let reshaped = np.reshape(numpyArrayInt32, [2, 3]) + let reshaped = np.reshape(numpyArrayInt32, [2, 3] as TensorShape) if let tensor = expectNotNil(Tensor(numpy: reshaped)) { expectEqual(ShapedArray(shape: [2, 3], scalars: [1, 2, 3, 4, 5, 6]), tensor.array) From 38003a25288776ffe0e924739419deabbc1846a5 Mon Sep 17 00:00:00 2001 From: Doug Friedman Date: Fri, 5 Apr 2019 01:45:47 -0400 Subject: [PATCH 6/8] move all tests to TensorFlowRuntime --- test/Python/numpy_conversion.swift | 12 ------------ test/TensorFlowRuntime/numpy_conversion.swift | 6 ++++++ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/test/Python/numpy_conversion.swift b/test/Python/numpy_conversion.swift index b488db348c422..266f39aab00cc 100644 --- a/test/Python/numpy_conversion.swift +++ b/test/Python/numpy_conversion.swift @@ -4,7 +4,6 @@ // `numpy.ndarray` conversion tests. import Python -import struct TensorFlow.TensorShape import StdlibUnittest var NumpyConversionTests = TestSuite("NumpyConversion") @@ -36,17 +35,6 @@ NumpyConversionTests.test("array-conversion") { let numpyArray2D = np.ones([2, 3]) expectNil(Array(numpy: numpyArray2D)) - let reshaped = np.reshape(numpyArray2D, 6) - if let array = expectNotNil(Array(numpy: reshaped)) { - expectEqual([1.0, 1.0, 1.0, 1.0, 1.0, 1.0], array) - } - - let numpyArray1D = np.ones(28) - let reshaped3D = np.reshape(numpyArray1D, [2, 7, 2] as TensorShape) - expectEqual(TensorShape(reshaped3D.shape), [2, 7, 2]) - let reshaped2D = np.reshape(reshaped3D, [14, 2] as TensorShape) - expectEqual(TensorShape(reshaped2D.shape), [14, 2]) - let numpyArrayStrided = np.array([[1, 2], [1, 2]], dtype: np.int32)[ Python.slice(Python.None), 1] // Assert that the array has a stride, so that we're certainly testing a diff --git a/test/TensorFlowRuntime/numpy_conversion.swift b/test/TensorFlowRuntime/numpy_conversion.swift index 8ad1f0f0b6d4e..66d3419d62f08 100644 --- a/test/TensorFlowRuntime/numpy_conversion.swift +++ b/test/TensorFlowRuntime/numpy_conversion.swift @@ -58,6 +58,12 @@ NumpyConversionTests.test("shaped-array-conversion") { array) } + let numpyArray1D = np.ones(28) + let reshaped3D = np.reshape(numpyArray1D, [2, 7, 2] as TensorShape) + expectEqual(TensorShape(reshaped3D.shape), [2, 7, 2]) + let reshaped2D = np.reshape(reshaped3D, [14, 2] as TensorShape) + expectEqual(TensorShape(reshaped2D.shape), [14, 2]) + let numpyArrayStrided = np.array([[1, 2], [1, 2]], dtype: np.int32)[ Python.slice(Python.None), 1] // Assert that the array has a stride, so that we're certainly testing a From 8faa293cbbcc8ee7388aa8766baf0295af96a29a Mon Sep 17 00:00:00 2001 From: Doug Friedman Date: Fri, 5 Apr 2019 02:00:43 -0400 Subject: [PATCH 7/8] mv TensorShape test --- test/Python/python_runtime.swift | 4 ---- test/TensorFlowRuntime/numpy_conversion.swift | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/Python/python_runtime.swift b/test/Python/python_runtime.swift index 51f0055d6f24a..5dbbc239aa4c5 100644 --- a/test/Python/python_runtime.swift +++ b/test/Python/python_runtime.swift @@ -5,7 +5,6 @@ import Python import StdlibUnittest -import struct TensorFlow.TensorShape /// The gc module is an interface to the Python garbage collector. let gc = Python.import("gc") @@ -289,7 +288,6 @@ PythonRuntimeTestSuite.testWithLeakChecking("ConvertibleFromPython") { expectEqual("abc", String(string)) expectEqual([2, 3], Array(intArray)) - expectEqual(TensorShape(2, 3), TensorShape(intArray)) expectEqual(["abc" : 97], Dictionary(dict)) expectNil(String(zero)) @@ -321,8 +319,6 @@ PythonRuntimeTestSuite.testWithLeakChecking("PythonConvertible") { expectEqual(intArray, [2, 3].pythonObject) expectEqual(dict, ["abc" : 7].pythonObject) - - expectEqual(intArray, TensorShape(2, 3).pythonObject) } PythonRuntimeTestSuite.testWithLeakChecking("Optional") { diff --git a/test/TensorFlowRuntime/numpy_conversion.swift b/test/TensorFlowRuntime/numpy_conversion.swift index 66d3419d62f08..b27b7002d3009 100644 --- a/test/TensorFlowRuntime/numpy_conversion.swift +++ b/test/TensorFlowRuntime/numpy_conversion.swift @@ -136,6 +136,12 @@ NumpyConversionTests.test("tensor-round-trip") { let t3 = Tensor(repeating: 30, shape: [8,5,4]) expectEqual(t3, Tensor(numpy: t3.makeNumpyArray())!) } + +NumpyConversionTests.test("tensor-shape") { + let pyArray = [2, 3].pythonObject + expectEqual(pyArray, TensorShape(2, 3).pythonObject) + expectEqual(TensorShape(2, 3), TensorShape(pyArray)) +} #endif runAllTests() From 89a3f616eae098772256df56959b7551912c3425 Mon Sep 17 00:00:00 2001 From: Doug Friedman Date: Mon, 8 Apr 2019 23:02:27 -0400 Subject: [PATCH 8/8] add support for noniterable arg and step around unrelated test failure --- stdlib/public/TensorFlow/TensorShape.swift | 12 ++++++++---- test/TensorFlow/integration.swift | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/stdlib/public/TensorFlow/TensorShape.swift b/stdlib/public/TensorFlow/TensorShape.swift index 9d1543d570219..68a16136f30bd 100644 --- a/stdlib/public/TensorFlow/TensorShape.swift +++ b/stdlib/public/TensorFlow/TensorShape.swift @@ -161,15 +161,19 @@ extension TensorShape : Codable { } } -extension TensorShape: PythonConvertible { +extension TensorShape : PythonConvertible { public var pythonObject: PythonObject { return dimensions.pythonObject } public init?(_ pythonObject: PythonObject) { - guard let array = [Int32](pythonObject) else { - return nil + let hasLen = Bool(Python.hasattr(pythonObject, "__len__")) + if(hasLen == true) { + guard let array = [Int32](pythonObject) else { return nil } + self.init(array) + } else { + guard let num = Int32(pythonObject) else { return nil } + self.init(num) } - self.init(array) } } diff --git a/test/TensorFlow/integration.swift b/test/TensorFlow/integration.swift index bb21daba09f79..ed1fd83063c1a 100644 --- a/test/TensorFlow/integration.swift +++ b/test/TensorFlow/integration.swift @@ -377,7 +377,7 @@ public func testResourceAndVariants() { // expected-error @+1 {{op named 'TensorDataSet' is not registered in TensorFlow}} #tfop("TensorDataSet", values, Toutput_types$dtype: [Float.tensorFlowDataType], - output_shapes: [TensorShape(1)]) + output_shapes: [TensorShape([1])]) // REGISTER_OP("Iterator") // .Output("handle: resource") @@ -388,7 +388,7 @@ public func testResourceAndVariants() { // .SetShapeFn(shape_inference::ScalarShape); let iterator: ResourceHandle = #tfop("Iterator", shared_name: "foo", container: "bar", - output_types$dtype: [Float.tensorFlowDataType], output_shapes: [TensorShape(1)]) + output_types$dtype: [Float.tensorFlowDataType], output_shapes: [TensorShape([1])]) // REGISTER_OP("MakeIterator") // .Input("dataset: variant")