From ee905cf9063465dfc7d48be027d57cd30f2604f7 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Tue, 21 May 2019 10:46:37 -0700
Subject: [PATCH 01/28] Initial implementation of Complex numbers.

---
 Sources/TensorFlow/Core/Complex.swift | 396 ++++++++++++++++++++++++++
 1 file changed, 396 insertions(+)
 create mode 100644 Sources/TensorFlow/Core/Complex.swift

diff --git a/Sources/TensorFlow/Core/Complex.swift b/Sources/TensorFlow/Core/Complex.swift
new file mode 100644
index 000000000..41faee7e1
--- /dev/null
+++ b/Sources/TensorFlow/Core/Complex.swift
@@ -0,0 +1,396 @@
+import TensorFlow
+
+//
+//  Complex.swift
+//  NumericAnnex
+//
+//  Created by Xiaodi Wu on 3/25/17.
+//
+//  Note
+//  ====
+//
+//  For maximum consistency with corresponding functions in C/C++, checks for
+//  special values in `naturalExponential()`, `squareRoot()`, trigonometric
+//  functions, and hyperbolic functions are adapted from libc++.
+//
+//  Code in libc++ is dual-licensed under the MIT and UIUC/NCSA licenses.
+//  Copyright © 2009-2017 contributors to the LLVM/libc++ project.
+
+/// A type to represent a complex value in Cartesian form.
+///
+/// Additional Considerations
+/// -------------------------
+///
+/// Floating-point types have special values that represent infinity or NaN
+/// ("not a number"). Complex functions in different languages may return
+/// different results when working with special values.
+///
+/// Implementations in `Complex<T>` adhere to the [C standard][std] (Annex G) as
+/// closely as possible with respect to special values and branch cuts.
+///
+/// To users unfamiliar with complex functions, the principal value returned by
+/// some complex functions may be unexpected. For example,
+/// `Double.cbrt(-8) == -2`, which is the __real root__, while
+/// `Complex.cbrt(-8) == 2 * Complex.exp(.i * .pi / 3)`, which is the
+/// __principal root__.
+///
+/// [dfn]: http://mathworld.wolfram.com/BranchCut.html
+/// [std]: http://www.open-std.org/JTC1/SC22/WG14/www/standards.html#9899
+@_fixed_layout
+public struct Complex<T : FloatingPoint> : Codable
+where T : Codable & _ExpressibleByBuiltinFloatLiteral {
+  // ---------------------------------------------------------------------------
+  // MARK: Stored Properties
+  // ---------------------------------------------------------------------------
+
+  /// The real component of the complex value.
+  public var real: T
+
+  /// The imaginary component of the complex value.
+  public var imaginary: T
+
+  // ---------------------------------------------------------------------------
+  // MARK: Initializers
+  // ---------------------------------------------------------------------------
+  public init(real: T = 0, imaginary: T = 0) {
+	self.real = real
+	self.imaginary = imaginary
+  }
+}
+
+extension Complex {
+  // ---------------------------------------------------------------------------
+  // MARK: Static Properties
+  // ---------------------------------------------------------------------------
+
+  /// The imaginary unit _i_.
+  @_transparent // @_inlineable
+  public static var i: Complex {
+    return Complex(real: 0, imaginary: 1)
+  }
+
+  /// A Boolean value indicating whether the instance is finite.
+  ///
+  /// A complex value is finite if its real and imaginary components are both
+  /// finite. A component is finite if it is not infinity or NaN.
+  @_transparent // @_inlineable
+  public var isFinite: Bool {
+    return real.isFinite && imaginary.isFinite
+  }
+
+  /// A Boolean value indicating whether the instance is infinite.
+  ///
+  /// A complex value is infinite if at least one of its components (real or
+  /// imaginary) is infinite, even if the other component is NaN.
+  ///
+  /// Note that `isFinite` and `isInfinite` do not form a dichotomy because NaN
+  /// is neither finite nor infinite.
+  @_transparent // @_inlineable
+  public var isInfinite: Bool {
+    return real.isInfinite || imaginary.isInfinite
+  }
+
+  /// A Boolean value indicating whether the instance is NaN ("not a number").
+  ///
+  /// A complex value is NaN if at least one of its components (real or
+  /// imaginary) is NaN and the other component is not infinite.
+  ///
+  /// Because NaN is not equal to any value, including NaN, use this property
+  /// instead of the equal-to operator (`==`) or not-equal-to operator (`!=`) to
+  /// test whether a value is or is not NaN.
+  ///
+  /// This property is `true` for both quiet and signaling NaNs.
+  @_transparent // @_inlineable
+  public var isNaN: Bool {
+    return (real.isNaN && !imaginary.isInfinite) ||
+      (imaginary.isNaN && !real.isInfinite)
+  }
+
+  /// A Boolean value indicating whether the instance is equal to zero.
+  ///
+  /// A complex value is equal to zero if its real and imaginary components both
+  /// represent either `-0.0` or `+0.0`.
+  @_transparent // @_inlineable
+  public var isZero: Bool {
+    return real.isZero && imaginary.isZero
+  }
+}
+
+extension Complex : ExpressibleByIntegerLiteral {
+  // ---------------------------------------------------------------------------
+  // MARK: ExpressibleByIntegerLiteral
+  // ---------------------------------------------------------------------------
+
+  @_transparent // @_inlineable
+  public init(integerLiteral value: Int) {
+    self.real = T(value)
+    self.imaginary = 0
+  }
+}
+
+extension Complex : CustomStringConvertible {
+  // ---------------------------------------------------------------------------
+  // MARK: CustomStringConvertible
+  // ---------------------------------------------------------------------------
+
+  @_transparent // @_inlineable
+  public var description: String {
+    return real.isNaN && real.sign == .minus
+      // At present, -NaN is described as "nan", which is acceptable for real
+      // values. However, it is arguably misleading to describe -NaN - NaNi as
+      // "nan + nani" or "nan - nani". Therefore, handle this case separately.
+      ? imaginary.sign == .minus
+        ? "-\(-real) - \(-imaginary)i"
+        : "-\(-real) + \(imaginary)i"
+      : imaginary.sign == .minus
+        ? "\(real) - \(-imaginary)i"
+        : "\(real) + \(imaginary)i"
+  }
+}
+
+extension Complex : Equatable {
+  // ---------------------------------------------------------------------------
+  // MARK: Equatable
+  // ---------------------------------------------------------------------------
+
+  @_transparent // @_inlineable
+  public static func == (lhs: Complex, rhs: Complex) -> Bool {
+    return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
+  }
+}
+
+extension Complex : AdditiveArithmetic {
+  // ---------------------------------------------------------------------------
+  // MARK: AdditiveArithmetic
+  // ---------------------------------------------------------------------------
+
+  @_transparent // @_inlineable
+  @differentiable(vjp: _vjpAdd(lhs:rhs:))
+  public static func + (lhs: Complex, rhs: Complex) -> Complex {
+    var lhs = lhs
+    lhs += rhs
+    return lhs
+  }
+
+  @_transparent // @_inlineable
+  public static func += (lhs: inout Complex, rhs: Complex) {
+    lhs.real += rhs.real
+    lhs.imaginary += rhs.imaginary
+  }
+
+  @_transparent // @_inlineable
+  @differentiable(vjp: _vjpSubtract(lhs:rhs:))
+  public static func - (lhs: Complex, rhs: Complex) -> Complex {
+    var lhs = lhs
+    lhs -= rhs
+    return lhs
+  }
+
+  @_transparent // @_inlineable
+  public static func -= (lhs: inout Complex, rhs: Complex) {
+    lhs.real -= rhs.real
+    lhs.imaginary -= rhs.imaginary
+  }
+}
+
+extension Complex : Numeric {
+  // ---------------------------------------------------------------------------
+  // MARK: Numeric
+  // ---------------------------------------------------------------------------
+
+  public init?<U>(exactly source: U) where U : BinaryInteger {
+    guard let t = T(exactly: source) else { return nil }
+    self.real = t
+    self.imaginary = 0
+  }
+
+  @_transparent // @_inlineable
+  @differentiable(vjp: _vjpMultiply(lhs:rhs:))
+  public static func * (lhs: Complex, rhs: Complex) -> Complex {
+    var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
+    let ac = a * c, bd = b * d, ad = a * d, bc = b * c
+    let x = ac - bd
+    let y = ad + bc
+    // Recover infinities that computed as NaN + iNaN.
+    // See C11 Annex G.
+    if x.isNaN && y.isNaN {
+      var recalculate = false
+      if a.isInfinite || b.isInfinite {
+        // "Box" the infinity and change NaNs in the other operand to 0.
+        a = T(signOf: a, magnitudeOf: a.isInfinite ? 1 : 0)
+        b = T(signOf: b, magnitudeOf: b.isInfinite ? 1 : 0)
+        if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
+        if d.isNaN { d = T(signOf: d, magnitudeOf: 0) }
+        recalculate = true
+      }
+      if c.isInfinite || d.isInfinite {
+        // "Box" the infinity and change NaNs in the other operand to 0.
+        if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
+        if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
+        c = T(signOf: c, magnitudeOf: c.isInfinite ? 1 : 0)
+        d = T(signOf: d, magnitudeOf: d.isInfinite ? 1 : 0)
+        recalculate = true
+      }
+      if !recalculate &&
+        (ac.isInfinite || bd.isInfinite || ad.isInfinite || bc.isInfinite) {
+        // Recover infinities from overflow by changing NaNs to 0.
+        if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
+        if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
+        if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
+        if d.isNaN { d = T(signOf: d, magnitudeOf: 0) }
+        recalculate = true
+      }
+      if recalculate {
+        return Complex(
+          real: .infinity * (a * c - b * d),
+          imaginary: .infinity * (a * d + b * c)
+        )
+      }
+    }
+    return Complex(real: x, imaginary: y)
+  }
+
+  @_transparent // @_inlineable
+  public static func *= (lhs: inout Complex, rhs: Complex) {
+    lhs = lhs * rhs
+  }
+
+  @_transparent // @_inlineable
+  public var magnitude: T {
+    var x = abs(real)
+    var y = abs(imaginary)
+    if x.isInfinite { return x }
+    if y.isInfinite { return y }
+    if x == 0 { return y }
+    if x < y { swap(&x, &y) }
+    let ratio = y / x
+    return x * (1 + ratio * ratio).squareRoot()
+  }
+}
+
+extension Complex : SignedNumeric {
+  // ---------------------------------------------------------------------------
+  // MARK: SignedNumeric
+  // ---------------------------------------------------------------------------
+
+  @_transparent // @_inlineable
+  @differentiable(vjp: _vjpNegate)
+  public static prefix func - (operand: Complex) -> Complex {
+    return Complex(real: -operand.real, imaginary: -operand.imaginary)
+  }
+
+  @_transparent // @_inlineable
+  public mutating func negate() {
+    real.negate()
+    imaginary.negate()
+  }
+}
+
+extension Complex {
+
+  @_transparent // @_inlineable
+  @differentiable(vjp: _vjpDivide(lhs:rhs:))
+  public static func / (lhs: Complex, rhs: Complex) -> Complex {
+    var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
+    var x: T
+    var y: T
+    // Prevent avoidable overflow; see Numerical Recipes.
+    if c.magnitude >= d.magnitude {
+      let ratio = d / c
+      let denominator = c + d * ratio
+      x = (a + b * ratio) / denominator
+      y = (b - a * ratio) / denominator
+    } else {
+      let ratio = c / d
+      let denominator = c * ratio + d
+      x = (a * ratio + b) / denominator
+      y = (b * ratio - a) / denominator
+    }
+    // Recover infinities and zeros that computed as NaN + iNaN.
+    // See C11 Annex G.
+    if x.isNaN && y.isNaN {
+      if c == 0 && d == 0 && (!a.isNaN || !b.isNaN) {
+        x = T(signOf: c, magnitudeOf: .infinity) * a
+        y = T(signOf: c /* sic */, magnitudeOf: .infinity) * b
+      } else if (a.isInfinite || b.isInfinite) && c.isFinite && d.isFinite {
+        a = T(signOf: a, magnitudeOf: a.isInfinite ? 1 : 0)
+        b = T(signOf: b, magnitudeOf: b.isInfinite ? 1 : 0)
+        x = .infinity * (a * c + b * d)
+        y = .infinity * (b * c - a * d)
+      } else if (c.isInfinite || d.isInfinite) && a.isFinite && b.isFinite {
+        c = T(signOf: c, magnitudeOf: c.isInfinite ? 1 : 0)
+        d = T(signOf: d, magnitudeOf: d.isInfinite ? 1 : 0)
+        x = 0 * (a * c + b * d)
+        y = 0 * (b * c - a * d)
+      }
+    }
+    return Complex(real: x, imaginary: y)
+  }
+
+  @_transparent // @_inlineable
+  public static func /= (lhs: inout Complex, rhs: Complex) {
+    lhs = lhs / rhs
+  }
+}
+
+extension Complex {
+  func complexConjugate() -> Complex {
+    return Complex(real: real, imaginary: -imaginary)
+  }
+}
+
+/// Returns the absolute value (magnitude, modulus) of `z`.
+@_transparent
+public func abs<T>(_ z: Complex<T>) -> Complex<T> {
+  return Complex(real: z.magnitude)
+}
+
+extension Complex : Differentiable where T : Differentiable {
+  public typealias TangentVector = Complex
+  public typealias CotangentVector = Complex
+  public typealias AllDifferentiableVariables = Complex
+  public func tangentVector(from cotangent: CotangentVector) -> TangentVector {
+    return cotangent
+  }
+}
+
+// extension Complex where T : Differentiable {
+//     @inlinable
+//   static func _vjpInit(real: T, imaginary: T) -> (Complex, (Complex.CotangentVector) -> (T.CotangentVector, T.CotangentVector)) {
+//     return (Complex(real: real, imaginary: imaginary), { (v: Complex) in
+//       return (v.CotangentVector.real, v.CotangentVector.imaginary) 
+//     })
+//   }
+// }
+
+extension Complex {
+  @inlinable
+  static func _vjpAdd(lhs: Complex, rhs: Complex) 
+  -> (Complex, (Complex) -> (Complex, Complex)) {
+    return (lhs * rhs, { v in (v, v) })
+  }
+
+  @inlinable
+  static func _vjpSubtract(lhs: Complex, rhs: Complex) 
+  -> (Complex, (Complex) -> (Complex, Complex)) {
+    return (lhs * rhs, { v in (v, -v) })
+  }
+
+  @inlinable
+  static func _vjpMultiply(lhs: Complex, rhs: Complex) 
+  -> (Complex, (Complex) -> (Complex, Complex)) {
+    return (lhs * rhs, { v in (rhs * v, lhs * v) })
+  }
+
+  @inlinable
+  static func _vjpDivide(lhs: Complex, rhs: Complex) 
+  -> (Complex, (Complex) -> (Complex, Complex)) {
+    return (lhs * rhs, { v in (v / rhs, -lhs / (rhs * rhs) * v) })
+  }
+
+  @inlinable
+  static func _vjpNegate (operand: Complex)
+  -> (Complex, (Complex) -> Complex) {
+    return (-operand, { v in -v})
+  }
+}
\ No newline at end of file

From 9c2d11b7c659fc06f52b94e31b5674f30246d333 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Tue, 21 May 2019 11:11:32 -0700
Subject: [PATCH 02/28] WIP: too strict conformances.

---
 Sources/TensorFlow/Core/Complex.swift | 35 +++++++++++++++------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/Sources/TensorFlow/Core/Complex.swift b/Sources/TensorFlow/Core/Complex.swift
index 41faee7e1..7dfcbc1d6 100644
--- a/Sources/TensorFlow/Core/Complex.swift
+++ b/Sources/TensorFlow/Core/Complex.swift
@@ -37,8 +37,7 @@ import TensorFlow
 /// [dfn]: http://mathworld.wolfram.com/BranchCut.html
 /// [std]: http://www.open-std.org/JTC1/SC22/WG14/www/standards.html#9899
 @_fixed_layout
-public struct Complex<T : FloatingPoint> : Codable
-where T : Codable & _ExpressibleByBuiltinFloatLiteral {
+public struct Complex<T : TensorFlowFloatingPoint> {
   // ---------------------------------------------------------------------------
   // MARK: Stored Properties
   // ---------------------------------------------------------------------------
@@ -52,6 +51,7 @@ where T : Codable & _ExpressibleByBuiltinFloatLiteral {
   // ---------------------------------------------------------------------------
   // MARK: Initializers
   // ---------------------------------------------------------------------------
+  @differentiable(wrt: (real, imaginary), vjp: _vjpInit)
   public init(real: T = 0, imaginary: T = 0) {
 	self.real = real
 	self.imaginary = imaginary
@@ -165,7 +165,7 @@ extension Complex : AdditiveArithmetic {
   // ---------------------------------------------------------------------------
 
   @_transparent // @_inlineable
-  @differentiable(vjp: _vjpAdd(lhs:rhs:))
+  @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
   public static func + (lhs: Complex, rhs: Complex) -> Complex {
     var lhs = lhs
     lhs += rhs
@@ -179,7 +179,7 @@ extension Complex : AdditiveArithmetic {
   }
 
   @_transparent // @_inlineable
-  @differentiable(vjp: _vjpSubtract(lhs:rhs:))
+  @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
   public static func - (lhs: Complex, rhs: Complex) -> Complex {
     var lhs = lhs
     lhs -= rhs
@@ -205,7 +205,7 @@ extension Complex : Numeric {
   }
 
   @_transparent // @_inlineable
-  @differentiable(vjp: _vjpMultiply(lhs:rhs:))
+  @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable)
   public static func * (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
     let ac = a * c, bd = b * d, ad = a * d, bc = b * c
@@ -274,7 +274,7 @@ extension Complex : SignedNumeric {
   // ---------------------------------------------------------------------------
 
   @_transparent // @_inlineable
-  @differentiable(vjp: _vjpNegate)
+  @differentiable(vjp: _vjpNegate where T : Differentiable)
   public static prefix func - (operand: Complex) -> Complex {
     return Complex(real: -operand.real, imaginary: -operand.imaginary)
   }
@@ -289,7 +289,7 @@ extension Complex : SignedNumeric {
 extension Complex {
 
   @_transparent // @_inlineable
-  @differentiable(vjp: _vjpDivide(lhs:rhs:))
+  @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
   public static func / (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
     var x: T
@@ -354,16 +354,19 @@ extension Complex : Differentiable where T : Differentiable {
   }
 }
 
-// extension Complex where T : Differentiable {
-//     @inlinable
-//   static func _vjpInit(real: T, imaginary: T) -> (Complex, (Complex.CotangentVector) -> (T.CotangentVector, T.CotangentVector)) {
-//     return (Complex(real: real, imaginary: imaginary), { (v: Complex) in
-//       return (v.CotangentVector.real, v.CotangentVector.imaginary) 
-//     })
-//   }
-// }
-
 extension Complex {
+
+  @inlinable
+  static func _vjpInit(real: T, imaginary: T) -> (Complex<T>, (Complex<T>) -> (T, T)) {
+    // let orig: Complex<T> = Complex(real: real, imaginary: imaginary)
+    // let pb: (Complex) -> (T, T) = { v in
+    //   return (v.real, v.imaginary) 
+    // }
+    return (Complex(real: real, imaginary: imaginary), { v in
+      return (v.real, v.imaginary) 
+    })
+  }
+
   @inlinable
   static func _vjpAdd(lhs: Complex, rhs: Complex) 
   -> (Complex, (Complex) -> (Complex, Complex)) {

From 346b9b73d9184e2fe837dc8c0b3e257c67532b81 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Tue, 21 May 2019 15:22:48 -0700
Subject: [PATCH 03/28] Force complex numbers to be differentiable always due
 to crash.

---
 Sources/TensorFlow/Core/Complex.swift | 67 +++++++++++++++++++++++----
 1 file changed, 58 insertions(+), 9 deletions(-)

diff --git a/Sources/TensorFlow/Core/Complex.swift b/Sources/TensorFlow/Core/Complex.swift
index 7dfcbc1d6..46a5e20a1 100644
--- a/Sources/TensorFlow/Core/Complex.swift
+++ b/Sources/TensorFlow/Core/Complex.swift
@@ -37,15 +37,17 @@ import TensorFlow
 /// [dfn]: http://mathworld.wolfram.com/BranchCut.html
 /// [std]: http://www.open-std.org/JTC1/SC22/WG14/www/standards.html#9899
 @_fixed_layout
-public struct Complex<T : TensorFlowFloatingPoint> {
+public struct Complex<T : TensorFlowFloatingPoint> : Differentiable {
   // ---------------------------------------------------------------------------
   // MARK: Stored Properties
   // ---------------------------------------------------------------------------
 
   /// The real component of the complex value.
+  // @differentiable(vjp: _vjpReal)
   public var real: T
 
   /// The imaginary component of the complex value.
+  // @differentiable(vjp: _vjpImaginary)
   public var imaginary: T
 
   // ---------------------------------------------------------------------------
@@ -56,6 +58,13 @@ public struct Complex<T : TensorFlowFloatingPoint> {
 	self.real = real
 	self.imaginary = imaginary
   }
+
+  public typealias TangentVector = Complex
+  public typealias CotangentVector = Complex
+  public typealias AllDifferentiableVariables = Complex
+  public func tangentVector(from cotangent: CotangentVector) -> TangentVector {
+    return cotangent
+  }
 }
 
 extension Complex {
@@ -345,17 +354,37 @@ public func abs<T>(_ z: Complex<T>) -> Complex<T> {
   return Complex(real: z.magnitude)
 }
 
-extension Complex : Differentiable where T : Differentiable {
-  public typealias TangentVector = Complex
-  public typealias CotangentVector = Complex
-  public typealias AllDifferentiableVariables = Complex
-  public func tangentVector(from cotangent: CotangentVector) -> TangentVector {
-    return cotangent
+extension Complex {
+  @differentiable(vjp: _vjpAdding(real:))
+  public func adding(real: T) -> Complex {
+    var c = self
+    c.real += real
+    return c
+  }
+
+  @differentiable(vjp: _vjpSubtracting(real:))
+  public func subtracting(real: T) -> Complex {
+    var c = self
+    c.real -= real
+    return c
+  }
+
+  @differentiable(vjp: _vjpAdding(imaginary:))
+  public func adding(imaginary: T) -> Complex {
+    var c = self
+    c.imaginary += imaginary
+    return c
+  }
+
+  @differentiable(vjp: _vjpSubtracting(imaginary:))
+  public func subtracting(imaginary: T) -> Complex {
+    var c = self
+    c.imaginary -= imaginary
+    return c
   }
 }
 
 extension Complex {
-
   @inlinable
   static func _vjpInit(real: T, imaginary: T) -> (Complex<T>, (Complex<T>) -> (T, T)) {
     // let orig: Complex<T> = Complex(real: real, imaginary: imaginary)
@@ -392,8 +421,28 @@ extension Complex {
   }
 
   @inlinable
-  static func _vjpNegate (operand: Complex)
+  static func _vjpNegate(operand: Complex)
   -> (Complex, (Complex) -> Complex) {
     return (-operand, { v in -v})
   }
+
+  @inlinable
+  func _vjpAdding(real: T) -> (Complex, (Complex) -> (Complex, T)) {
+    return (self.adding(real: real), { ($0, $0.real) })
+  }
+
+  @inlinable
+  func _vjpSubtracting(real: T) -> (Complex, (Complex) -> (Complex, T)) {
+    return (self.subtracting(real: real), { ($0, -$0.real) })
+  }
+
+  @inlinable
+  func _vjpAdding(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
+    return (self.adding(real: real), { ($0, $0.imaginary) })
+  }
+
+  @inlinable
+  func _vjpSubtracting(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
+    return (self.subtracting(real: real), { ($0, -$0.imaginary) })
+  }
 }
\ No newline at end of file

From d5e78cf01f4fccb1e618831be8190cc57b2c5b7b Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Wed, 22 May 2019 13:24:44 -0700
Subject: [PATCH 04/28] Loosen constraints on complex number.

---
 Sources/TensorFlow/Core/Complex.swift | 136 +++++++++-----------------
 1 file changed, 47 insertions(+), 89 deletions(-)

diff --git a/Sources/TensorFlow/Core/Complex.swift b/Sources/TensorFlow/Core/Complex.swift
index 46a5e20a1..2d9199d76 100644
--- a/Sources/TensorFlow/Core/Complex.swift
+++ b/Sources/TensorFlow/Core/Complex.swift
@@ -1,70 +1,31 @@
 import TensorFlow
-
-//
-//  Complex.swift
-//  NumericAnnex
-//
-//  Created by Xiaodi Wu on 3/25/17.
-//
-//  Note
-//  ====
-//
-//  For maximum consistency with corresponding functions in C/C++, checks for
-//  special values in `naturalExponential()`, `squareRoot()`, trigonometric
-//  functions, and hyperbolic functions are adapted from libc++.
-//
-//  Code in libc++ is dual-licensed under the MIT and UIUC/NCSA licenses.
-//  Copyright © 2009-2017 contributors to the LLVM/libc++ project.
-
-/// A type to represent a complex value in Cartesian form.
-///
-/// Additional Considerations
-/// -------------------------
-///
-/// Floating-point types have special values that represent infinity or NaN
-/// ("not a number"). Complex functions in different languages may return
-/// different results when working with special values.
-///
-/// Implementations in `Complex<T>` adhere to the [C standard][std] (Annex G) as
-/// closely as possible with respect to special values and branch cuts.
-///
-/// To users unfamiliar with complex functions, the principal value returned by
-/// some complex functions may be unexpected. For example,
-/// `Double.cbrt(-8) == -2`, which is the __real root__, while
-/// `Complex.cbrt(-8) == 2 * Complex.exp(.i * .pi / 3)`, which is the
-/// __principal root__.
-///
-/// [dfn]: http://mathworld.wolfram.com/BranchCut.html
-/// [std]: http://www.open-std.org/JTC1/SC22/WG14/www/standards.html#9899
-@_fixed_layout
-public struct Complex<T : TensorFlowFloatingPoint> : Differentiable {
+// T : FloatingPoint & Differentiable
+public struct Complex<T : FloatingPoint> {
   // ---------------------------------------------------------------------------
   // MARK: Stored Properties
   // ---------------------------------------------------------------------------
 
   /// The real component of the complex value.
-  // @differentiable(vjp: _vjpReal)
   public var real: T
 
   /// The imaginary component of the complex value.
-  // @differentiable(vjp: _vjpImaginary)
   public var imaginary: T
 
   // ---------------------------------------------------------------------------
   // MARK: Initializers
   // ---------------------------------------------------------------------------
-  @differentiable(wrt: (real, imaginary), vjp: _vjpInit)
   public init(real: T = 0, imaginary: T = 0) {
 	self.real = real
 	self.imaginary = imaginary
   }
+}
 
+extension Complex : Differentiable where T : Differentiable, T.TangentVector == T {
+  // ---------------------------------------------------------------------------
+  // MARK: Differentiability
+  // ---------------------------------------------------------------------------
   public typealias TangentVector = Complex
-  public typealias CotangentVector = Complex
   public typealias AllDifferentiableVariables = Complex
-  public func tangentVector(from cotangent: CotangentVector) -> TangentVector {
-    return cotangent
-  }
 }
 
 extension Complex {
@@ -73,7 +34,7 @@ extension Complex {
   // ---------------------------------------------------------------------------
 
   /// The imaginary unit _i_.
-  @_transparent // @_inlineable
+  @inlinable
   public static var i: Complex {
     return Complex(real: 0, imaginary: 1)
   }
@@ -82,7 +43,7 @@ extension Complex {
   ///
   /// A complex value is finite if its real and imaginary components are both
   /// finite. A component is finite if it is not infinity or NaN.
-  @_transparent // @_inlineable
+  @inlinable
   public var isFinite: Bool {
     return real.isFinite && imaginary.isFinite
   }
@@ -94,7 +55,7 @@ extension Complex {
   ///
   /// Note that `isFinite` and `isInfinite` do not form a dichotomy because NaN
   /// is neither finite nor infinite.
-  @_transparent // @_inlineable
+  @inlinable
   public var isInfinite: Bool {
     return real.isInfinite || imaginary.isInfinite
   }
@@ -109,7 +70,7 @@ extension Complex {
   /// test whether a value is or is not NaN.
   ///
   /// This property is `true` for both quiet and signaling NaNs.
-  @_transparent // @_inlineable
+  @inlinable
   public var isNaN: Bool {
     return (real.isNaN && !imaginary.isInfinite) ||
       (imaginary.isNaN && !real.isInfinite)
@@ -119,7 +80,7 @@ extension Complex {
   ///
   /// A complex value is equal to zero if its real and imaginary components both
   /// represent either `-0.0` or `+0.0`.
-  @_transparent // @_inlineable
+  @inlinable
   public var isZero: Bool {
     return real.isZero && imaginary.isZero
   }
@@ -130,7 +91,7 @@ extension Complex : ExpressibleByIntegerLiteral {
   // MARK: ExpressibleByIntegerLiteral
   // ---------------------------------------------------------------------------
 
-  @_transparent // @_inlineable
+  @inlinable
   public init(integerLiteral value: Int) {
     self.real = T(value)
     self.imaginary = 0
@@ -142,7 +103,7 @@ extension Complex : CustomStringConvertible {
   // MARK: CustomStringConvertible
   // ---------------------------------------------------------------------------
 
-  @_transparent // @_inlineable
+  @inlinable
   public var description: String {
     return real.isNaN && real.sign == .minus
       // At present, -NaN is described as "nan", which is acceptable for real
@@ -162,7 +123,7 @@ extension Complex : Equatable {
   // MARK: Equatable
   // ---------------------------------------------------------------------------
 
-  @_transparent // @_inlineable
+  @inlinable
   public static func == (lhs: Complex, rhs: Complex) -> Bool {
     return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
   }
@@ -173,29 +134,29 @@ extension Complex : AdditiveArithmetic {
   // MARK: AdditiveArithmetic
   // ---------------------------------------------------------------------------
 
-  @_transparent // @_inlineable
-  @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
+  @inlinable
+  @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable, T.TangentVector == T)
   public static func + (lhs: Complex, rhs: Complex) -> Complex {
     var lhs = lhs
     lhs += rhs
     return lhs
   }
 
-  @_transparent // @_inlineable
+  @inlinable
   public static func += (lhs: inout Complex, rhs: Complex) {
     lhs.real += rhs.real
     lhs.imaginary += rhs.imaginary
   }
 
-  @_transparent // @_inlineable
-  @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
+  @inlinable
+  @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable, T.TangentVector == T)
   public static func - (lhs: Complex, rhs: Complex) -> Complex {
     var lhs = lhs
     lhs -= rhs
     return lhs
   }
 
-  @_transparent // @_inlineable
+  @inlinable
   public static func -= (lhs: inout Complex, rhs: Complex) {
     lhs.real -= rhs.real
     lhs.imaginary -= rhs.imaginary
@@ -213,8 +174,8 @@ extension Complex : Numeric {
     self.imaginary = 0
   }
 
-  @_transparent // @_inlineable
-  @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable)
+  @inlinable
+  @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable, T.TangentVector == T)
   public static func * (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
     let ac = a * c, bd = b * d, ad = a * d, bc = b * c
@@ -259,12 +220,12 @@ extension Complex : Numeric {
     return Complex(real: x, imaginary: y)
   }
 
-  @_transparent // @_inlineable
+  @inlinable
   public static func *= (lhs: inout Complex, rhs: Complex) {
     lhs = lhs * rhs
   }
 
-  @_transparent // @_inlineable
+  @inlinable
   public var magnitude: T {
     var x = abs(real)
     var y = abs(imaginary)
@@ -282,13 +243,13 @@ extension Complex : SignedNumeric {
   // MARK: SignedNumeric
   // ---------------------------------------------------------------------------
 
-  @_transparent // @_inlineable
-  @differentiable(vjp: _vjpNegate where T : Differentiable)
+  @inlinable
+  @differentiable(vjp: _vjpNegate where T : Differentiable, T.TangentVector == T)
   public static prefix func - (operand: Complex) -> Complex {
     return Complex(real: -operand.real, imaginary: -operand.imaginary)
   }
 
-  @_transparent // @_inlineable
+  @inlinable
   public mutating func negate() {
     real.negate()
     imaginary.negate()
@@ -296,9 +257,12 @@ extension Complex : SignedNumeric {
 }
 
 extension Complex {
+  // ---------------------------------------------------------------------------
+  // MARK: Division
+  // ---------------------------------------------------------------------------
 
-  @_transparent // @_inlineable
-  @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
+  @inlinable
+  @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable, T.TangentVector == T)
   public static func / (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
     var x: T
@@ -336,47 +300,52 @@ extension Complex {
     return Complex(real: x, imaginary: y)
   }
 
-  @_transparent // @_inlineable
+  @inlinable
   public static func /= (lhs: inout Complex, rhs: Complex) {
     lhs = lhs / rhs
   }
 }
 
 extension Complex {
-  func complexConjugate() -> Complex {
+  @inlinable
+  public func complexConjugate() -> Complex {
     return Complex(real: real, imaginary: -imaginary)
   }
 }
 
 /// Returns the absolute value (magnitude, modulus) of `z`.
-@_transparent
+@inlinable
 public func abs<T>(_ z: Complex<T>) -> Complex<T> {
   return Complex(real: z.magnitude)
 }
 
 extension Complex {
-  @differentiable(vjp: _vjpAdding(real:))
+  @inlinable
+  @differentiable(vjp: _vjpAdding(real:) where T : Differentiable, T.TangentVector == T)
   public func adding(real: T) -> Complex {
     var c = self
     c.real += real
     return c
   }
 
-  @differentiable(vjp: _vjpSubtracting(real:))
+  @inlinable
+  @differentiable(vjp: _vjpSubtracting(real:) where T : Differentiable, T.TangentVector == T)
   public func subtracting(real: T) -> Complex {
     var c = self
     c.real -= real
     return c
   }
 
-  @differentiable(vjp: _vjpAdding(imaginary:))
+  @inlinable
+  @differentiable(vjp: _vjpAdding(imaginary:) where T : Differentiable, T.TangentVector == T)
   public func adding(imaginary: T) -> Complex {
     var c = self
     c.imaginary += imaginary
     return c
   }
-
-  @differentiable(vjp: _vjpSubtracting(imaginary:))
+  
+  @inlinable
+  @differentiable(vjp: _vjpSubtracting(imaginary:) where T : Differentiable, T.TangentVector == T)
   public func subtracting(imaginary: T) -> Complex {
     var c = self
     c.imaginary -= imaginary
@@ -384,18 +353,7 @@ extension Complex {
   }
 }
 
-extension Complex {
-  @inlinable
-  static func _vjpInit(real: T, imaginary: T) -> (Complex<T>, (Complex<T>) -> (T, T)) {
-    // let orig: Complex<T> = Complex(real: real, imaginary: imaginary)
-    // let pb: (Complex) -> (T, T) = { v in
-    //   return (v.real, v.imaginary) 
-    // }
-    return (Complex(real: real, imaginary: imaginary), { v in
-      return (v.real, v.imaginary) 
-    })
-  }
-
+extension Complex where T : Differentiable, T.TangentVector == T {
   @inlinable
   static func _vjpAdd(lhs: Complex, rhs: Complex) 
   -> (Complex, (Complex) -> (Complex, Complex)) {

From de575fe15735da66b67b981e8e5867eed7bc23ef Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Wed, 22 May 2019 13:58:24 -0700
Subject: [PATCH 05/28] Move T Tangent constraint.

---
 Sources/TensorFlow/Core/Complex.swift | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/Sources/TensorFlow/Core/Complex.swift b/Sources/TensorFlow/Core/Complex.swift
index 2d9199d76..19209f8f2 100644
--- a/Sources/TensorFlow/Core/Complex.swift
+++ b/Sources/TensorFlow/Core/Complex.swift
@@ -20,7 +20,7 @@ public struct Complex<T : FloatingPoint> {
   }
 }
 
-extension Complex : Differentiable where T : Differentiable, T.TangentVector == T {
+extension Complex : Differentiable where T : Differentiable/*, T.TangentVector == T*/ {
   // ---------------------------------------------------------------------------
   // MARK: Differentiability
   // ---------------------------------------------------------------------------
@@ -135,7 +135,7 @@ extension Complex : AdditiveArithmetic {
   // ---------------------------------------------------------------------------
 
   @inlinable
-  @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable, T.TangentVector == T)
+  @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
   public static func + (lhs: Complex, rhs: Complex) -> Complex {
     var lhs = lhs
     lhs += rhs
@@ -149,7 +149,7 @@ extension Complex : AdditiveArithmetic {
   }
 
   @inlinable
-  @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable, T.TangentVector == T)
+  @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
   public static func - (lhs: Complex, rhs: Complex) -> Complex {
     var lhs = lhs
     lhs -= rhs
@@ -175,7 +175,7 @@ extension Complex : Numeric {
   }
 
   @inlinable
-  @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable, T.TangentVector == T)
+  @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable)
   public static func * (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
     let ac = a * c, bd = b * d, ad = a * d, bc = b * c
@@ -244,7 +244,7 @@ extension Complex : SignedNumeric {
   // ---------------------------------------------------------------------------
 
   @inlinable
-  @differentiable(vjp: _vjpNegate where T : Differentiable, T.TangentVector == T)
+  @differentiable(vjp: _vjpNegate where T : Differentiable)
   public static prefix func - (operand: Complex) -> Complex {
     return Complex(real: -operand.real, imaginary: -operand.imaginary)
   }
@@ -262,7 +262,7 @@ extension Complex {
   // ---------------------------------------------------------------------------
 
   @inlinable
-  @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable, T.TangentVector == T)
+  @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
   public static func / (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
     var x: T
@@ -353,7 +353,7 @@ extension Complex {
   }
 }
 
-extension Complex where T : Differentiable, T.TangentVector == T {
+extension Complex where T : Differentiable/*, T.TangentVector == T*/ {
   @inlinable
   static func _vjpAdd(lhs: Complex, rhs: Complex) 
   -> (Complex, (Complex) -> (Complex, Complex)) {
@@ -383,7 +383,9 @@ extension Complex where T : Differentiable, T.TangentVector == T {
   -> (Complex, (Complex) -> Complex) {
     return (-operand, { v in -v})
   }
+}
 
+extension Complex where T : Differentiable, T.TangentVector == T {
   @inlinable
   func _vjpAdding(real: T) -> (Complex, (Complex) -> (Complex, T)) {
     return (self.adding(real: real), { ($0, $0.real) })
@@ -403,4 +405,4 @@ extension Complex where T : Differentiable, T.TangentVector == T {
   func _vjpSubtracting(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
     return (self.subtracting(real: real), { ($0, -$0.imaginary) })
   }
-}
\ No newline at end of file
+}

From 683366de4ea946402a470c76358ae17c0ed58883 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Wed, 22 May 2019 14:05:39 -0700
Subject: [PATCH 06/28] Cleanup comments and commented out code.

---
 Sources/TensorFlow/Core/Complex.swift | 92 ++-------------------------
 1 file changed, 4 insertions(+), 88 deletions(-)

diff --git a/Sources/TensorFlow/Core/Complex.swift b/Sources/TensorFlow/Core/Complex.swift
index 19209f8f2..ccac1873d 100644
--- a/Sources/TensorFlow/Core/Complex.swift
+++ b/Sources/TensorFlow/Core/Complex.swift
@@ -1,85 +1,40 @@
-import TensorFlow
-// T : FloatingPoint & Differentiable
 public struct Complex<T : FloatingPoint> {
-  // ---------------------------------------------------------------------------
-  // MARK: Stored Properties
-  // ---------------------------------------------------------------------------
-
-  /// The real component of the complex value.
   public var real: T
-
-  /// The imaginary component of the complex value.
   public var imaginary: T
 
-  // ---------------------------------------------------------------------------
-  // MARK: Initializers
-  // ---------------------------------------------------------------------------
   public init(real: T = 0, imaginary: T = 0) {
 	self.real = real
 	self.imaginary = imaginary
   }
 }
 
-extension Complex : Differentiable where T : Differentiable/*, T.TangentVector == T*/ {
-  // ---------------------------------------------------------------------------
-  // MARK: Differentiability
-  // ---------------------------------------------------------------------------
+extension Complex : Differentiable where T : Differentiable {
   public typealias TangentVector = Complex
   public typealias AllDifferentiableVariables = Complex
 }
 
 extension Complex {
-  // ---------------------------------------------------------------------------
-  // MARK: Static Properties
-  // ---------------------------------------------------------------------------
-
-  /// The imaginary unit _i_.
   @inlinable
   public static var i: Complex {
     return Complex(real: 0, imaginary: 1)
   }
 
-  /// A Boolean value indicating whether the instance is finite.
-  ///
-  /// A complex value is finite if its real and imaginary components are both
-  /// finite. A component is finite if it is not infinity or NaN.
   @inlinable
   public var isFinite: Bool {
     return real.isFinite && imaginary.isFinite
   }
 
-  /// A Boolean value indicating whether the instance is infinite.
-  ///
-  /// A complex value is infinite if at least one of its components (real or
-  /// imaginary) is infinite, even if the other component is NaN.
-  ///
-  /// Note that `isFinite` and `isInfinite` do not form a dichotomy because NaN
-  /// is neither finite nor infinite.
   @inlinable
   public var isInfinite: Bool {
     return real.isInfinite || imaginary.isInfinite
   }
 
-  /// A Boolean value indicating whether the instance is NaN ("not a number").
-  ///
-  /// A complex value is NaN if at least one of its components (real or
-  /// imaginary) is NaN and the other component is not infinite.
-  ///
-  /// Because NaN is not equal to any value, including NaN, use this property
-  /// instead of the equal-to operator (`==`) or not-equal-to operator (`!=`) to
-  /// test whether a value is or is not NaN.
-  ///
-  /// This property is `true` for both quiet and signaling NaNs.
   @inlinable
   public var isNaN: Bool {
     return (real.isNaN && !imaginary.isInfinite) ||
       (imaginary.isNaN && !real.isInfinite)
   }
 
-  /// A Boolean value indicating whether the instance is equal to zero.
-  ///
-  /// A complex value is equal to zero if its real and imaginary components both
-  /// represent either `-0.0` or `+0.0`.
   @inlinable
   public var isZero: Bool {
     return real.isZero && imaginary.isZero
@@ -87,10 +42,6 @@ extension Complex {
 }
 
 extension Complex : ExpressibleByIntegerLiteral {
-  // ---------------------------------------------------------------------------
-  // MARK: ExpressibleByIntegerLiteral
-  // ---------------------------------------------------------------------------
-
   @inlinable
   public init(integerLiteral value: Int) {
     self.real = T(value)
@@ -99,16 +50,9 @@ extension Complex : ExpressibleByIntegerLiteral {
 }
 
 extension Complex : CustomStringConvertible {
-  // ---------------------------------------------------------------------------
-  // MARK: CustomStringConvertible
-  // ---------------------------------------------------------------------------
-
   @inlinable
   public var description: String {
     return real.isNaN && real.sign == .minus
-      // At present, -NaN is described as "nan", which is acceptable for real
-      // values. However, it is arguably misleading to describe -NaN - NaNi as
-      // "nan + nani" or "nan - nani". Therefore, handle this case separately.
       ? imaginary.sign == .minus
         ? "-\(-real) - \(-imaginary)i"
         : "-\(-real) + \(imaginary)i"
@@ -119,10 +63,6 @@ extension Complex : CustomStringConvertible {
 }
 
 extension Complex : Equatable {
-  // ---------------------------------------------------------------------------
-  // MARK: Equatable
-  // ---------------------------------------------------------------------------
-
   @inlinable
   public static func == (lhs: Complex, rhs: Complex) -> Bool {
     return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
@@ -130,10 +70,6 @@ extension Complex : Equatable {
 }
 
 extension Complex : AdditiveArithmetic {
-  // ---------------------------------------------------------------------------
-  // MARK: AdditiveArithmetic
-  // ---------------------------------------------------------------------------
-
   @inlinable
   @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
   public static func + (lhs: Complex, rhs: Complex) -> Complex {
@@ -164,10 +100,6 @@ extension Complex : AdditiveArithmetic {
 }
 
 extension Complex : Numeric {
-  // ---------------------------------------------------------------------------
-  // MARK: Numeric
-  // ---------------------------------------------------------------------------
-
   public init?<U>(exactly source: U) where U : BinaryInteger {
     guard let t = T(exactly: source) else { return nil }
     self.real = t
@@ -181,12 +113,10 @@ extension Complex : Numeric {
     let ac = a * c, bd = b * d, ad = a * d, bc = b * c
     let x = ac - bd
     let y = ad + bc
-    // Recover infinities that computed as NaN + iNaN.
-    // See C11 Annex G.
+
     if x.isNaN && y.isNaN {
       var recalculate = false
       if a.isInfinite || b.isInfinite {
-        // "Box" the infinity and change NaNs in the other operand to 0.
         a = T(signOf: a, magnitudeOf: a.isInfinite ? 1 : 0)
         b = T(signOf: b, magnitudeOf: b.isInfinite ? 1 : 0)
         if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
@@ -194,7 +124,6 @@ extension Complex : Numeric {
         recalculate = true
       }
       if c.isInfinite || d.isInfinite {
-        // "Box" the infinity and change NaNs in the other operand to 0.
         if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
         if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
         c = T(signOf: c, magnitudeOf: c.isInfinite ? 1 : 0)
@@ -203,7 +132,6 @@ extension Complex : Numeric {
       }
       if !recalculate &&
         (ac.isInfinite || bd.isInfinite || ad.isInfinite || bc.isInfinite) {
-        // Recover infinities from overflow by changing NaNs to 0.
         if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
         if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
         if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
@@ -239,10 +167,6 @@ extension Complex : Numeric {
 }
 
 extension Complex : SignedNumeric {
-  // ---------------------------------------------------------------------------
-  // MARK: SignedNumeric
-  // ---------------------------------------------------------------------------
-
   @inlinable
   @differentiable(vjp: _vjpNegate where T : Differentiable)
   public static prefix func - (operand: Complex) -> Complex {
@@ -257,17 +181,12 @@ extension Complex : SignedNumeric {
 }
 
 extension Complex {
-  // ---------------------------------------------------------------------------
-  // MARK: Division
-  // ---------------------------------------------------------------------------
-
   @inlinable
   @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
   public static func / (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
     var x: T
     var y: T
-    // Prevent avoidable overflow; see Numerical Recipes.
     if c.magnitude >= d.magnitude {
       let ratio = d / c
       let denominator = c + d * ratio
@@ -279,12 +198,10 @@ extension Complex {
       x = (a * ratio + b) / denominator
       y = (b * ratio - a) / denominator
     }
-    // Recover infinities and zeros that computed as NaN + iNaN.
-    // See C11 Annex G.
     if x.isNaN && y.isNaN {
       if c == 0 && d == 0 && (!a.isNaN || !b.isNaN) {
         x = T(signOf: c, magnitudeOf: .infinity) * a
-        y = T(signOf: c /* sic */, magnitudeOf: .infinity) * b
+        y = T(signOf: c, magnitudeOf: .infinity) * b
       } else if (a.isInfinite || b.isInfinite) && c.isFinite && d.isFinite {
         a = T(signOf: a, magnitudeOf: a.isInfinite ? 1 : 0)
         b = T(signOf: b, magnitudeOf: b.isInfinite ? 1 : 0)
@@ -313,7 +230,6 @@ extension Complex {
   }
 }
 
-/// Returns the absolute value (magnitude, modulus) of `z`.
 @inlinable
 public func abs<T>(_ z: Complex<T>) -> Complex<T> {
   return Complex(real: z.magnitude)
@@ -353,7 +269,7 @@ extension Complex {
   }
 }
 
-extension Complex where T : Differentiable/*, T.TangentVector == T*/ {
+extension Complex where T : Differentiable {
   @inlinable
   static func _vjpAdd(lhs: Complex, rhs: Complex) 
   -> (Complex, (Complex) -> (Complex, Complex)) {

From 01efbd9b7641caf362d47521e3f51e800d78dbbc Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Wed, 22 May 2019 17:59:42 -0700
Subject: [PATCH 07/28] Flip from inlinable to usableFromInline.

---
 Sources/TensorFlow/Core/Complex.swift | 87 +++++++++++++++------------
 1 file changed, 47 insertions(+), 40 deletions(-)

diff --git a/Sources/TensorFlow/Core/Complex.swift b/Sources/TensorFlow/Core/Complex.swift
index ccac1873d..19adb62d1 100644
--- a/Sources/TensorFlow/Core/Complex.swift
+++ b/Sources/TensorFlow/Core/Complex.swift
@@ -2,9 +2,10 @@ public struct Complex<T : FloatingPoint> {
   public var real: T
   public var imaginary: T
 
+  @differentiable(where T : Differentiable)
   public init(real: T = 0, imaginary: T = 0) {
-	self.real = real
-	self.imaginary = imaginary
+    self.real = real
+    self.imaginary = imaginary
   }
 }
 
@@ -14,35 +15,35 @@ extension Complex : Differentiable where T : Differentiable {
 }
 
 extension Complex {
-  @inlinable
+   
   public static var i: Complex {
     return Complex(real: 0, imaginary: 1)
   }
 
-  @inlinable
+   
   public var isFinite: Bool {
     return real.isFinite && imaginary.isFinite
   }
 
-  @inlinable
+   
   public var isInfinite: Bool {
     return real.isInfinite || imaginary.isInfinite
   }
 
-  @inlinable
+   
   public var isNaN: Bool {
     return (real.isNaN && !imaginary.isInfinite) ||
       (imaginary.isNaN && !real.isInfinite)
   }
 
-  @inlinable
+   
   public var isZero: Bool {
     return real.isZero && imaginary.isZero
   }
 }
 
 extension Complex : ExpressibleByIntegerLiteral {
-  @inlinable
+   
   public init(integerLiteral value: Int) {
     self.real = T(value)
     self.imaginary = 0
@@ -50,7 +51,7 @@ extension Complex : ExpressibleByIntegerLiteral {
 }
 
 extension Complex : CustomStringConvertible {
-  @inlinable
+   
   public var description: String {
     return real.isNaN && real.sign == .minus
       ? imaginary.sign == .minus
@@ -63,14 +64,14 @@ extension Complex : CustomStringConvertible {
 }
 
 extension Complex : Equatable {
-  @inlinable
+   
   public static func == (lhs: Complex, rhs: Complex) -> Bool {
     return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
   }
 }
 
 extension Complex : AdditiveArithmetic {
-  @inlinable
+   
   @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
   public static func + (lhs: Complex, rhs: Complex) -> Complex {
     var lhs = lhs
@@ -78,13 +79,13 @@ extension Complex : AdditiveArithmetic {
     return lhs
   }
 
-  @inlinable
+   
   public static func += (lhs: inout Complex, rhs: Complex) {
     lhs.real += rhs.real
     lhs.imaginary += rhs.imaginary
   }
 
-  @inlinable
+   
   @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
   public static func - (lhs: Complex, rhs: Complex) -> Complex {
     var lhs = lhs
@@ -92,7 +93,7 @@ extension Complex : AdditiveArithmetic {
     return lhs
   }
 
-  @inlinable
+   
   public static func -= (lhs: inout Complex, rhs: Complex) {
     lhs.real -= rhs.real
     lhs.imaginary -= rhs.imaginary
@@ -106,7 +107,7 @@ extension Complex : Numeric {
     self.imaginary = 0
   }
 
-  @inlinable
+   
   @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable)
   public static func * (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
@@ -148,12 +149,12 @@ extension Complex : Numeric {
     return Complex(real: x, imaginary: y)
   }
 
-  @inlinable
+   
   public static func *= (lhs: inout Complex, rhs: Complex) {
     lhs = lhs * rhs
   }
 
-  @inlinable
+   
   public var magnitude: T {
     var x = abs(real)
     var y = abs(imaginary)
@@ -167,13 +168,13 @@ extension Complex : Numeric {
 }
 
 extension Complex : SignedNumeric {
-  @inlinable
+   
   @differentiable(vjp: _vjpNegate where T : Differentiable)
   public static prefix func - (operand: Complex) -> Complex {
     return Complex(real: -operand.real, imaginary: -operand.imaginary)
   }
 
-  @inlinable
+   
   public mutating func negate() {
     real.negate()
     imaginary.negate()
@@ -181,7 +182,7 @@ extension Complex : SignedNumeric {
 }
 
 extension Complex {
-  @inlinable
+   
   @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
   public static func / (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
@@ -217,26 +218,27 @@ extension Complex {
     return Complex(real: x, imaginary: y)
   }
 
-  @inlinable
+   
   public static func /= (lhs: inout Complex, rhs: Complex) {
     lhs = lhs / rhs
   }
 }
 
 extension Complex {
-  @inlinable
+   
+  @differentiable(vjp: _vjpComplexConjugate where T : Differentiable)
   public func complexConjugate() -> Complex {
     return Complex(real: real, imaginary: -imaginary)
   }
 }
 
-@inlinable
+ 
 public func abs<T>(_ z: Complex<T>) -> Complex<T> {
   return Complex(real: z.magnitude)
 }
 
 extension Complex {
-  @inlinable
+   
   @differentiable(vjp: _vjpAdding(real:) where T : Differentiable, T.TangentVector == T)
   public func adding(real: T) -> Complex {
     var c = self
@@ -244,7 +246,7 @@ extension Complex {
     return c
   }
 
-  @inlinable
+   
   @differentiable(vjp: _vjpSubtracting(real:) where T : Differentiable, T.TangentVector == T)
   public func subtracting(real: T) -> Complex {
     var c = self
@@ -252,7 +254,7 @@ extension Complex {
     return c
   }
 
-  @inlinable
+   
   @differentiable(vjp: _vjpAdding(imaginary:) where T : Differentiable, T.TangentVector == T)
   public func adding(imaginary: T) -> Complex {
     var c = self
@@ -260,7 +262,7 @@ extension Complex {
     return c
   }
   
-  @inlinable
+   
   @differentiable(vjp: _vjpSubtracting(imaginary:) where T : Differentiable, T.TangentVector == T)
   public func subtracting(imaginary: T) -> Complex {
     var c = self
@@ -270,54 +272,59 @@ extension Complex {
 }
 
 extension Complex where T : Differentiable {
-  @inlinable
+  @usableFromInline
   static func _vjpAdd(lhs: Complex, rhs: Complex) 
   -> (Complex, (Complex) -> (Complex, Complex)) {
-    return (lhs * rhs, { v in (v, v) })
+    return (lhs + rhs, { v in (v, v) })
   }
 
-  @inlinable
+  @usableFromInline
   static func _vjpSubtract(lhs: Complex, rhs: Complex) 
   -> (Complex, (Complex) -> (Complex, Complex)) {
-    return (lhs * rhs, { v in (v, -v) })
+    return (lhs - rhs, { v in (v, -v) })
   }
 
-  @inlinable
+  @usableFromInline
   static func _vjpMultiply(lhs: Complex, rhs: Complex) 
   -> (Complex, (Complex) -> (Complex, Complex)) {
     return (lhs * rhs, { v in (rhs * v, lhs * v) })
   }
 
-  @inlinable
+  @usableFromInline
   static func _vjpDivide(lhs: Complex, rhs: Complex) 
   -> (Complex, (Complex) -> (Complex, Complex)) {
-    return (lhs * rhs, { v in (v / rhs, -lhs / (rhs * rhs) * v) })
+    return (lhs / rhs, { v in (v / rhs, -lhs / (rhs * rhs) * v) })
   }
 
-  @inlinable
+  @usableFromInline
   static func _vjpNegate(operand: Complex)
   -> (Complex, (Complex) -> Complex) {
-    return (-operand, { v in -v})
+    return (-operand, { -$0 })
+  }
+
+  @usableFromInline
+  func _vjpComplexConjugate() -> (Complex, (Complex) -> Complex) {
+    return (complexConjugate(), { v in v.complexConjugate() })
   }
 }
 
 extension Complex where T : Differentiable, T.TangentVector == T {
-  @inlinable
+  @usableFromInline
   func _vjpAdding(real: T) -> (Complex, (Complex) -> (Complex, T)) {
     return (self.adding(real: real), { ($0, $0.real) })
   }
 
-  @inlinable
+  @usableFromInline
   func _vjpSubtracting(real: T) -> (Complex, (Complex) -> (Complex, T)) {
     return (self.subtracting(real: real), { ($0, -$0.real) })
   }
 
-  @inlinable
+  @usableFromInline
   func _vjpAdding(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
     return (self.adding(real: real), { ($0, $0.imaginary) })
   }
 
-  @inlinable
+  @usableFromInline
   func _vjpSubtracting(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
     return (self.subtracting(real: real), { ($0, -$0.imaginary) })
   }

From 23870cbcfb161b4a165941f66ef976398259fb09 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Wed, 22 May 2019 18:07:33 -0700
Subject: [PATCH 08/28] Remove trailing whitespace, add comment to make init
 differentiable.

---
 Sources/TensorFlow/Core/Complex.swift | 74 +++++++++++++--------------
 1 file changed, 37 insertions(+), 37 deletions(-)

diff --git a/Sources/TensorFlow/Core/Complex.swift b/Sources/TensorFlow/Core/Complex.swift
index 19adb62d1..a1ea4d82e 100644
--- a/Sources/TensorFlow/Core/Complex.swift
+++ b/Sources/TensorFlow/Core/Complex.swift
@@ -2,7 +2,7 @@ public struct Complex<T : FloatingPoint> {
   public var real: T
   public var imaginary: T
 
-  @differentiable(where T : Differentiable)
+  // TODO: Make differentiable, crashing right now
   public init(real: T = 0, imaginary: T = 0) {
     self.real = real
     self.imaginary = imaginary
@@ -15,35 +15,35 @@ extension Complex : Differentiable where T : Differentiable {
 }
 
 extension Complex {
-   
+
   public static var i: Complex {
     return Complex(real: 0, imaginary: 1)
   }
 
-   
+
   public var isFinite: Bool {
     return real.isFinite && imaginary.isFinite
   }
 
-   
+
   public var isInfinite: Bool {
     return real.isInfinite || imaginary.isInfinite
   }
 
-   
+
   public var isNaN: Bool {
     return (real.isNaN && !imaginary.isInfinite) ||
       (imaginary.isNaN && !real.isInfinite)
   }
 
-   
+
   public var isZero: Bool {
     return real.isZero && imaginary.isZero
   }
 }
 
 extension Complex : ExpressibleByIntegerLiteral {
-   
+
   public init(integerLiteral value: Int) {
     self.real = T(value)
     self.imaginary = 0
@@ -51,7 +51,7 @@ extension Complex : ExpressibleByIntegerLiteral {
 }
 
 extension Complex : CustomStringConvertible {
-   
+
   public var description: String {
     return real.isNaN && real.sign == .minus
       ? imaginary.sign == .minus
@@ -64,36 +64,36 @@ extension Complex : CustomStringConvertible {
 }
 
 extension Complex : Equatable {
-   
+
   public static func == (lhs: Complex, rhs: Complex) -> Bool {
     return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
   }
 }
 
 extension Complex : AdditiveArithmetic {
-   
+
   @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
   public static func + (lhs: Complex, rhs: Complex) -> Complex {
-    var lhs = lhs
-    lhs += rhs
-    return lhs
+    var temp = lhs
+    temp += rhs
+    return temp
   }
 
-   
+
   public static func += (lhs: inout Complex, rhs: Complex) {
     lhs.real += rhs.real
     lhs.imaginary += rhs.imaginary
   }
 
-   
+
   @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
   public static func - (lhs: Complex, rhs: Complex) -> Complex {
-    var lhs = lhs
-    lhs -= rhs
-    return lhs
+    var temp = lhs
+    temp -= rhs
+    return temp
   }
 
-   
+
   public static func -= (lhs: inout Complex, rhs: Complex) {
     lhs.real -= rhs.real
     lhs.imaginary -= rhs.imaginary
@@ -107,7 +107,7 @@ extension Complex : Numeric {
     self.imaginary = 0
   }
 
-   
+
   @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable)
   public static func * (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
@@ -149,12 +149,12 @@ extension Complex : Numeric {
     return Complex(real: x, imaginary: y)
   }
 
-   
+
   public static func *= (lhs: inout Complex, rhs: Complex) {
     lhs = lhs * rhs
   }
 
-   
+
   public var magnitude: T {
     var x = abs(real)
     var y = abs(imaginary)
@@ -168,13 +168,13 @@ extension Complex : Numeric {
 }
 
 extension Complex : SignedNumeric {
-   
+
   @differentiable(vjp: _vjpNegate where T : Differentiable)
   public static prefix func - (operand: Complex) -> Complex {
     return Complex(real: -operand.real, imaginary: -operand.imaginary)
   }
 
-   
+
   public mutating func negate() {
     real.negate()
     imaginary.negate()
@@ -182,7 +182,7 @@ extension Complex : SignedNumeric {
 }
 
 extension Complex {
-   
+
   @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
   public static func / (lhs: Complex, rhs: Complex) -> Complex {
     var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
@@ -218,27 +218,27 @@ extension Complex {
     return Complex(real: x, imaginary: y)
   }
 
-   
+
   public static func /= (lhs: inout Complex, rhs: Complex) {
     lhs = lhs / rhs
   }
 }
 
 extension Complex {
-   
+
   @differentiable(vjp: _vjpComplexConjugate where T : Differentiable)
   public func complexConjugate() -> Complex {
     return Complex(real: real, imaginary: -imaginary)
   }
 }
 
- 
+
 public func abs<T>(_ z: Complex<T>) -> Complex<T> {
   return Complex(real: z.magnitude)
 }
 
 extension Complex {
-   
+
   @differentiable(vjp: _vjpAdding(real:) where T : Differentiable, T.TangentVector == T)
   public func adding(real: T) -> Complex {
     var c = self
@@ -246,7 +246,7 @@ extension Complex {
     return c
   }
 
-   
+
   @differentiable(vjp: _vjpSubtracting(real:) where T : Differentiable, T.TangentVector == T)
   public func subtracting(real: T) -> Complex {
     var c = self
@@ -254,15 +254,15 @@ extension Complex {
     return c
   }
 
-   
+
   @differentiable(vjp: _vjpAdding(imaginary:) where T : Differentiable, T.TangentVector == T)
   public func adding(imaginary: T) -> Complex {
     var c = self
     c.imaginary += imaginary
     return c
   }
-  
-   
+
+
   @differentiable(vjp: _vjpSubtracting(imaginary:) where T : Differentiable, T.TangentVector == T)
   public func subtracting(imaginary: T) -> Complex {
     var c = self
@@ -273,25 +273,25 @@ extension Complex {
 
 extension Complex where T : Differentiable {
   @usableFromInline
-  static func _vjpAdd(lhs: Complex, rhs: Complex) 
+  static func _vjpAdd(lhs: Complex, rhs: Complex)
   -> (Complex, (Complex) -> (Complex, Complex)) {
     return (lhs + rhs, { v in (v, v) })
   }
 
   @usableFromInline
-  static func _vjpSubtract(lhs: Complex, rhs: Complex) 
+  static func _vjpSubtract(lhs: Complex, rhs: Complex)
   -> (Complex, (Complex) -> (Complex, Complex)) {
     return (lhs - rhs, { v in (v, -v) })
   }
 
   @usableFromInline
-  static func _vjpMultiply(lhs: Complex, rhs: Complex) 
+  static func _vjpMultiply(lhs: Complex, rhs: Complex)
   -> (Complex, (Complex) -> (Complex, Complex)) {
     return (lhs * rhs, { v in (rhs * v, lhs * v) })
   }
 
   @usableFromInline
-  static func _vjpDivide(lhs: Complex, rhs: Complex) 
+  static func _vjpDivide(lhs: Complex, rhs: Complex)
   -> (Complex, (Complex) -> (Complex, Complex)) {
     return (lhs / rhs, { v in (v / rhs, -lhs / (rhs * rhs) * v) })
   }

From fdd1c5055ae974eb6fa56d63045d735103c34f5d Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Thu, 23 May 2019 10:44:05 -0700
Subject: [PATCH 09/28] Move around tests and file for complex.

---
 .../Core => DeepLearning}/Complex.swift       |  23 +--
 .../ComplexAutoDiffTests.swift                |  98 ++++++++++++
 Tests/DeepLearningTests/ComplexTests.swift    | 149 ++++++++++++++++++
 3 files changed, 255 insertions(+), 15 deletions(-)
 rename Sources/{TensorFlow/Core => DeepLearning}/Complex.swift (96%)
 create mode 100644 Tests/DeepLearningTests/ComplexAutoDiffTests.swift
 create mode 100644 Tests/DeepLearningTests/ComplexTests.swift

diff --git a/Sources/TensorFlow/Core/Complex.swift b/Sources/DeepLearning/Complex.swift
similarity index 96%
rename from Sources/TensorFlow/Core/Complex.swift
rename to Sources/DeepLearning/Complex.swift
index a1ea4d82e..a62cb1685 100644
--- a/Sources/TensorFlow/Core/Complex.swift
+++ b/Sources/DeepLearning/Complex.swift
@@ -2,7 +2,7 @@ public struct Complex<T : FloatingPoint> {
   public var real: T
   public var imaginary: T
 
-  // TODO: Make differentiable, crashing right now
+  @differentiable(vjp: _vjpInit where T : Differentiable, T.TangentVector == T)
   public init(real: T = 0, imaginary: T = 0) {
     self.real = real
     self.imaginary = imaginary
@@ -15,28 +15,23 @@ extension Complex : Differentiable where T : Differentiable {
 }
 
 extension Complex {
-
   public static var i: Complex {
     return Complex(real: 0, imaginary: 1)
   }
 
-
   public var isFinite: Bool {
     return real.isFinite && imaginary.isFinite
   }
 
-
   public var isInfinite: Bool {
     return real.isInfinite || imaginary.isInfinite
   }
 
-
   public var isNaN: Bool {
     return (real.isNaN && !imaginary.isInfinite) ||
       (imaginary.isNaN && !real.isInfinite)
   }
 
-
   public var isZero: Bool {
     return real.isZero && imaginary.isZero
   }
@@ -79,13 +74,11 @@ extension Complex : AdditiveArithmetic {
     return temp
   }
 
-
   public static func += (lhs: inout Complex, rhs: Complex) {
     lhs.real += rhs.real
     lhs.imaginary += rhs.imaginary
   }
 
-
   @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
   public static func - (lhs: Complex, rhs: Complex) -> Complex {
     var temp = lhs
@@ -93,7 +86,6 @@ extension Complex : AdditiveArithmetic {
     return temp
   }
 
-
   public static func -= (lhs: inout Complex, rhs: Complex) {
     lhs.real -= rhs.real
     lhs.imaginary -= rhs.imaginary
@@ -149,12 +141,10 @@ extension Complex : Numeric {
     return Complex(real: x, imaginary: y)
   }
 
-
   public static func *= (lhs: inout Complex, rhs: Complex) {
     lhs = lhs * rhs
   }
 
-
   public var magnitude: T {
     var x = abs(real)
     var y = abs(imaginary)
@@ -174,7 +164,6 @@ extension Complex : SignedNumeric {
     return Complex(real: -operand.real, imaginary: -operand.imaginary)
   }
 
-
   public mutating func negate() {
     real.negate()
     imaginary.negate()
@@ -246,7 +235,6 @@ extension Complex {
     return c
   }
 
-
   @differentiable(vjp: _vjpSubtracting(real:) where T : Differentiable, T.TangentVector == T)
   public func subtracting(real: T) -> Complex {
     var c = self
@@ -254,7 +242,6 @@ extension Complex {
     return c
   }
 
-
   @differentiable(vjp: _vjpAdding(imaginary:) where T : Differentiable, T.TangentVector == T)
   public func adding(imaginary: T) -> Complex {
     var c = self
@@ -262,7 +249,6 @@ extension Complex {
     return c
   }
 
-
   @differentiable(vjp: _vjpSubtracting(imaginary:) where T : Differentiable, T.TangentVector == T)
   public func subtracting(imaginary: T) -> Complex {
     var c = self
@@ -271,6 +257,13 @@ extension Complex {
   }
 }
 
+extension Complex where T : Differentiable, T.TangentVector == T {
+  @usableFromInline
+  static func _vjpInit(real: T, imaginary: T) -> (Complex, (Complex) -> (T, T)) {
+    return (Complex(real: real, imaginary: imaginary), { ($0.real, $0.imaginary) })
+  }
+}
+
 extension Complex where T : Differentiable {
   @usableFromInline
   static func _vjpAdd(lhs: Complex, rhs: Complex)
diff --git a/Tests/DeepLearningTests/ComplexAutoDiffTests.swift b/Tests/DeepLearningTests/ComplexAutoDiffTests.swift
new file mode 100644
index 000000000..b49e85e34
--- /dev/null
+++ b/Tests/DeepLearningTests/ComplexAutoDiffTests.swift
@@ -0,0 +1,98 @@
+// RUN: %target-run-simple-swift
+// REQUIRES: executable_test
+//
+// Complex API tests.
+
+// TODO: remove import
+import TensorFlow
+
+import StdlibUnittest
+
+var AutoDiffComplexTests = TestSuite("AutoDiffComplex")
+
+AutoDiffComplexTests.test("_vjpAdd") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+    return x + Complex<Float>(real: 5, imaginary: 6)
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+}
+
+AutoDiffComplexTests.test("_vjpSubtract") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+    return Complex<Float>(real: 5, imaginary: 6) - x
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+}
+
+AutoDiffComplexTests.test("_vjpMultiply") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+    return x * x
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 4, imaginary: 6))
+  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: -6, imaginary: 4))
+  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -2, imaginary: 10))
+}
+
+AutoDiffComplexTests.test("_vjpDivide") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+    return x / Complex<Float>(real: 2, imaginary: 2)
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 0.25, imaginary: -0.25))
+  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0.25, imaginary: 0.25))
+}
+
+AutoDiffComplexTests.test("_vjpNegate") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+    return -x
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
+  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
+  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+}
+
+AutoDiffComplexTests.test("_vjpComplexConjugate") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+    return x.complexConjugate()
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
+  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
+  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+}
+
+AutoDiffComplexTests.test("_vjpAdding(real:)") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+    return x.adding(real: 5)
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+}
+
+AutoDiffComplexTests.test("_vjpAdding(imaginary:)") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+    return x.adding(imaginary: 5)
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+}
+
+AutoDiffComplexTests.test("_vjpSubtracting(real:)") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+    return x.subtracting(real: 5)
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+}
+
+AutoDiffComplexTests.test("_vjpSubtracting(imaginary:)") {
+  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+    return x.subtracting(imaginary: 5)
+  }
+  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+}
+
+runAllTests()
\ No newline at end of file
diff --git a/Tests/DeepLearningTests/ComplexTests.swift b/Tests/DeepLearningTests/ComplexTests.swift
new file mode 100644
index 000000000..5d9b9e00f
--- /dev/null
+++ b/Tests/DeepLearningTests/ComplexTests.swift
@@ -0,0 +1,149 @@
+// RUN: %target-run-simple-swift
+// REQUIRES: executable_test
+//
+// Complex API tests.
+
+// TODO: remove import
+import TensorFlow
+
+import StdlibUnittest
+
+var ComplexTests = TestSuite("Complex")
+
+ComplexTests.test("Initializer") {
+  let complex = Complex<Float>(real: 2, imaginary: 3)
+  expectEqual(complex.real, 2)
+  expectEqual(complex.imaginary, 3)
+}
+
+ComplexTests.test("Static Imaginary") {
+  let imaginary = Complex<Float>(real: 0, imaginary: 1)
+  expectEqual(imaginary, Complex.i)
+}
+
+ComplexTests.test("isFinite") {
+  var complex = Complex<Float>(real: 999, imaginary: 0)
+  expectTrue(complex.isFinite)
+
+  complex = Complex(real: 1.0 / 0.0, imaginary: 1)
+  expectFalse(complex.isFinite)
+
+  complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
+  expectFalse(complex.isFinite)
+}
+
+ComplexTests.test("isInfinite") {
+  var complex = Complex<Float>(real: 999, imaginary: 0)
+  expectFalse(complex.isInfinite)
+
+  complex = Complex(real: 1.0 / 0.0, imaginary: 1)
+  expectTrue(complex.isInfinite)
+
+  complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
+  expectTrue(complex.isInfinite)
+}
+
+ComplexTests.test("isNaN") {
+  var complex = Complex<Float>(real: 999, imaginary: 0)
+  expectFalse(complex.isNaN)
+
+  complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 1)
+  expectTrue(complex.isNaN)
+
+  complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
+  expectTrue(complex.isNaN)
+}
+
+ComplexTests.test("isZero") {
+  var complex = Complex<Float>(real: 999, imaginary: 0)
+  expectFalse(complex.isZero)
+
+  complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0)
+  expectFalse(complex.isZero)
+
+  complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
+  expectFalse(complex.isZero)
+
+  complex = Complex(real: 0, imaginary: 0)
+  expectTrue(complex.isZero)
+}
+
+ComplexTests.test("==") {
+  var complexA = Complex<Float>(real: 999, imaginary: 0)
+  let complexB = Complex<Float>(real: 999, imaginary: 0)
+  expectEqual(complexA, complexB)
+
+  complexA = Complex(real: 5, imaginary: 0)
+  expectNotEqual(complexA, complexB)
+}
+
+ComplexTests.test("+") {
+  let input = Complex<Float>(real: 5, imaginary: 1)
+  let expected = Complex<Float>(real: 10, imaginary: 2)
+  expectEqual(expected, input + input)
+}
+
+ComplexTests.test("-") {
+  let inputA = Complex<Float>(real: 6, imaginary: 2)
+  let inputB = Complex<Float>(real: 5, imaginary: 1)
+  let expected = Complex<Float>(real: 1, imaginary: 1)
+  expectEqual(expected, inputA - inputB)
+}
+
+ComplexTests.test("*") {
+  let inputA = Complex<Float>(real: 6, imaginary: 2)
+  let inputB = Complex<Float>(real: 5, imaginary: 1)
+  let expected = Complex<Float>(real: 28, imaginary: 16)
+  expectEqual(expected, inputA * inputB)
+}
+
+ComplexTests.test("negate") {
+  var input = Complex<Float>(real: 6, imaginary: 2)
+  let negated = Complex<Float>(real: -6, imaginary: -2)
+  expectEqual(-input, negated)
+  input.negate()
+  expectEqual(input, negated)
+}
+
+ComplexTests.test("/") {
+  let inputA = Complex<Float>(real: 20, imaginary: -4)
+  let inputB = Complex<Float>(real: 3, imaginary: 2)
+  let expected = Complex<Float>(real: 4, imaginary: -4)
+  expectEqual(expected, inputA / inputB)
+}
+
+ComplexTests.test("complexConjugate") {
+  var input = Complex<Float>(real: 2, imaginary: -4)
+  var expected = Complex<Float>(real: 2, imaginary: 4)
+  expectEqual(expected, input.complexConjugate())
+
+  input = Complex<Float>(real: -2, imaginary: -4)
+  expected = Complex<Float>(real: -2, imaginary: 4)
+  expectEqual(expected, input.complexConjugate())
+
+  input = Complex<Float>(real: 2, imaginary: 4)
+  expected = Complex<Float>(real: 2, imaginary: -4)
+  expectEqual(expected, input.complexConjugate())
+}
+
+ComplexTests.test("adding") {
+  var input = Complex<Float>(real: 2, imaginary: -4)
+  var expected = Complex<Float>(real: 3, imaginary: -4)
+  expectEqual(expected, input.adding(real: 1))
+
+  input = Complex<Float>(real: 2, imaginary: -4)
+  expected = Complex<Float>(real: 2, imaginary: -3)
+  expectEqual(expected, input.adding(imaginary: 1))
+}
+
+ComplexTests.test("subtracting") {
+  var input = Complex<Float>(real: 2, imaginary: -4)
+  var expected = Complex<Float>(real: 1, imaginary: -4)
+  expectEqual(expected, input.subtracting(real: 1))
+
+  input = Complex<Float>(real: 2, imaginary: -4)
+  expected = Complex<Float>(real: 2, imaginary: -5)
+  expectEqual(expected, input.subtracting(imaginary: 1))
+}
+
+runAllTests()
\ No newline at end of file

From 3e1d56ec4f4db91ceb4f1f44390cdb9baa092568 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Thu, 23 May 2019 12:44:28 -0700
Subject: [PATCH 10/28] Add license and modify spacing.

---
 Sources/DeepLearning/Complex.swift        | 324 ---------------------
 Sources/third_party/Complex/Complex.swift | 325 ++++++++++++++++++++++
 Sources/third_party/Complex/LICENSE       |  19 ++
 3 files changed, 344 insertions(+), 324 deletions(-)
 delete mode 100644 Sources/DeepLearning/Complex.swift
 create mode 100644 Sources/third_party/Complex/Complex.swift
 create mode 100644 Sources/third_party/Complex/LICENSE

diff --git a/Sources/DeepLearning/Complex.swift b/Sources/DeepLearning/Complex.swift
deleted file mode 100644
index a62cb1685..000000000
--- a/Sources/DeepLearning/Complex.swift
+++ /dev/null
@@ -1,324 +0,0 @@
-public struct Complex<T : FloatingPoint> {
-  public var real: T
-  public var imaginary: T
-
-  @differentiable(vjp: _vjpInit where T : Differentiable, T.TangentVector == T)
-  public init(real: T = 0, imaginary: T = 0) {
-    self.real = real
-    self.imaginary = imaginary
-  }
-}
-
-extension Complex : Differentiable where T : Differentiable {
-  public typealias TangentVector = Complex
-  public typealias AllDifferentiableVariables = Complex
-}
-
-extension Complex {
-  public static var i: Complex {
-    return Complex(real: 0, imaginary: 1)
-  }
-
-  public var isFinite: Bool {
-    return real.isFinite && imaginary.isFinite
-  }
-
-  public var isInfinite: Bool {
-    return real.isInfinite || imaginary.isInfinite
-  }
-
-  public var isNaN: Bool {
-    return (real.isNaN && !imaginary.isInfinite) ||
-      (imaginary.isNaN && !real.isInfinite)
-  }
-
-  public var isZero: Bool {
-    return real.isZero && imaginary.isZero
-  }
-}
-
-extension Complex : ExpressibleByIntegerLiteral {
-
-  public init(integerLiteral value: Int) {
-    self.real = T(value)
-    self.imaginary = 0
-  }
-}
-
-extension Complex : CustomStringConvertible {
-
-  public var description: String {
-    return real.isNaN && real.sign == .minus
-      ? imaginary.sign == .minus
-        ? "-\(-real) - \(-imaginary)i"
-        : "-\(-real) + \(imaginary)i"
-      : imaginary.sign == .minus
-        ? "\(real) - \(-imaginary)i"
-        : "\(real) + \(imaginary)i"
-  }
-}
-
-extension Complex : Equatable {
-
-  public static func == (lhs: Complex, rhs: Complex) -> Bool {
-    return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
-  }
-}
-
-extension Complex : AdditiveArithmetic {
-
-  @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
-  public static func + (lhs: Complex, rhs: Complex) -> Complex {
-    var temp = lhs
-    temp += rhs
-    return temp
-  }
-
-  public static func += (lhs: inout Complex, rhs: Complex) {
-    lhs.real += rhs.real
-    lhs.imaginary += rhs.imaginary
-  }
-
-  @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
-  public static func - (lhs: Complex, rhs: Complex) -> Complex {
-    var temp = lhs
-    temp -= rhs
-    return temp
-  }
-
-  public static func -= (lhs: inout Complex, rhs: Complex) {
-    lhs.real -= rhs.real
-    lhs.imaginary -= rhs.imaginary
-  }
-}
-
-extension Complex : Numeric {
-  public init?<U>(exactly source: U) where U : BinaryInteger {
-    guard let t = T(exactly: source) else { return nil }
-    self.real = t
-    self.imaginary = 0
-  }
-
-
-  @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable)
-  public static func * (lhs: Complex, rhs: Complex) -> Complex {
-    var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
-    let ac = a * c, bd = b * d, ad = a * d, bc = b * c
-    let x = ac - bd
-    let y = ad + bc
-
-    if x.isNaN && y.isNaN {
-      var recalculate = false
-      if a.isInfinite || b.isInfinite {
-        a = T(signOf: a, magnitudeOf: a.isInfinite ? 1 : 0)
-        b = T(signOf: b, magnitudeOf: b.isInfinite ? 1 : 0)
-        if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
-        if d.isNaN { d = T(signOf: d, magnitudeOf: 0) }
-        recalculate = true
-      }
-      if c.isInfinite || d.isInfinite {
-        if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
-        if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
-        c = T(signOf: c, magnitudeOf: c.isInfinite ? 1 : 0)
-        d = T(signOf: d, magnitudeOf: d.isInfinite ? 1 : 0)
-        recalculate = true
-      }
-      if !recalculate &&
-        (ac.isInfinite || bd.isInfinite || ad.isInfinite || bc.isInfinite) {
-        if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
-        if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
-        if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
-        if d.isNaN { d = T(signOf: d, magnitudeOf: 0) }
-        recalculate = true
-      }
-      if recalculate {
-        return Complex(
-          real: .infinity * (a * c - b * d),
-          imaginary: .infinity * (a * d + b * c)
-        )
-      }
-    }
-    return Complex(real: x, imaginary: y)
-  }
-
-  public static func *= (lhs: inout Complex, rhs: Complex) {
-    lhs = lhs * rhs
-  }
-
-  public var magnitude: T {
-    var x = abs(real)
-    var y = abs(imaginary)
-    if x.isInfinite { return x }
-    if y.isInfinite { return y }
-    if x == 0 { return y }
-    if x < y { swap(&x, &y) }
-    let ratio = y / x
-    return x * (1 + ratio * ratio).squareRoot()
-  }
-}
-
-extension Complex : SignedNumeric {
-
-  @differentiable(vjp: _vjpNegate where T : Differentiable)
-  public static prefix func - (operand: Complex) -> Complex {
-    return Complex(real: -operand.real, imaginary: -operand.imaginary)
-  }
-
-  public mutating func negate() {
-    real.negate()
-    imaginary.negate()
-  }
-}
-
-extension Complex {
-
-  @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
-  public static func / (lhs: Complex, rhs: Complex) -> Complex {
-    var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
-    var x: T
-    var y: T
-    if c.magnitude >= d.magnitude {
-      let ratio = d / c
-      let denominator = c + d * ratio
-      x = (a + b * ratio) / denominator
-      y = (b - a * ratio) / denominator
-    } else {
-      let ratio = c / d
-      let denominator = c * ratio + d
-      x = (a * ratio + b) / denominator
-      y = (b * ratio - a) / denominator
-    }
-    if x.isNaN && y.isNaN {
-      if c == 0 && d == 0 && (!a.isNaN || !b.isNaN) {
-        x = T(signOf: c, magnitudeOf: .infinity) * a
-        y = T(signOf: c, magnitudeOf: .infinity) * b
-      } else if (a.isInfinite || b.isInfinite) && c.isFinite && d.isFinite {
-        a = T(signOf: a, magnitudeOf: a.isInfinite ? 1 : 0)
-        b = T(signOf: b, magnitudeOf: b.isInfinite ? 1 : 0)
-        x = .infinity * (a * c + b * d)
-        y = .infinity * (b * c - a * d)
-      } else if (c.isInfinite || d.isInfinite) && a.isFinite && b.isFinite {
-        c = T(signOf: c, magnitudeOf: c.isInfinite ? 1 : 0)
-        d = T(signOf: d, magnitudeOf: d.isInfinite ? 1 : 0)
-        x = 0 * (a * c + b * d)
-        y = 0 * (b * c - a * d)
-      }
-    }
-    return Complex(real: x, imaginary: y)
-  }
-
-
-  public static func /= (lhs: inout Complex, rhs: Complex) {
-    lhs = lhs / rhs
-  }
-}
-
-extension Complex {
-
-  @differentiable(vjp: _vjpComplexConjugate where T : Differentiable)
-  public func complexConjugate() -> Complex {
-    return Complex(real: real, imaginary: -imaginary)
-  }
-}
-
-
-public func abs<T>(_ z: Complex<T>) -> Complex<T> {
-  return Complex(real: z.magnitude)
-}
-
-extension Complex {
-
-  @differentiable(vjp: _vjpAdding(real:) where T : Differentiable, T.TangentVector == T)
-  public func adding(real: T) -> Complex {
-    var c = self
-    c.real += real
-    return c
-  }
-
-  @differentiable(vjp: _vjpSubtracting(real:) where T : Differentiable, T.TangentVector == T)
-  public func subtracting(real: T) -> Complex {
-    var c = self
-    c.real -= real
-    return c
-  }
-
-  @differentiable(vjp: _vjpAdding(imaginary:) where T : Differentiable, T.TangentVector == T)
-  public func adding(imaginary: T) -> Complex {
-    var c = self
-    c.imaginary += imaginary
-    return c
-  }
-
-  @differentiable(vjp: _vjpSubtracting(imaginary:) where T : Differentiable, T.TangentVector == T)
-  public func subtracting(imaginary: T) -> Complex {
-    var c = self
-    c.imaginary -= imaginary
-    return c
-  }
-}
-
-extension Complex where T : Differentiable, T.TangentVector == T {
-  @usableFromInline
-  static func _vjpInit(real: T, imaginary: T) -> (Complex, (Complex) -> (T, T)) {
-    return (Complex(real: real, imaginary: imaginary), { ($0.real, $0.imaginary) })
-  }
-}
-
-extension Complex where T : Differentiable {
-  @usableFromInline
-  static func _vjpAdd(lhs: Complex, rhs: Complex)
-  -> (Complex, (Complex) -> (Complex, Complex)) {
-    return (lhs + rhs, { v in (v, v) })
-  }
-
-  @usableFromInline
-  static func _vjpSubtract(lhs: Complex, rhs: Complex)
-  -> (Complex, (Complex) -> (Complex, Complex)) {
-    return (lhs - rhs, { v in (v, -v) })
-  }
-
-  @usableFromInline
-  static func _vjpMultiply(lhs: Complex, rhs: Complex)
-  -> (Complex, (Complex) -> (Complex, Complex)) {
-    return (lhs * rhs, { v in (rhs * v, lhs * v) })
-  }
-
-  @usableFromInline
-  static func _vjpDivide(lhs: Complex, rhs: Complex)
-  -> (Complex, (Complex) -> (Complex, Complex)) {
-    return (lhs / rhs, { v in (v / rhs, -lhs / (rhs * rhs) * v) })
-  }
-
-  @usableFromInline
-  static func _vjpNegate(operand: Complex)
-  -> (Complex, (Complex) -> Complex) {
-    return (-operand, { -$0 })
-  }
-
-  @usableFromInline
-  func _vjpComplexConjugate() -> (Complex, (Complex) -> Complex) {
-    return (complexConjugate(), { v in v.complexConjugate() })
-  }
-}
-
-extension Complex where T : Differentiable, T.TangentVector == T {
-  @usableFromInline
-  func _vjpAdding(real: T) -> (Complex, (Complex) -> (Complex, T)) {
-    return (self.adding(real: real), { ($0, $0.real) })
-  }
-
-  @usableFromInline
-  func _vjpSubtracting(real: T) -> (Complex, (Complex) -> (Complex, T)) {
-    return (self.subtracting(real: real), { ($0, -$0.real) })
-  }
-
-  @usableFromInline
-  func _vjpAdding(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
-    return (self.adding(real: real), { ($0, $0.imaginary) })
-  }
-
-  @usableFromInline
-  func _vjpSubtracting(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
-    return (self.subtracting(real: real), { ($0, -$0.imaginary) })
-  }
-}
diff --git a/Sources/third_party/Complex/Complex.swift b/Sources/third_party/Complex/Complex.swift
new file mode 100644
index 000000000..b79ba8afa
--- /dev/null
+++ b/Sources/third_party/Complex/Complex.swift
@@ -0,0 +1,325 @@
+// A major part of the Complex number implementation comes from
+// [@xwu](https://github.com/xwu)'s project, [NumericAnnex](https://github.com/xwu/NumericAnnex).
+
+public struct Complex<T : FloatingPoint> {
+    public var real: T
+    public var imaginary: T
+
+    @differentiable(vjp: _vjpInit where T : Differentiable, T.TangentVector == T)
+    public init(real: T = 0, imaginary: T = 0) {
+        self.real = real
+        self.imaginary = imaginary
+    }
+}
+
+extension Complex : Differentiable where T : Differentiable {
+    public typealias TangentVector = Complex
+    public typealias AllDifferentiableVariables = Complex
+}
+
+extension Complex {
+    public static var i: Complex {
+        return Complex(real: 0, imaginary: 1)
+    }
+
+    public var isFinite: Bool {
+        return real.isFinite && imaginary.isFinite
+    }
+
+    public var isInfinite: Bool {
+        return real.isInfinite || imaginary.isInfinite
+    }
+
+    public var isNaN: Bool {
+        return (real.isNaN && !imaginary.isInfinite) ||
+        (imaginary.isNaN && !real.isInfinite)
+    }
+
+    public var isZero: Bool {
+        return real.isZero && imaginary.isZero
+    }
+}
+
+extension Complex : ExpressibleByIntegerLiteral {
+    public init(integerLiteral value: Int) {
+        self.real = T(value)
+        self.imaginary = 0
+    }
+}
+
+extension Complex : CustomStringConvertible {
+    public var description: String {
+        return real.isNaN && real.sign == .minus
+            ? imaginary.sign == .minus
+                ? "-\(-real) - \(-imaginary)i"
+                : "-\(-real) + \(imaginary)i"
+            : imaginary.sign == .minus
+                ? "\(real) - \(-imaginary)i"
+                : "\(real) + \(imaginary)i"
+    }
+}
+
+extension Complex : Equatable {
+    public static func == (lhs: Complex, rhs: Complex) -> Bool {
+        return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
+    }
+}
+
+extension Complex : AdditiveArithmetic {
+    @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
+    public static func + (lhs: Complex, rhs: Complex) -> Complex {
+        var temp = lhs
+        temp += rhs
+        return temp
+    }
+
+    public static func += (lhs: inout Complex, rhs: Complex) {
+        lhs.real += rhs.real
+        lhs.imaginary += rhs.imaginary
+    }
+
+    @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
+    public static func - (lhs: Complex, rhs: Complex) -> Complex {
+        var temp = lhs
+        temp -= rhs
+        return temp
+    }
+
+    public static func -= (lhs: inout Complex, rhs: Complex) {
+        lhs.real -= rhs.real
+        lhs.imaginary -= rhs.imaginary
+    }
+}
+
+extension Complex : Numeric {
+    public init?<U>(exactly source: U) where U : BinaryInteger {
+        guard let t = T(exactly: source) else { return nil }
+        self.real = t
+        self.imaginary = 0
+    }
+
+    @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable)
+    public static func * (lhs: Complex, rhs: Complex) -> Complex {
+        var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
+        let ac = a * c, bd = b * d, ad = a * d, bc = b * c
+        let x = ac - bd
+        let y = ad + bc
+
+        if x.isNaN && y.isNaN {
+            var recalculate = false
+            if a.isInfinite || b.isInfinite {
+                a = T(signOf: a, magnitudeOf: a.isInfinite ? 1 : 0)
+                b = T(signOf: b, magnitudeOf: b.isInfinite ? 1 : 0)
+                if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
+                if d.isNaN { d = T(signOf: d, magnitudeOf: 0) }
+                recalculate = true
+            }
+            if c.isInfinite || d.isInfinite {
+                if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
+                if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
+                c = T(signOf: c, magnitudeOf: c.isInfinite ? 1 : 0)
+                d = T(signOf: d, magnitudeOf: d.isInfinite ? 1 : 0)
+                recalculate = true
+            }
+            if !recalculate &&
+                (ac.isInfinite || bd.isInfinite || ad.isInfinite || bc.isInfinite) {
+                if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
+                if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
+                if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
+                if d.isNaN { d = T(signOf: d, magnitudeOf: 0) }
+                recalculate = true
+            }
+            if recalculate {
+                return Complex(
+                real: .infinity * (a * c - b * d),
+                imaginary: .infinity * (a * d + b * c)
+                )
+            }
+        }
+        return Complex(real: x, imaginary: y)
+    }
+
+    public static func *= (lhs: inout Complex, rhs: Complex) {
+        lhs = lhs * rhs
+    }
+
+    public var magnitude: T {
+        var x = abs(real)
+        var y = abs(imaginary)
+        if x.isInfinite { return x }
+        if y.isInfinite { return y }
+        if x == 0 { return y }
+        if x < y { swap(&x, &y) }
+        let ratio = y / x
+        return x * (1 + ratio * ratio).squareRoot()
+    }
+}
+
+extension Complex : SignedNumeric {
+    @differentiable(vjp: _vjpNegate where T : Differentiable)
+    public static prefix func - (operand: Complex) -> Complex {
+        return Complex(real: -operand.real, imaginary: -operand.imaginary)
+    }
+
+    public mutating func negate() {
+        real.negate()
+        imaginary.negate()
+    }
+}
+
+extension Complex {
+    @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
+    public static func / (lhs: Complex, rhs: Complex) -> Complex {
+        var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
+        var x: T
+        var y: T
+        if c.magnitude >= d.magnitude {
+            let ratio = d / c
+            let denominator = c + d * ratio
+            x = (a + b * ratio) / denominator
+            y = (b - a * ratio) / denominator
+        } else {
+            let ratio = c / d
+            let denominator = c * ratio + d
+            x = (a * ratio + b) / denominator
+            y = (b * ratio - a) / denominator
+        }
+        if x.isNaN && y.isNaN {
+            if c == 0 && d == 0 && (!a.isNaN || !b.isNaN) {
+                x = T(signOf: c, magnitudeOf: .infinity) * a
+                y = T(signOf: c, magnitudeOf: .infinity) * b
+            } else if (a.isInfinite || b.isInfinite) && c.isFinite && d.isFinite {
+                a = T(signOf: a, magnitudeOf: a.isInfinite ? 1 : 0)
+                b = T(signOf: b, magnitudeOf: b.isInfinite ? 1 : 0)
+                x = .infinity * (a * c + b * d)
+                y = .infinity * (b * c - a * d)
+            } else if (c.isInfinite || d.isInfinite) && a.isFinite && b.isFinite {
+                c = T(signOf: c, magnitudeOf: c.isInfinite ? 1 : 0)
+                d = T(signOf: d, magnitudeOf: d.isInfinite ? 1 : 0)
+                x = 0 * (a * c + b * d)
+                y = 0 * (b * c - a * d)
+            }
+        }
+        return Complex(real: x, imaginary: y)
+    }
+
+
+    public static func /= (lhs: inout Complex, rhs: Complex) {
+        lhs = lhs / rhs
+    }
+}
+
+extension Complex {
+    @differentiable(vjp: _vjpComplexConjugate where T : Differentiable)
+    public func complexConjugate() -> Complex {
+        return Complex(real: real, imaginary: -imaginary)
+    }
+}
+
+public func abs<T>(_ z: Complex<T>) -> Complex<T> {
+    return Complex(real: z.magnitude)
+}
+
+extension Complex {
+    @differentiable(vjp: _vjpAdding(real:)
+        where T : Differentiable,
+              T.TangentVector == T)
+    public func adding(real: T) -> Complex {
+        var c = self
+        c.real += real
+        return c
+    }
+
+    @differentiable(vjp: _vjpSubtracting(real:)
+        where T : Differentiable,
+              T.TangentVector == T)
+    public func subtracting(real: T) -> Complex {
+        var c = self
+        c.real -= real
+        return c
+    }
+
+    @differentiable(vjp: _vjpAdding(imaginary:)
+        where T : Differentiable,
+              T.TangentVector == T)
+    public func adding(imaginary: T) -> Complex {
+        var c = self
+        c.imaginary += imaginary
+        return c
+    }
+
+    @differentiable(vjp: _vjpSubtracting(imaginary:)
+        where T : Differentiable,
+              T.TangentVector == T)
+    public func subtracting(imaginary: T) -> Complex {
+        var c = self
+        c.imaginary -= imaginary
+        return c
+    }
+}
+
+extension Complex where T : Differentiable, T.TangentVector == T {
+    @usableFromInline
+    static func _vjpInit(real: T, imaginary: T) -> (Complex, (Complex) -> (T, T)) {
+        return (Complex(real: real, imaginary: imaginary), { ($0.real, $0.imaginary) })
+    }
+}
+
+extension Complex where T : Differentiable {
+    @usableFromInline
+    static func _vjpAdd(lhs: Complex, rhs: Complex)
+    -> (Complex, (Complex) -> (Complex, Complex)) {
+        return (lhs + rhs, { v in (v, v) })
+    }
+
+    @usableFromInline
+    static func _vjpSubtract(lhs: Complex, rhs: Complex)
+    -> (Complex, (Complex) -> (Complex, Complex)) {
+        return (lhs - rhs, { v in (v, -v) })
+    }
+
+    @usableFromInline
+    static func _vjpMultiply(lhs: Complex, rhs: Complex)
+    -> (Complex, (Complex) -> (Complex, Complex)) {
+        return (lhs * rhs, { v in (rhs * v, lhs * v) })
+    }
+
+    @usableFromInline
+    static func _vjpDivide(lhs: Complex, rhs: Complex)
+    -> (Complex, (Complex) -> (Complex, Complex)) {
+        return (lhs / rhs, { v in (v / rhs, -lhs / (rhs * rhs) * v) })
+    }
+
+    @usableFromInline
+    static func _vjpNegate(operand: Complex)
+    -> (Complex, (Complex) -> Complex) {
+        return (-operand, { -$0 })
+    }
+
+    @usableFromInline
+    func _vjpComplexConjugate() -> (Complex, (Complex) -> Complex) {
+        return (complexConjugate(), { v in v.complexConjugate() })
+    }
+}
+
+extension Complex where T : Differentiable, T.TangentVector == T {
+    @usableFromInline
+    func _vjpAdding(real: T) -> (Complex, (Complex) -> (Complex, T)) {
+        return (self.adding(real: real), { ($0, $0.real) })
+    }
+
+    @usableFromInline
+    func _vjpSubtracting(real: T) -> (Complex, (Complex) -> (Complex, T)) {
+        return (self.subtracting(real: real), { ($0, -$0.real) })
+    }
+
+    @usableFromInline
+    func _vjpAdding(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
+        return (self.adding(real: real), { ($0, $0.imaginary) })
+    }
+
+    @usableFromInline
+    func _vjpSubtracting(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
+        return (self.subtracting(real: real), { ($0, -$0.imaginary) })
+    }
+}
diff --git a/Sources/third_party/Complex/LICENSE b/Sources/third_party/Complex/LICENSE
new file mode 100644
index 000000000..a3c663297
--- /dev/null
+++ b/Sources/third_party/Complex/LICENSE
@@ -0,0 +1,19 @@
+Copyright © 2017 Xiaodi Wu (https://github.com/xwu).
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

From 1e5758a0cc532fb3b60dc67f4611f61cfae3db66 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Thu, 23 May 2019 16:03:26 -0700
Subject: [PATCH 11/28] public -> internal.

---
 Sources/third_party/Complex/Complex.swift | 64 +++++++++++------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/Sources/third_party/Complex/Complex.swift b/Sources/third_party/Complex/Complex.swift
index b79ba8afa..182d22af3 100644
--- a/Sources/third_party/Complex/Complex.swift
+++ b/Sources/third_party/Complex/Complex.swift
@@ -1,54 +1,54 @@
 // A major part of the Complex number implementation comes from
 // [@xwu](https://github.com/xwu)'s project, [NumericAnnex](https://github.com/xwu/NumericAnnex).
 
-public struct Complex<T : FloatingPoint> {
-    public var real: T
-    public var imaginary: T
+struct Complex<T : FloatingPoint> {
+    var real: T
+    var imaginary: T
 
     @differentiable(vjp: _vjpInit where T : Differentiable, T.TangentVector == T)
-    public init(real: T = 0, imaginary: T = 0) {
+    init(real: T = 0, imaginary: T = 0) {
         self.real = real
         self.imaginary = imaginary
     }
 }
 
 extension Complex : Differentiable where T : Differentiable {
-    public typealias TangentVector = Complex
-    public typealias AllDifferentiableVariables = Complex
+    typealias TangentVector = Complex
+    typealias AllDifferentiableVariables = Complex
 }
 
 extension Complex {
-    public static var i: Complex {
+    static var i: Complex {
         return Complex(real: 0, imaginary: 1)
     }
 
-    public var isFinite: Bool {
+    var isFinite: Bool {
         return real.isFinite && imaginary.isFinite
     }
 
-    public var isInfinite: Bool {
+    var isInfinite: Bool {
         return real.isInfinite || imaginary.isInfinite
     }
 
-    public var isNaN: Bool {
+    var isNaN: Bool {
         return (real.isNaN && !imaginary.isInfinite) ||
         (imaginary.isNaN && !real.isInfinite)
     }
 
-    public var isZero: Bool {
+    var isZero: Bool {
         return real.isZero && imaginary.isZero
     }
 }
 
 extension Complex : ExpressibleByIntegerLiteral {
-    public init(integerLiteral value: Int) {
+    init(integerLiteral value: Int) {
         self.real = T(value)
         self.imaginary = 0
     }
 }
 
 extension Complex : CustomStringConvertible {
-    public var description: String {
+    var description: String {
         return real.isNaN && real.sign == .minus
             ? imaginary.sign == .minus
                 ? "-\(-real) - \(-imaginary)i"
@@ -60,46 +60,46 @@ extension Complex : CustomStringConvertible {
 }
 
 extension Complex : Equatable {
-    public static func == (lhs: Complex, rhs: Complex) -> Bool {
+    c static func == (lhs: Complex, rhs: Complex) -> Bool {
         return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
     }
 }
 
 extension Complex : AdditiveArithmetic {
     @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
-    public static func + (lhs: Complex, rhs: Complex) -> Complex {
+    static func + (lhs: Complex, rhs: Complex) -> Complex {
         var temp = lhs
         temp += rhs
         return temp
     }
 
-    public static func += (lhs: inout Complex, rhs: Complex) {
+    static func += (lhs: inout Complex, rhs: Complex) {
         lhs.real += rhs.real
         lhs.imaginary += rhs.imaginary
     }
 
     @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
-    public static func - (lhs: Complex, rhs: Complex) -> Complex {
+    static func - (lhs: Complex, rhs: Complex) -> Complex {
         var temp = lhs
         temp -= rhs
         return temp
     }
 
-    public static func -= (lhs: inout Complex, rhs: Complex) {
+    static func -= (lhs: inout Complex, rhs: Complex) {
         lhs.real -= rhs.real
         lhs.imaginary -= rhs.imaginary
     }
 }
 
 extension Complex : Numeric {
-    public init?<U>(exactly source: U) where U : BinaryInteger {
+    init?<U>(exactly source: U) where U : BinaryInteger {
         guard let t = T(exactly: source) else { return nil }
         self.real = t
         self.imaginary = 0
     }
 
     @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable)
-    public static func * (lhs: Complex, rhs: Complex) -> Complex {
+    static func * (lhs: Complex, rhs: Complex) -> Complex {
         var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
         let ac = a * c, bd = b * d, ad = a * d, bc = b * c
         let x = ac - bd
@@ -139,11 +139,11 @@ extension Complex : Numeric {
         return Complex(real: x, imaginary: y)
     }
 
-    public static func *= (lhs: inout Complex, rhs: Complex) {
+    static func *= (lhs: inout Complex, rhs: Complex) {
         lhs = lhs * rhs
     }
 
-    public var magnitude: T {
+    var magnitude: T {
         var x = abs(real)
         var y = abs(imaginary)
         if x.isInfinite { return x }
@@ -157,11 +157,11 @@ extension Complex : Numeric {
 
 extension Complex : SignedNumeric {
     @differentiable(vjp: _vjpNegate where T : Differentiable)
-    public static prefix func - (operand: Complex) -> Complex {
+    static prefix func - (operand: Complex) -> Complex {
         return Complex(real: -operand.real, imaginary: -operand.imaginary)
     }
 
-    public mutating func negate() {
+    mutating func negate() {
         real.negate()
         imaginary.negate()
     }
@@ -169,7 +169,7 @@ extension Complex : SignedNumeric {
 
 extension Complex {
     @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
-    public static func / (lhs: Complex, rhs: Complex) -> Complex {
+    static func / (lhs: Complex, rhs: Complex) -> Complex {
         var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
         var x: T
         var y: T
@@ -204,19 +204,19 @@ extension Complex {
     }
 
 
-    public static func /= (lhs: inout Complex, rhs: Complex) {
+    static func /= (lhs: inout Complex, rhs: Complex) {
         lhs = lhs / rhs
     }
 }
 
 extension Complex {
     @differentiable(vjp: _vjpComplexConjugate where T : Differentiable)
-    public func complexConjugate() -> Complex {
+    func complexConjugate() -> Complex {
         return Complex(real: real, imaginary: -imaginary)
     }
 }
 
-public func abs<T>(_ z: Complex<T>) -> Complex<T> {
+func abs<T>(_ z: Complex<T>) -> Complex<T> {
     return Complex(real: z.magnitude)
 }
 
@@ -224,7 +224,7 @@ extension Complex {
     @differentiable(vjp: _vjpAdding(real:)
         where T : Differentiable,
               T.TangentVector == T)
-    public func adding(real: T) -> Complex {
+    func adding(real: T) -> Complex {
         var c = self
         c.real += real
         return c
@@ -233,7 +233,7 @@ extension Complex {
     @differentiable(vjp: _vjpSubtracting(real:)
         where T : Differentiable,
               T.TangentVector == T)
-    public func subtracting(real: T) -> Complex {
+    func subtracting(real: T) -> Complex {
         var c = self
         c.real -= real
         return c
@@ -242,7 +242,7 @@ extension Complex {
     @differentiable(vjp: _vjpAdding(imaginary:)
         where T : Differentiable,
               T.TangentVector == T)
-    public func adding(imaginary: T) -> Complex {
+    func adding(imaginary: T) -> Complex {
         var c = self
         c.imaginary += imaginary
         return c
@@ -251,7 +251,7 @@ extension Complex {
     @differentiable(vjp: _vjpSubtracting(imaginary:)
         where T : Differentiable,
               T.TangentVector == T)
-    public func subtracting(imaginary: T) -> Complex {
+    func subtracting(imaginary: T) -> Complex {
         var c = self
         c.imaginary -= imaginary
         return c

From e58f4003c9c9a0ccca9495e08cf520d7f288cc4e Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Thu, 23 May 2019 16:14:04 -0700
Subject: [PATCH 12/28] Fix tests for running with XCTest.

---
 .../ComplexAutoDiffTests.swift                |  98 -----
 Tests/DeepLearningTests/ComplexTests.swift    | 364 +++++++++++-------
 2 files changed, 224 insertions(+), 238 deletions(-)
 delete mode 100644 Tests/DeepLearningTests/ComplexAutoDiffTests.swift

diff --git a/Tests/DeepLearningTests/ComplexAutoDiffTests.swift b/Tests/DeepLearningTests/ComplexAutoDiffTests.swift
deleted file mode 100644
index b49e85e34..000000000
--- a/Tests/DeepLearningTests/ComplexAutoDiffTests.swift
+++ /dev/null
@@ -1,98 +0,0 @@
-// RUN: %target-run-simple-swift
-// REQUIRES: executable_test
-//
-// Complex API tests.
-
-// TODO: remove import
-import TensorFlow
-
-import StdlibUnittest
-
-var AutoDiffComplexTests = TestSuite("AutoDiffComplex")
-
-AutoDiffComplexTests.test("_vjpAdd") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
-    return x + Complex<Float>(real: 5, imaginary: 6)
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-}
-
-AutoDiffComplexTests.test("_vjpSubtract") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
-    return Complex<Float>(real: 5, imaginary: 6) - x
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
-}
-
-AutoDiffComplexTests.test("_vjpMultiply") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
-    return x * x
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 4, imaginary: 6))
-  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: -6, imaginary: 4))
-  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -2, imaginary: 10))
-}
-
-AutoDiffComplexTests.test("_vjpDivide") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-    return x / Complex<Float>(real: 2, imaginary: 2)
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 0.25, imaginary: -0.25))
-  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0.25, imaginary: 0.25))
-}
-
-AutoDiffComplexTests.test("_vjpNegate") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-    return -x
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
-  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
-  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
-}
-
-AutoDiffComplexTests.test("_vjpComplexConjugate") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-    return x.complexConjugate()
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
-  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
-  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
-}
-
-AutoDiffComplexTests.test("_vjpAdding(real:)") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-    return x.adding(real: 5)
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-}
-
-AutoDiffComplexTests.test("_vjpAdding(imaginary:)") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-    return x.adding(imaginary: 5)
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-}
-
-AutoDiffComplexTests.test("_vjpSubtracting(real:)") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-    return x.subtracting(real: 5)
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-}
-
-AutoDiffComplexTests.test("_vjpSubtracting(imaginary:)") {
-  let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-    return x.subtracting(imaginary: 5)
-  }
-  expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-  expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-  expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-}
-
-runAllTests()
\ No newline at end of file
diff --git a/Tests/DeepLearningTests/ComplexTests.swift b/Tests/DeepLearningTests/ComplexTests.swift
index 5d9b9e00f..f2afe1b30 100644
--- a/Tests/DeepLearningTests/ComplexTests.swift
+++ b/Tests/DeepLearningTests/ComplexTests.swift
@@ -3,147 +3,231 @@
 //
 // Complex API tests.
 
-// TODO: remove import
-import TensorFlow
-
-import StdlibUnittest
-
-var ComplexTests = TestSuite("Complex")
-
-ComplexTests.test("Initializer") {
-  let complex = Complex<Float>(real: 2, imaginary: 3)
-  expectEqual(complex.real, 2)
-  expectEqual(complex.imaginary, 3)
-}
-
-ComplexTests.test("Static Imaginary") {
-  let imaginary = Complex<Float>(real: 0, imaginary: 1)
-  expectEqual(imaginary, Complex.i)
-}
-
-ComplexTests.test("isFinite") {
-  var complex = Complex<Float>(real: 999, imaginary: 0)
-  expectTrue(complex.isFinite)
-
-  complex = Complex(real: 1.0 / 0.0, imaginary: 1)
-  expectFalse(complex.isFinite)
-
-  complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
-  expectFalse(complex.isFinite)
-}
-
-ComplexTests.test("isInfinite") {
-  var complex = Complex<Float>(real: 999, imaginary: 0)
-  expectFalse(complex.isInfinite)
-
-  complex = Complex(real: 1.0 / 0.0, imaginary: 1)
-  expectTrue(complex.isInfinite)
-
-  complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
-  expectTrue(complex.isInfinite)
-}
-
-ComplexTests.test("isNaN") {
-  var complex = Complex<Float>(real: 999, imaginary: 0)
-  expectFalse(complex.isNaN)
-
-  complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 1)
-  expectTrue(complex.isNaN)
-
-  complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
-  expectTrue(complex.isNaN)
-}
-
-ComplexTests.test("isZero") {
-  var complex = Complex<Float>(real: 999, imaginary: 0)
-  expectFalse(complex.isZero)
-
-  complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0)
-  expectFalse(complex.isZero)
-
-  complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
-  expectFalse(complex.isZero)
-
-  complex = Complex(real: 0, imaginary: 0)
-  expectTrue(complex.isZero)
+import XCTest
+@testable import DeepLearning
+
+final class ComplexTests: XCTestCase {
+  func testInitializer() {
+    let complex = Complex<Float>(real: 2, imaginary: 3)
+    XCTAssertEqual(complex.real, 2)
+    XCTAssertEqual(complex.imaginary, 3)
+  }
+
+  func testStaticImaginary() {
+    let imaginary = Complex<Float>(real: 0, imaginary: 1)
+    XCTAssertEqual(imaginary, Complex.i)
+  }
+
+  func testIsFinite() {
+    var complex = Complex<Float>(real: 999, imaginary: 0)
+    XCTAssertTrue(complex.isFinite)
+
+    complex = Complex(real: 1.0 / 0.0, imaginary: 1)
+    XCTAssertFalse(complex.isFinite)
+
+    complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
+    XCTAssertFalse(complex.isFinite)
+  }
+
+  func testIsInfinite() {
+    var complex = Complex<Float>(real: 999, imaginary: 0)
+    XCTAssertFalse(complex.isInfinite)
+
+    complex = Complex(real: 1.0 / 0.0, imaginary: 1)
+    XCTAssertTrue(complex.isInfinite)
+
+    complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
+    XCTAssertTrue(complex.isInfinite)
+  }
+
+  func testIsNaN() {
+    var complex = Complex<Float>(real: 999, imaginary: 0)
+    XCTAssertFalse(complex.isNaN)
+
+    complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 1)
+    XCTAssertTrue(complex.isNaN)
+
+    complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
+    XCTAssertTrue(complex.isNaN)
+  }
+
+  func testIsZero() {
+    var complex = Complex<Float>(real: 999, imaginary: 0)
+    XCTAssertFalse(complex.isZero)
+
+    complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0)
+    XCTAssertFalse(complex.isZero)
+
+    complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
+    XCTAssertFalse(complex.isZero)
+
+    complex = Complex(real: 0, imaginary: 0)
+    XCTAssertTrue(complex.isZero)
+  }
+
+  func testEquals() {
+    var complexA = Complex<Float>(real: 999, imaginary: 0)
+    let complexB = Complex<Float>(real: 999, imaginary: 0)
+    XCTAssertEqual(complexA, complexB)
+
+    complexA = Complex(real: 5, imaginary: 0)
+    expectNotEqual(complexA, complexB)
+  }
+
+  func testPlus() {
+    let input = Complex<Float>(real: 5, imaginary: 1)
+    let expected = Complex<Float>(real: 10, imaginary: 2)
+    XCTAssertEqual(expected, input + input)
+  }
+
+  func testMinus() {
+    let inputA = Complex<Float>(real: 6, imaginary: 2)
+    let inputB = Complex<Float>(real: 5, imaginary: 1)
+    let expected = Complex<Float>(real: 1, imaginary: 1)
+    XCTAssertEqual(expected, inputA - inputB)
+  }
+
+  func testTimes() {
+    let inputA = Complex<Float>(real: 6, imaginary: 2)
+    let inputB = Complex<Float>(real: 5, imaginary: 1)
+    let expected = Complex<Float>(real: 28, imaginary: 16)
+    XCTAssertEqual(expected, inputA * inputB)
+  }
+
+  func testNegate() {
+    var input = Complex<Float>(real: 6, imaginary: 2)
+    let negated = Complex<Float>(real: -6, imaginary: -2)
+    XCTAssertEqual(-input, negated)
+    input.negate()
+    XCTAssertEqual(input, negated)
+  }
+
+  func testDivide() {
+    let inputA = Complex<Float>(real: 20, imaginary: -4)
+    let inputB = Complex<Float>(real: 3, imaginary: 2)
+    let expected = Complex<Float>(real: 4, imaginary: -4)
+    XCTAssertEqual(expected, inputA / inputB)
+  }
+
+  func testComplexConjugate() {
+    var input = Complex<Float>(real: 2, imaginary: -4)
+    var expected = Complex<Float>(real: 2, imaginary: 4)
+    XCTAssertEqual(expected, input.complexConjugate())
+
+    input = Complex<Float>(real: -2, imaginary: -4)
+    expected = Complex<Float>(real: -2, imaginary: 4)
+    XCTAssertEqual(expected, input.complexConjugate())
+
+    input = Complex<Float>(real: 2, imaginary: 4)
+    expected = Complex<Float>(real: 2, imaginary: -4)
+    XCTAssertEqual(expected, input.complexConjugate())
+  }
+
+  func testAdding() {
+    var input = Complex<Float>(real: 2, imaginary: -4)
+    var expected = Complex<Float>(real: 3, imaginary: -4)
+    XCTAssertEqual(expected, input.adding(real: 1))
+
+    input = Complex<Float>(real: 2, imaginary: -4)
+    expected = Complex<Float>(real: 2, imaginary: -3)
+    XCTAssertEqual(expected, input.adding(imaginary: 1))
+  }
+
+  func testSubtracting() {
+    var input = Complex<Float>(real: 2, imaginary: -4)
+    var expected = Complex<Float>(real: 1, imaginary: -4)
+    XCTAssertEqual(expected, input.subtracting(real: 1))
+
+    input = Complex<Float>(real: 2, imaginary: -4)
+    expected = Complex<Float>(real: 2, imaginary: -5)
+    XCTAssertEqual(expected, input.subtracting(imaginary: 1))
+  }
+
+  func testVjpAdd() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+      return x + Complex<Float>(real: 5, imaginary: 6)
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+  }
+
+  func testVjpSubtract() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+      return Complex<Float>(real: 5, imaginary: 6) - x
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+  }
+
+  func testVjpMultiply() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+      return x * x
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 4, imaginary: 6))
+    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: -6, imaginary: 4))
+    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -2, imaginary: 10))
+  }
+
+  func testVjpDivide() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+      return x / Complex<Float>(real: 2, imaginary: 2)
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 0.25, imaginary: -0.25))
+    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0.25, imaginary: 0.25))
+  }
+
+  func testVjpNegate() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+      return -x
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
+    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
+    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+  }
+
+  func testVjpComplexConjugate() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+      return x.complexConjugate()
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
+    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
+    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+  }
+
+  func testVjpAdding() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+      return x.adding(real: 5)
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+  }
+
+  func testVjpAdding() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+      return x.adding(imaginary: 5)
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+  }
+
+  func testVjpSubtracting() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+      return x.subtracting(real: 5)
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+  }
+
+  func testVjpSubtracting() {
+    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+      return x.subtracting(imaginary: 5)
+    }
+    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+  }
 }
 
-ComplexTests.test("==") {
-  var complexA = Complex<Float>(real: 999, imaginary: 0)
-  let complexB = Complex<Float>(real: 999, imaginary: 0)
-  expectEqual(complexA, complexB)
-
-  complexA = Complex(real: 5, imaginary: 0)
-  expectNotEqual(complexA, complexB)
-}
-
-ComplexTests.test("+") {
-  let input = Complex<Float>(real: 5, imaginary: 1)
-  let expected = Complex<Float>(real: 10, imaginary: 2)
-  expectEqual(expected, input + input)
-}
-
-ComplexTests.test("-") {
-  let inputA = Complex<Float>(real: 6, imaginary: 2)
-  let inputB = Complex<Float>(real: 5, imaginary: 1)
-  let expected = Complex<Float>(real: 1, imaginary: 1)
-  expectEqual(expected, inputA - inputB)
-}
-
-ComplexTests.test("*") {
-  let inputA = Complex<Float>(real: 6, imaginary: 2)
-  let inputB = Complex<Float>(real: 5, imaginary: 1)
-  let expected = Complex<Float>(real: 28, imaginary: 16)
-  expectEqual(expected, inputA * inputB)
-}
-
-ComplexTests.test("negate") {
-  var input = Complex<Float>(real: 6, imaginary: 2)
-  let negated = Complex<Float>(real: -6, imaginary: -2)
-  expectEqual(-input, negated)
-  input.negate()
-  expectEqual(input, negated)
-}
-
-ComplexTests.test("/") {
-  let inputA = Complex<Float>(real: 20, imaginary: -4)
-  let inputB = Complex<Float>(real: 3, imaginary: 2)
-  let expected = Complex<Float>(real: 4, imaginary: -4)
-  expectEqual(expected, inputA / inputB)
-}
-
-ComplexTests.test("complexConjugate") {
-  var input = Complex<Float>(real: 2, imaginary: -4)
-  var expected = Complex<Float>(real: 2, imaginary: 4)
-  expectEqual(expected, input.complexConjugate())
-
-  input = Complex<Float>(real: -2, imaginary: -4)
-  expected = Complex<Float>(real: -2, imaginary: 4)
-  expectEqual(expected, input.complexConjugate())
-
-  input = Complex<Float>(real: 2, imaginary: 4)
-  expected = Complex<Float>(real: 2, imaginary: -4)
-  expectEqual(expected, input.complexConjugate())
-}
-
-ComplexTests.test("adding") {
-  var input = Complex<Float>(real: 2, imaginary: -4)
-  var expected = Complex<Float>(real: 3, imaginary: -4)
-  expectEqual(expected, input.adding(real: 1))
-
-  input = Complex<Float>(real: 2, imaginary: -4)
-  expected = Complex<Float>(real: 2, imaginary: -3)
-  expectEqual(expected, input.adding(imaginary: 1))
-}
-
-ComplexTests.test("subtracting") {
-  var input = Complex<Float>(real: 2, imaginary: -4)
-  var expected = Complex<Float>(real: 1, imaginary: -4)
-  expectEqual(expected, input.subtracting(real: 1))
-
-  input = Complex<Float>(real: 2, imaginary: -4)
-  expected = Complex<Float>(real: 2, imaginary: -5)
-  expectEqual(expected, input.subtracting(imaginary: 1))
-}
 
 runAllTests()
\ No newline at end of file

From 4ac97e3de50690bd3c016ee4c95a23f6c30140a5 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Thu, 23 May 2019 20:58:28 -0700
Subject: [PATCH 13/28] Add target to package for third_party.

---
 Package.swift                                 |  5 +-
 .../third_party/{Complex => }/Complex.swift   |  2 +-
 Sources/third_party/{Complex => }/LICENSE     |  0
 Tests/DeepLearningTests/ComplexTests.swift    | 50 +++++++++----------
 4 files changed, 30 insertions(+), 27 deletions(-)
 rename Sources/third_party/{Complex => }/Complex.swift (99%)
 rename Sources/third_party/{Complex => }/LICENSE (100%)

diff --git a/Package.swift b/Package.swift
index 1ee137e0d..a9d99e42d 100644
--- a/Package.swift
+++ b/Package.swift
@@ -22,13 +22,16 @@ let package = Package(
     products: [
         .library(
             name: "DeepLearning",
-            targets: ["DeepLearning"]),
+            targets: ["DeepLearning", "third_party"]),
     ],
     dependencies: [],
     targets: [
         .target(
             name: "DeepLearning",
             dependencies: []),
+        .target(
+            name: "third_party",
+            dependencies: []),
         .testTarget(
             name: "DeepLearningTests",
             dependencies: ["DeepLearning"]),
diff --git a/Sources/third_party/Complex/Complex.swift b/Sources/third_party/Complex.swift
similarity index 99%
rename from Sources/third_party/Complex/Complex.swift
rename to Sources/third_party/Complex.swift
index 182d22af3..db9634509 100644
--- a/Sources/third_party/Complex/Complex.swift
+++ b/Sources/third_party/Complex.swift
@@ -60,7 +60,7 @@ extension Complex : CustomStringConvertible {
 }
 
 extension Complex : Equatable {
-    c static func == (lhs: Complex, rhs: Complex) -> Bool {
+    static func == (lhs: Complex, rhs: Complex) -> Bool {
         return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
     }
 }
diff --git a/Sources/third_party/Complex/LICENSE b/Sources/third_party/LICENSE
similarity index 100%
rename from Sources/third_party/Complex/LICENSE
rename to Sources/third_party/LICENSE
diff --git a/Tests/DeepLearningTests/ComplexTests.swift b/Tests/DeepLearningTests/ComplexTests.swift
index dc95e1866..938ad5798 100644
--- a/Tests/DeepLearningTests/ComplexTests.swift
+++ b/Tests/DeepLearningTests/ComplexTests.swift
@@ -147,84 +147,84 @@ final class ComplexTests: XCTestCase {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
       return x + Complex<Float>(real: 5, imaginary: 6)
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
   }
 
   func testVjpSubtract() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
       return Complex<Float>(real: 5, imaginary: 6) - x
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
   }
 
   func testVjpMultiply() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
       return x * x
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 4, imaginary: 6))
-    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: -6, imaginary: 4))
-    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -2, imaginary: 10))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 4, imaginary: 6))
+    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: -6, imaginary: 4))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -2, imaginary: 10))
   }
 
   func testVjpDivide() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x / Complex<Float>(real: 2, imaginary: 2)
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 0.25, imaginary: -0.25))
-    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0.25, imaginary: 0.25))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 0.25, imaginary: -0.25))
+    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0.25, imaginary: 0.25))
   }
 
   func testVjpNegate() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return -x
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
-    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
-    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
+    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
   }
 
   func testVjpComplexConjugate() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x.complexConjugate()
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
-    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
-    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
+    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
   }
 
   func testVjpAdding() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x.adding(real: 5)
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
   }
 
   func testVjpAdding() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x.adding(imaginary: 5)
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
   }
 
   func testVjpSubtracting() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x.subtracting(real: 5)
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
   }
 
   func testVjpSubtracting() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x.subtracting(imaginary: 5)
     }
-    expectEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-    expectEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-    expectEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
   }
 }
\ No newline at end of file

From 3350c2112e4322a3ee149436cceb0e5aa1a86218 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Thu, 23 May 2019 21:51:51 -0700
Subject: [PATCH 14/28] WIP: see if tests pass while figuring out target issue.

---
 Package.swift                                       |  8 ++++----
 Sources/{third_party => DeepLearning}/Complex.swift |  0
 Tests/DeepLearningTests/ComplexTests.swift          | 10 +++++-----
 3 files changed, 9 insertions(+), 9 deletions(-)
 rename Sources/{third_party => DeepLearning}/Complex.swift (100%)

diff --git a/Package.swift b/Package.swift
index a9d99e42d..4a981ba26 100644
--- a/Package.swift
+++ b/Package.swift
@@ -22,16 +22,16 @@ let package = Package(
     products: [
         .library(
             name: "DeepLearning",
-            targets: ["DeepLearning", "third_party"]),
+            targets: ["DeepLearning"/*, "third_party"*/]),
     ],
     dependencies: [],
     targets: [
         .target(
             name: "DeepLearning",
             dependencies: []),
-        .target(
-            name: "third_party",
-            dependencies: []),
+        // .target(
+        //     name: "third_party",
+        //     dependencies: []),
         .testTarget(
             name: "DeepLearningTests",
             dependencies: ["DeepLearning"]),
diff --git a/Sources/third_party/Complex.swift b/Sources/DeepLearning/Complex.swift
similarity index 100%
rename from Sources/third_party/Complex.swift
rename to Sources/DeepLearning/Complex.swift
diff --git a/Tests/DeepLearningTests/ComplexTests.swift b/Tests/DeepLearningTests/ComplexTests.swift
index 938ad5798..904c949e7 100644
--- a/Tests/DeepLearningTests/ComplexTests.swift
+++ b/Tests/DeepLearningTests/ComplexTests.swift
@@ -71,7 +71,7 @@ final class ComplexTests: XCTestCase {
     XCTAssertEqual(complexA, complexB)
 
     complexA = Complex(real: 5, imaginary: 0)
-    expectNotEqual(complexA, complexB)
+    XCTAssertNotEqual(complexA, complexB)
   }
 
   func testPlus() {
@@ -192,7 +192,7 @@ final class ComplexTests: XCTestCase {
     XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
   }
 
-  func testVjpAdding() {
+  func testVjpAddingReal() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x.adding(real: 5)
     }
@@ -201,7 +201,7 @@ final class ComplexTests: XCTestCase {
     XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
   }
 
-  func testVjpAdding() {
+  func testVjpAddingImaginary() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x.adding(imaginary: 5)
     }
@@ -210,7 +210,7 @@ final class ComplexTests: XCTestCase {
     XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
   }
 
-  func testVjpSubtracting() {
+  func testVjpSubtractingReal() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x.subtracting(real: 5)
     }
@@ -219,7 +219,7 @@ final class ComplexTests: XCTestCase {
     XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
   }
 
-  func testVjpSubtracting() {
+  func testVjpSubtractingImaginary() {
     let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
       return x.subtracting(imaginary: 5)
     }

From 85b43bbf6ca7a9d0ba2c2c07e3212179bb09c2b4 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Fri, 24 May 2019 09:35:14 -0700
Subject: [PATCH 15/28] WIP: see if tests pass while local toolchain is old.

---
 Package.swift                                 |  14 +-
 .../Complex.swift                             |   0
 Tests/DeepLearningTests/ComplexTests.swift    | 502 ++++++++++--------
 3 files changed, 289 insertions(+), 227 deletions(-)
 rename Sources/{DeepLearning => third_party}/Complex.swift (100%)

diff --git a/Package.swift b/Package.swift
index 4a981ba26..9d1531ef7 100644
--- a/Package.swift
+++ b/Package.swift
@@ -22,18 +22,22 @@ let package = Package(
     products: [
         .library(
             name: "DeepLearning",
-            targets: ["DeepLearning"/*, "third_party"*/]),
+            targets: ["DeepLearning"]),
+        .library(
+            name: "third_party",
+            targets: ["third_party"])
     ],
     dependencies: [],
     targets: [
         .target(
             name: "DeepLearning",
             dependencies: []),
-        // .target(
-        //     name: "third_party",
-        //     dependencies: []),
+        .target(
+            name: "third_party",
+            dependencies: [],
+            path: "Sources/third_party"),
         .testTarget(
             name: "DeepLearningTests",
-            dependencies: ["DeepLearning"]),
+            dependencies: ["DeepLearning", "third_party"]),
     ]
 )
diff --git a/Sources/DeepLearning/Complex.swift b/Sources/third_party/Complex.swift
similarity index 100%
rename from Sources/DeepLearning/Complex.swift
rename to Sources/third_party/Complex.swift
diff --git a/Tests/DeepLearningTests/ComplexTests.swift b/Tests/DeepLearningTests/ComplexTests.swift
index 904c949e7..13e584498 100644
--- a/Tests/DeepLearningTests/ComplexTests.swift
+++ b/Tests/DeepLearningTests/ComplexTests.swift
@@ -1,230 +1,288 @@
-// RUN: %target-run-simple-swift
-// REQUIRES: executable_test
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 //
-// Complex API tests.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 import XCTest
-@testable import DeepLearning
+@testable import third_party
 
 final class ComplexTests: XCTestCase {
-  func testInitializer() {
-    let complex = Complex<Float>(real: 2, imaginary: 3)
-    XCTAssertEqual(complex.real, 2)
-    XCTAssertEqual(complex.imaginary, 3)
-  }
-
-  func testStaticImaginary() {
-    let imaginary = Complex<Float>(real: 0, imaginary: 1)
-    XCTAssertEqual(imaginary, Complex.i)
-  }
-
-  func testIsFinite() {
-    var complex = Complex<Float>(real: 999, imaginary: 0)
-    XCTAssertTrue(complex.isFinite)
-
-    complex = Complex(real: 1.0 / 0.0, imaginary: 1)
-    XCTAssertFalse(complex.isFinite)
-
-    complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
-    XCTAssertFalse(complex.isFinite)
-  }
-
-  func testIsInfinite() {
-    var complex = Complex<Float>(real: 999, imaginary: 0)
-    XCTAssertFalse(complex.isInfinite)
-
-    complex = Complex(real: 1.0 / 0.0, imaginary: 1)
-    XCTAssertTrue(complex.isInfinite)
-
-    complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
-    XCTAssertTrue(complex.isInfinite)
-  }
-
-  func testIsNaN() {
-    var complex = Complex<Float>(real: 999, imaginary: 0)
-    XCTAssertFalse(complex.isNaN)
-
-    complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 1)
-    XCTAssertTrue(complex.isNaN)
-
-    complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
-    XCTAssertTrue(complex.isNaN)
-  }
-
-  func testIsZero() {
-    var complex = Complex<Float>(real: 999, imaginary: 0)
-    XCTAssertFalse(complex.isZero)
-
-    complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0)
-    XCTAssertFalse(complex.isZero)
-
-    complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
-    XCTAssertFalse(complex.isZero)
-
-    complex = Complex(real: 0, imaginary: 0)
-    XCTAssertTrue(complex.isZero)
-  }
-
-  func testEquals() {
-    var complexA = Complex<Float>(real: 999, imaginary: 0)
-    let complexB = Complex<Float>(real: 999, imaginary: 0)
-    XCTAssertEqual(complexA, complexB)
-
-    complexA = Complex(real: 5, imaginary: 0)
-    XCTAssertNotEqual(complexA, complexB)
-  }
-
-  func testPlus() {
-    let input = Complex<Float>(real: 5, imaginary: 1)
-    let expected = Complex<Float>(real: 10, imaginary: 2)
-    XCTAssertEqual(expected, input + input)
-  }
-
-  func testMinus() {
-    let inputA = Complex<Float>(real: 6, imaginary: 2)
-    let inputB = Complex<Float>(real: 5, imaginary: 1)
-    let expected = Complex<Float>(real: 1, imaginary: 1)
-    XCTAssertEqual(expected, inputA - inputB)
-  }
-
-  func testTimes() {
-    let inputA = Complex<Float>(real: 6, imaginary: 2)
-    let inputB = Complex<Float>(real: 5, imaginary: 1)
-    let expected = Complex<Float>(real: 28, imaginary: 16)
-    XCTAssertEqual(expected, inputA * inputB)
-  }
-
-  func testNegate() {
-    var input = Complex<Float>(real: 6, imaginary: 2)
-    let negated = Complex<Float>(real: -6, imaginary: -2)
-    XCTAssertEqual(-input, negated)
-    input.negate()
-    XCTAssertEqual(input, negated)
-  }
-
-  func testDivide() {
-    let inputA = Complex<Float>(real: 20, imaginary: -4)
-    let inputB = Complex<Float>(real: 3, imaginary: 2)
-    let expected = Complex<Float>(real: 4, imaginary: -4)
-    XCTAssertEqual(expected, inputA / inputB)
-  }
-
-  func testComplexConjugate() {
-    var input = Complex<Float>(real: 2, imaginary: -4)
-    var expected = Complex<Float>(real: 2, imaginary: 4)
-    XCTAssertEqual(expected, input.complexConjugate())
-
-    input = Complex<Float>(real: -2, imaginary: -4)
-    expected = Complex<Float>(real: -2, imaginary: 4)
-    XCTAssertEqual(expected, input.complexConjugate())
-
-    input = Complex<Float>(real: 2, imaginary: 4)
-    expected = Complex<Float>(real: 2, imaginary: -4)
-    XCTAssertEqual(expected, input.complexConjugate())
-  }
-
-  func testAdding() {
-    var input = Complex<Float>(real: 2, imaginary: -4)
-    var expected = Complex<Float>(real: 3, imaginary: -4)
-    XCTAssertEqual(expected, input.adding(real: 1))
-
-    input = Complex<Float>(real: 2, imaginary: -4)
-    expected = Complex<Float>(real: 2, imaginary: -3)
-    XCTAssertEqual(expected, input.adding(imaginary: 1))
-  }
-
-  func testSubtracting() {
-    var input = Complex<Float>(real: 2, imaginary: -4)
-    var expected = Complex<Float>(real: 1, imaginary: -4)
-    XCTAssertEqual(expected, input.subtracting(real: 1))
-
-    input = Complex<Float>(real: 2, imaginary: -4)
-    expected = Complex<Float>(real: 2, imaginary: -5)
-    XCTAssertEqual(expected, input.subtracting(imaginary: 1))
-  }
-
-  func testVjpAdd() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
-      return x + Complex<Float>(real: 5, imaginary: 6)
+    func testInitializer() {
+        let complex = Complex<Float>(real: 2, imaginary: 3)
+        XCTAssertEqual(complex.real, 2)
+        XCTAssertEqual(complex.imaginary, 3)
     }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-  }
-
-  func testVjpSubtract() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
-      return Complex<Float>(real: 5, imaginary: 6) - x
+    
+    func testStaticImaginary() {
+        let imaginary = Complex<Float>(real: 0, imaginary: 1)
+        XCTAssertEqual(imaginary, Complex.i)
     }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
-  }
-
-  func testVjpMultiply() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
-      return x * x
-    }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 4, imaginary: 6))
-    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: -6, imaginary: 4))
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -2, imaginary: 10))
-  }
-
-  func testVjpDivide() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-      return x / Complex<Float>(real: 2, imaginary: 2)
+    
+    func testIsFinite() {
+        var complex = Complex<Float>(real: 999, imaginary: 0)
+        XCTAssertTrue(complex.isFinite)
+        
+        complex = Complex(real: 1.0 / 0.0, imaginary: 1)
+        XCTAssertFalse(complex.isFinite)
+        
+        complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
+        XCTAssertFalse(complex.isFinite)
     }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 0.25, imaginary: -0.25))
-    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0.25, imaginary: 0.25))
-  }
-
-  func testVjpNegate() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-      return -x
-    }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
-    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
-  }
-
-  func testVjpComplexConjugate() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-      return x.complexConjugate()
-    }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
-    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
-  }
-
-  func testVjpAddingReal() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-      return x.adding(real: 5)
-    }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-  }
-
-  func testVjpAddingImaginary() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-      return x.adding(imaginary: 5)
-    }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-  }
-
-  func testVjpSubtractingReal() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-      return x.subtracting(real: 5)
-    }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-  }
-
-  func testVjpSubtractingImaginary() {
-    let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
-      return x.subtracting(imaginary: 5)
-    }
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
-    XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
-    XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
-  }
-}
\ No newline at end of file
+    
+    func testIsInfinite() {
+        var complex = Complex<Float>(real: 999, imaginary: 0)
+        XCTAssertFalse(complex.isInfinite)
+        
+        complex = Complex(real: 1.0 / 0.0, imaginary: 1)
+        XCTAssertTrue(complex.isInfinite)
+        
+        complex = Complex(real: 1.0 / 0.0, imaginary: 1.0 / 0.0)
+        XCTAssertTrue(complex.isInfinite)
+    }
+    
+    func testIsNaN() {
+        var complex = Complex<Float>(real: 999, imaginary: 0)
+        XCTAssertFalse(complex.isNaN)
+        
+        complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 1)
+        XCTAssertTrue(complex.isNaN)
+        
+        complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
+        XCTAssertTrue(complex.isNaN)
+    }
+    
+    func testIsZero() {
+        var complex = Complex<Float>(real: 999, imaginary: 0)
+        XCTAssertFalse(complex.isZero)
+        
+        complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0)
+        XCTAssertFalse(complex.isZero)
+        
+        complex = Complex(real: 0.0 * 1.0 / 0.0, imaginary: 0.0 * 1.0 / 0.0)
+        XCTAssertFalse(complex.isZero)
+        
+        complex = Complex(real: 0, imaginary: 0)
+        XCTAssertTrue(complex.isZero)
+    }
+    
+    func testEquals() {
+        var complexA = Complex<Float>(real: 999, imaginary: 0)
+        let complexB = Complex<Float>(real: 999, imaginary: 0)
+        XCTAssertEqual(complexA, complexB)
+        
+        complexA = Complex(real: 5, imaginary: 0)
+        XCTAssertNotEqual(complexA, complexB)
+    }
+    
+    func testPlus() {
+        let input = Complex<Float>(real: 5, imaginary: 1)
+        let expected = Complex<Float>(real: 10, imaginary: 2)
+        XCTAssertEqual(expected, input + input)
+    }
+    
+    func testMinus() {
+        let inputA = Complex<Float>(real: 6, imaginary: 2)
+        let inputB = Complex<Float>(real: 5, imaginary: 1)
+        let expected = Complex<Float>(real: 1, imaginary: 1)
+        XCTAssertEqual(expected, inputA - inputB)
+    }
+    
+    func testTimes() {
+        let inputA = Complex<Float>(real: 6, imaginary: 2)
+        let inputB = Complex<Float>(real: 5, imaginary: 1)
+        let expected = Complex<Float>(real: 28, imaginary: 16)
+        XCTAssertEqual(expected, inputA * inputB)
+    }
+    
+    func testNegate() {
+        var input = Complex<Float>(real: 6, imaginary: 2)
+        let negated = Complex<Float>(real: -6, imaginary: -2)
+        XCTAssertEqual(-input, negated)
+        input.negate()
+        XCTAssertEqual(input, negated)
+    }
+    
+    func testDivide() {
+        let inputA = Complex<Float>(real: 20, imaginary: -4)
+        let inputB = Complex<Float>(real: 3, imaginary: 2)
+        let expected = Complex<Float>(real: 4, imaginary: -4)
+        XCTAssertEqual(expected, inputA / inputB)
+    }
+    
+    func testComplexConjugate() {
+        var input = Complex<Float>(real: 2, imaginary: -4)
+        var expected = Complex<Float>(real: 2, imaginary: 4)
+        XCTAssertEqual(expected, input.complexConjugate())
+        
+        input = Complex<Float>(real: -2, imaginary: -4)
+        expected = Complex<Float>(real: -2, imaginary: 4)
+        XCTAssertEqual(expected, input.complexConjugate())
+        
+        input = Complex<Float>(real: 2, imaginary: 4)
+        expected = Complex<Float>(real: 2, imaginary: -4)
+        XCTAssertEqual(expected, input.complexConjugate())
+    }
+    
+    func testAdding() {
+        var input = Complex<Float>(real: 2, imaginary: -4)
+        var expected = Complex<Float>(real: 3, imaginary: -4)
+        XCTAssertEqual(expected, input.adding(real: 1))
+        
+        input = Complex<Float>(real: 2, imaginary: -4)
+        expected = Complex<Float>(real: 2, imaginary: -3)
+        XCTAssertEqual(expected, input.adding(imaginary: 1))
+    }
+    
+    func testSubtracting() {
+        var input = Complex<Float>(real: 2, imaginary: -4)
+        var expected = Complex<Float>(real: 1, imaginary: -4)
+        XCTAssertEqual(expected, input.subtracting(real: 1))
+        
+        input = Complex<Float>(real: 2, imaginary: -4)
+        expected = Complex<Float>(real: 2, imaginary: -5)
+        XCTAssertEqual(expected, input.subtracting(imaginary: 1))
+    }
+    
+    func testVjpAdd() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+            return x + Complex<Float>(real: 5, imaginary: 6)
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    }
+    
+    func testVjpSubtract() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+            return Complex<Float>(real: 5, imaginary: 6) - x
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+    }
+    
+    func testVjpMultiply() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+            return x * x
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 4, imaginary: 6))
+        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: -6, imaginary: 4))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -2, imaginary: 10))
+    }
+    
+    func testVjpDivide() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+            return x / Complex<Float>(real: 2, imaginary: 2)
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 0.25, imaginary: -0.25))
+        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0.25, imaginary: 0.25))
+    }
+    
+    func testVjpNegate() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+            return -x
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
+        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+    }
+    
+    func testVjpComplexConjugate() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+            return x.complexConjugate()
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
+        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+    }
+    
+    func testVjpAddingReal() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+            return x.adding(real: 5)
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    }
+    
+    func testVjpAddingImaginary() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+            return x.adding(imaginary: 5)
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    }
+    
+    func testVjpSubtractingReal() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+            return x.subtracting(real: 5)
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    }
+    
+    func testVjpSubtractingImaginary() {
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+            return x.subtracting(imaginary: 5)
+        }
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
+        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: 1))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+    }
+    
+//    func testJvpDotProduct() {
+//        struct ComplexVector : Differentiable & AdditiveArithmetic {
+//            @differentiable
+//            var w: Complex<Float>
+//            var x: Complex<Float>
+//            var y: Complex<Float>
+//            var z: Complex<Float>
+//
+//            init(w: Complex<Float>, x: Complex<Float>, y: Complex<Float>, z: Complex<Float>) {
+//                self.w = w
+//                self.x = x
+//                self.y = y
+//                self.z = z
+//            }
+//        }
+//
+//        func dot(lhs: ComplexVector, rhs: ComplexVector) -> Complex<Float> {
+//            var result: Complex<Float> = Complex(real: 0, imaginary: 0)
+//            result = result + lhs.w.complexConjugate() * rhs.w
+//            result = result + lhs.x.complexConjugate() * rhs.x
+//            result = result + lhs.y.complexConjugate() * rhs.y
+//            result = result + lhs.z.complexConjugate() * rhs.z
+//            return result
+//        }
+//
+//        let atVector = ComplexVector(
+//            w: Complex(real: 1, imaginary: 1),
+//            x: Complex(real: 1, imaginary: -1),
+//            y: Complex(real: -1, imaginary: 1),
+//            z: Complex(real: -1, imaginary: -1))
+//        let rhsVector = ComplexVector(
+//            w: Complex(real: 3, imaginary: -4),
+//            x: Complex(real: 6, imaginary: -2),
+//            y: Complex(real: 1, imaginary: 2),
+//            z: Complex(real: 4, imaginary: 3))
+//        let expectedVector = ComplexVector(
+//            w: Complex(real: 7, imaginary: 1),
+//            x: Complex(real: 8, imaginary: -4),
+//            y: Complex(real: -1, imaginary: -3),
+//            z: Complex(real: 1, imaginary: -7))
+//
+//        let (result, pbComplex) = valueWithPullback(at: atVector) { x in
+//            return dot(lhs: x, rhs: rhsVector)
+//        }
+//
+//        XCTAssertEqual(Complex(real: 1, imaginary: -5), result)
+//        XCTAssertEqual(expectedVector, pbComplex(Complex(real: 1, imaginary: 1)))
+//    }
+}

From 46e68c9fa3442a10869e89004da6daf5f12092e5 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Fri, 24 May 2019 10:27:58 -0700
Subject: [PATCH 16/28] Fix package.swift

---
 Package.swift                                 | 14 +--
 Sources/DeepLearning/Core/Dataset.swift       |  4 -
 .../{ => Experimental}/Complex.swift          |  0
 .../third_party/{ => Experimental}/LICENSE    |  0
 .../ComplexTests.swift                        | 98 +++++++++----------
 5 files changed, 56 insertions(+), 60 deletions(-)
 rename Sources/third_party/{ => Experimental}/Complex.swift (100%)
 rename Sources/third_party/{ => Experimental}/LICENSE (100%)
 rename Tests/{DeepLearningTests => ExperimentalTests}/ComplexTests.swift (83%)

diff --git a/Package.swift b/Package.swift
index 9d1531ef7..4668914e9 100644
--- a/Package.swift
+++ b/Package.swift
@@ -22,10 +22,7 @@ let package = Package(
     products: [
         .library(
             name: "DeepLearning",
-            targets: ["DeepLearning"]),
-        .library(
-            name: "third_party",
-            targets: ["third_party"])
+            targets: ["DeepLearning"])
     ],
     dependencies: [],
     targets: [
@@ -33,11 +30,14 @@ let package = Package(
             name: "DeepLearning",
             dependencies: []),
         .target(
-            name: "third_party",
+            name: "Experimental",
             dependencies: [],
-            path: "Sources/third_party"),
+            path: "Sources/third_party/Experimental"),
+        .testTarget(
+            name: "ExperimentalTests",
+            dependencies: ["Experimental"]),
         .testTarget(
             name: "DeepLearningTests",
-            dependencies: ["DeepLearning", "third_party"]),
+            dependencies: ["DeepLearning"]),
     ]
 )
diff --git a/Sources/DeepLearning/Core/Dataset.swift b/Sources/DeepLearning/Core/Dataset.swift
index cd5a4c1df..48e14764a 100644
--- a/Sources/DeepLearning/Core/Dataset.swift
+++ b/Sources/DeepLearning/Core/Dataset.swift
@@ -1,7 +1,3 @@
-#if !COMPILING_TENSORFLOW_MODULE
-import TensorFlow
-#endif
-
 //===-- Dataset.swift -----------------------------------------*- swift -*-===//
 //
 // This source file is part of the Swift.org open source project
diff --git a/Sources/third_party/Complex.swift b/Sources/third_party/Experimental/Complex.swift
similarity index 100%
rename from Sources/third_party/Complex.swift
rename to Sources/third_party/Experimental/Complex.swift
diff --git a/Sources/third_party/LICENSE b/Sources/third_party/Experimental/LICENSE
similarity index 100%
rename from Sources/third_party/LICENSE
rename to Sources/third_party/Experimental/LICENSE
diff --git a/Tests/DeepLearningTests/ComplexTests.swift b/Tests/ExperimentalTests/ComplexTests.swift
similarity index 83%
rename from Tests/DeepLearningTests/ComplexTests.swift
rename to Tests/ExperimentalTests/ComplexTests.swift
index 13e584498..20acac75b 100644
--- a/Tests/DeepLearningTests/ComplexTests.swift
+++ b/Tests/ExperimentalTests/ComplexTests.swift
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 import XCTest
-@testable import third_party
+@testable import Experimental
 
 final class ComplexTests: XCTestCase {
     func testInitializer() {
@@ -237,52 +237,52 @@ final class ComplexTests: XCTestCase {
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
     }
     
-//    func testJvpDotProduct() {
-//        struct ComplexVector : Differentiable & AdditiveArithmetic {
-//            @differentiable
-//            var w: Complex<Float>
-//            var x: Complex<Float>
-//            var y: Complex<Float>
-//            var z: Complex<Float>
-//
-//            init(w: Complex<Float>, x: Complex<Float>, y: Complex<Float>, z: Complex<Float>) {
-//                self.w = w
-//                self.x = x
-//                self.y = y
-//                self.z = z
-//            }
-//        }
-//
-//        func dot(lhs: ComplexVector, rhs: ComplexVector) -> Complex<Float> {
-//            var result: Complex<Float> = Complex(real: 0, imaginary: 0)
-//            result = result + lhs.w.complexConjugate() * rhs.w
-//            result = result + lhs.x.complexConjugate() * rhs.x
-//            result = result + lhs.y.complexConjugate() * rhs.y
-//            result = result + lhs.z.complexConjugate() * rhs.z
-//            return result
-//        }
-//
-//        let atVector = ComplexVector(
-//            w: Complex(real: 1, imaginary: 1),
-//            x: Complex(real: 1, imaginary: -1),
-//            y: Complex(real: -1, imaginary: 1),
-//            z: Complex(real: -1, imaginary: -1))
-//        let rhsVector = ComplexVector(
-//            w: Complex(real: 3, imaginary: -4),
-//            x: Complex(real: 6, imaginary: -2),
-//            y: Complex(real: 1, imaginary: 2),
-//            z: Complex(real: 4, imaginary: 3))
-//        let expectedVector = ComplexVector(
-//            w: Complex(real: 7, imaginary: 1),
-//            x: Complex(real: 8, imaginary: -4),
-//            y: Complex(real: -1, imaginary: -3),
-//            z: Complex(real: 1, imaginary: -7))
-//
-//        let (result, pbComplex) = valueWithPullback(at: atVector) { x in
-//            return dot(lhs: x, rhs: rhsVector)
-//        }
-//
-//        XCTAssertEqual(Complex(real: 1, imaginary: -5), result)
-//        XCTAssertEqual(expectedVector, pbComplex(Complex(real: 1, imaginary: 1)))
-//    }
+    func testJvpDotProduct() {
+        struct ComplexVector : Differentiable & AdditiveArithmetic {
+            @differentiable
+            var w: Complex<Float>
+            var x: Complex<Float>
+            var y: Complex<Float>
+            var z: Complex<Float>
+
+            init(w: Complex<Float>, x: Complex<Float>, y: Complex<Float>, z: Complex<Float>) {
+                self.w = w
+                self.x = x
+                self.y = y
+                self.z = z
+            }
+        }
+
+        func dot(lhs: ComplexVector, rhs: ComplexVector) -> Complex<Float> {
+            var result: Complex<Float> = Complex(real: 0, imaginary: 0)
+            result = result + lhs.w.complexConjugate() * rhs.w
+            result = result + lhs.x.complexConjugate() * rhs.x
+            result = result + lhs.y.complexConjugate() * rhs.y
+            result = result + lhs.z.complexConjugate() * rhs.z
+            return result
+        }
+
+        let atVector = ComplexVector(
+            w: Complex(real: 1, imaginary: 1),
+            x: Complex(real: 1, imaginary: -1),
+            y: Complex(real: -1, imaginary: 1),
+            z: Complex(real: -1, imaginary: -1))
+        let rhsVector = ComplexVector(
+            w: Complex(real: 3, imaginary: -4),
+            x: Complex(real: 6, imaginary: -2),
+            y: Complex(real: 1, imaginary: 2),
+            z: Complex(real: 4, imaginary: 3))
+        let expectedVector = ComplexVector(
+            w: Complex(real: 7, imaginary: 1),
+            x: Complex(real: 8, imaginary: -4),
+            y: Complex(real: -1, imaginary: -3),
+            z: Complex(real: 1, imaginary: -7))
+
+        let (result, pbComplex) = valueWithPullback(at: atVector) { x in
+            return dot(lhs: x, rhs: rhsVector)
+        }
+
+        XCTAssertEqual(Complex(real: 1, imaginary: -5), result)
+        XCTAssertEqual(expectedVector, pbComplex(Complex(real: 1, imaginary: 1)))
+    }
 }

From dc4930c84ccbdbbeb47e7c443621ee3ee81f61fa Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Fri, 24 May 2019 13:40:36 -0700
Subject: [PATCH 17/28] Fix typo in conjugate test.

---
 Tests/ExperimentalTests/ComplexTests.swift | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Tests/ExperimentalTests/ComplexTests.swift b/Tests/ExperimentalTests/ComplexTests.swift
index 20acac75b..29cb653a1 100644
--- a/Tests/ExperimentalTests/ComplexTests.swift
+++ b/Tests/ExperimentalTests/ComplexTests.swift
@@ -196,9 +196,9 @@ final class ComplexTests: XCTestCase {
         let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.complexConjugate()
         }
-        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
         XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0, imaginary: -1))
-        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
+        XCTAssertEqual(pb(Complex(real: -1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
     }
     
     func testVjpAddingReal() {

From e4b7106a6aef13eec4994641fd8bb2ebc0fd078a Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Fri, 24 May 2019 16:25:07 -0700
Subject: [PATCH 18/28] Fix colon spacing and Linux testing static var.

---
 .../third_party/Experimental/Complex.swift    | 46 +++++++++----------
 Tests/ExperimentalTests/ComplexTests.swift    | 29 ++++++++++++
 2 files changed, 52 insertions(+), 23 deletions(-)

diff --git a/Sources/third_party/Experimental/Complex.swift b/Sources/third_party/Experimental/Complex.swift
index db9634509..6c61db6d1 100644
--- a/Sources/third_party/Experimental/Complex.swift
+++ b/Sources/third_party/Experimental/Complex.swift
@@ -1,18 +1,18 @@
 // A major part of the Complex number implementation comes from
 // [@xwu](https://github.com/xwu)'s project, [NumericAnnex](https://github.com/xwu/NumericAnnex).
 
-struct Complex<T : FloatingPoint> {
+struct Complex<T: FloatingPoint> {
     var real: T
     var imaginary: T
 
-    @differentiable(vjp: _vjpInit where T : Differentiable, T.TangentVector == T)
+    @differentiable(vjp: _vjpInit where T: Differentiable, T.TangentVector == T)
     init(real: T = 0, imaginary: T = 0) {
         self.real = real
         self.imaginary = imaginary
     }
 }
 
-extension Complex : Differentiable where T : Differentiable {
+extension Complex: Differentiable where T: Differentiable {
     typealias TangentVector = Complex
     typealias AllDifferentiableVariables = Complex
 }
@@ -40,14 +40,14 @@ extension Complex {
     }
 }
 
-extension Complex : ExpressibleByIntegerLiteral {
+extension Complex: ExpressibleByIntegerLiteral {
     init(integerLiteral value: Int) {
         self.real = T(value)
         self.imaginary = 0
     }
 }
 
-extension Complex : CustomStringConvertible {
+extension Complex: CustomStringConvertible {
     var description: String {
         return real.isNaN && real.sign == .minus
             ? imaginary.sign == .minus
@@ -59,14 +59,14 @@ extension Complex : CustomStringConvertible {
     }
 }
 
-extension Complex : Equatable {
+extension Complex: Equatable {
     static func == (lhs: Complex, rhs: Complex) -> Bool {
         return lhs.real == rhs.real && lhs.imaginary == rhs.imaginary
     }
 }
 
-extension Complex : AdditiveArithmetic {
-    @differentiable(vjp: _vjpAdd(lhs:rhs:) where T : Differentiable)
+extension Complex: AdditiveArithmetic {
+    @differentiable(vjp: _vjpAdd(lhs:rhs:) where T: Differentiable)
     static func + (lhs: Complex, rhs: Complex) -> Complex {
         var temp = lhs
         temp += rhs
@@ -78,7 +78,7 @@ extension Complex : AdditiveArithmetic {
         lhs.imaginary += rhs.imaginary
     }
 
-    @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T : Differentiable)
+    @differentiable(vjp: _vjpSubtract(lhs:rhs:) where T: Differentiable)
     static func - (lhs: Complex, rhs: Complex) -> Complex {
         var temp = lhs
         temp -= rhs
@@ -91,14 +91,14 @@ extension Complex : AdditiveArithmetic {
     }
 }
 
-extension Complex : Numeric {
-    init?<U>(exactly source: U) where U : BinaryInteger {
+extension Complex: Numeric {
+    init?<U>(exactly source: U) where U: BinaryInteger {
         guard let t = T(exactly: source) else { return nil }
         self.real = t
         self.imaginary = 0
     }
 
-    @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T : Differentiable)
+    @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T: Differentiable)
     static func * (lhs: Complex, rhs: Complex) -> Complex {
         var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
         let ac = a * c, bd = b * d, ad = a * d, bc = b * c
@@ -155,8 +155,8 @@ extension Complex : Numeric {
     }
 }
 
-extension Complex : SignedNumeric {
-    @differentiable(vjp: _vjpNegate where T : Differentiable)
+extension Complex: SignedNumeric {
+    @differentiable(vjp: _vjpNegate where T: Differentiable)
     static prefix func - (operand: Complex) -> Complex {
         return Complex(real: -operand.real, imaginary: -operand.imaginary)
     }
@@ -168,7 +168,7 @@ extension Complex : SignedNumeric {
 }
 
 extension Complex {
-    @differentiable(vjp: _vjpDivide(lhs:rhs:) where T : Differentiable)
+    @differentiable(vjp: _vjpDivide(lhs:rhs:) where T: Differentiable)
     static func / (lhs: Complex, rhs: Complex) -> Complex {
         var a = lhs.real, b = lhs.imaginary, c = rhs.real, d = rhs.imaginary
         var x: T
@@ -210,7 +210,7 @@ extension Complex {
 }
 
 extension Complex {
-    @differentiable(vjp: _vjpComplexConjugate where T : Differentiable)
+    @differentiable(vjp: _vjpComplexConjugate where T: Differentiable)
     func complexConjugate() -> Complex {
         return Complex(real: real, imaginary: -imaginary)
     }
@@ -222,7 +222,7 @@ func abs<T>(_ z: Complex<T>) -> Complex<T> {
 
 extension Complex {
     @differentiable(vjp: _vjpAdding(real:)
-        where T : Differentiable,
+        where T: Differentiable,
               T.TangentVector == T)
     func adding(real: T) -> Complex {
         var c = self
@@ -231,7 +231,7 @@ extension Complex {
     }
 
     @differentiable(vjp: _vjpSubtracting(real:)
-        where T : Differentiable,
+        where T: Differentiable,
               T.TangentVector == T)
     func subtracting(real: T) -> Complex {
         var c = self
@@ -240,7 +240,7 @@ extension Complex {
     }
 
     @differentiable(vjp: _vjpAdding(imaginary:)
-        where T : Differentiable,
+        where T: Differentiable,
               T.TangentVector == T)
     func adding(imaginary: T) -> Complex {
         var c = self
@@ -249,7 +249,7 @@ extension Complex {
     }
 
     @differentiable(vjp: _vjpSubtracting(imaginary:)
-        where T : Differentiable,
+        where T: Differentiable,
               T.TangentVector == T)
     func subtracting(imaginary: T) -> Complex {
         var c = self
@@ -258,14 +258,14 @@ extension Complex {
     }
 }
 
-extension Complex where T : Differentiable, T.TangentVector == T {
+extension Complex where T: Differentiable, T.TangentVector == T {
     @usableFromInline
     static func _vjpInit(real: T, imaginary: T) -> (Complex, (Complex) -> (T, T)) {
         return (Complex(real: real, imaginary: imaginary), { ($0.real, $0.imaginary) })
     }
 }
 
-extension Complex where T : Differentiable {
+extension Complex where T: Differentiable {
     @usableFromInline
     static func _vjpAdd(lhs: Complex, rhs: Complex)
     -> (Complex, (Complex) -> (Complex, Complex)) {
@@ -302,7 +302,7 @@ extension Complex where T : Differentiable {
     }
 }
 
-extension Complex where T : Differentiable, T.TangentVector == T {
+extension Complex where T: Differentiable, T.TangentVector == T {
     @usableFromInline
     func _vjpAdding(real: T) -> (Complex, (Complex) -> (Complex, T)) {
         return (self.adding(real: real), { ($0, $0.real) })
diff --git a/Tests/ExperimentalTests/ComplexTests.swift b/Tests/ExperimentalTests/ComplexTests.swift
index 29cb653a1..3f95ac15e 100644
--- a/Tests/ExperimentalTests/ComplexTests.swift
+++ b/Tests/ExperimentalTests/ComplexTests.swift
@@ -285,4 +285,33 @@ final class ComplexTests: XCTestCase {
         XCTAssertEqual(Complex(real: 1, imaginary: -5), result)
         XCTAssertEqual(expectedVector, pbComplex(Complex(real: 1, imaginary: 1)))
     }
+    
+    static var allTests = [
+        ("testInitializer", testInitializer),
+        ("testStaticImaginary", testStaticImaginary),
+        ("testIsFinite", testIsFinite),
+        ("testIsInfinite", testIsInfinite),
+        ("testIsNaN", testIsNaN),
+        ("testIsZero", testIsZero),
+        ("testEquals", testEquals),
+        ("testPlus", testPlus),
+        ("testMinus", testMinus),
+        ("testTimes", testTimes),
+        ("testNegate", testNegate),
+        ("testDivide", testDivide),
+        ("testComplexConjugate", testComplexConjugate),
+        ("testAdding", testAdding),
+        ("testSubtracting", testSubtracting),
+        ("testVjpAdd", testVjpAdd),
+        ("testVjpSubtract", testVjpSubtract),
+        ("testVjpMultiply", testVjpMultiply),
+        ("testVjpDivide", testVjpDivide),
+        ("testVjpNegate", testVjpNegate),
+        ("testVjpComplexConjugate", testVjpComplexConjugate),
+        ("testVjpAddingReal", testVjpAddingReal),
+        ("testVjpAddingImaginary", testVjpAddingImaginary),
+        ("testVjpSubtractingReal", testVjpSubtractingReal),
+        ("testVjpSubtractingImaginary", testVjpSubtractingImaginary),
+        ("testJvpDotProduct", testJvpDotProduct)
+    ]
 }

From ae53238ec0facb60f8542016ad187f67bee57f5a Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Fri, 24 May 2019 16:39:49 -0700
Subject: [PATCH 19/28] Format and License fix.

---
 .../third_party/Experimental/Complex.swift    | 95 ++++++++++++-------
 1 file changed, 63 insertions(+), 32 deletions(-)

diff --git a/Sources/third_party/Experimental/Complex.swift b/Sources/third_party/Experimental/Complex.swift
index 6c61db6d1..1eb113364 100644
--- a/Sources/third_party/Experimental/Complex.swift
+++ b/Sources/third_party/Experimental/Complex.swift
@@ -1,5 +1,55 @@
-// A major part of the Complex number implementation comes from
-// [@xwu](https://github.com/xwu)'s project, [NumericAnnex](https://github.com/xwu/NumericAnnex).
+//
+//  Complex.swift
+//  NumericAnnex
+//
+//  Created by Xiaodi Wu on 3/25/17.
+//
+//  Note
+//  ====
+//
+//  For maximum consistency with corresponding functions in C/C++, checks for
+//  special values in `naturalExponential()`, `squareRoot()`, trigonometric
+//  functions, and hyperbolic functions are adapted from libc++.
+//
+//  Code in libc++ is dual-licensed under the MIT and UIUC/NCSA licenses.
+//  Copyright © 2009-2017 contributors to the LLVM/libc++ project, Google LLC.
+/// A type to represent a complex value in Cartesian form.
+///
+/// - Note: `Complex64` is a type alias for `Complex<Float>` and `Complex128` is
+///   a type alias for `Complex<Double>`.
+///
+/// Create new instances of `Complex<T>` using integer or floating-point
+/// literals and the imaginary unit `Complex<T>.i`. For example:
+///
+/// ```swift
+/// let x = 2 + 4 * .i // `x` is of type `Complex<Double>`
+/// let y = 3.5 + 7 * .i // `y` is of type `Complex<Double>`
+///
+/// let z: Complex64 = .e + .pi * .i // `z` is of type `Complex<Float>`
+/// ```
+///
+/// Additional Considerations
+/// -------------------------
+///
+/// Floating-point types have special values that represent infinity or NaN
+/// ("not a number"). Complex functions in different languages may return
+/// different results when working with special values.
+///
+/// Many complex functions have [branch cuts][dfn], which are curves in the
+/// complex plane across which a function is discontinuous. Different languages
+/// may adopt different branch cut structures for the same complex function.
+///
+/// Implementations in `Complex<T>` adhere to the [C standard][std] (Annex G) as
+/// closely as possible with respect to special values and branch cuts.
+///
+/// To users unfamiliar with complex functions, the principal value returned by
+/// some complex functions may be unexpected. For example,
+/// `Double.cbrt(-8) == -2`, which is the __real root__, while
+/// `Complex.cbrt(-8) == 2 * Complex.exp(.i * .pi / 3)`, which is the
+/// __principal root__.
+///
+/// [dfn]: http://mathworld.wolfram.com/BranchCut.html
+/// [std]: http://www.open-std.org/JTC1/SC22/WG14/www/standards.html#9899
 
 struct Complex<T: FloatingPoint> {
     var real: T
@@ -131,8 +181,8 @@ extension Complex: Numeric {
             }
             if recalculate {
                 return Complex(
-                real: .infinity * (a * c - b * d),
-                imaginary: .infinity * (a * d + b * c)
+                    real: .infinity * (a * c - b * d),
+                    imaginary: .infinity * (a * d + b * c)
                 )
             }
         }
@@ -221,36 +271,28 @@ func abs<T>(_ z: Complex<T>) -> Complex<T> {
 }
 
 extension Complex {
-    @differentiable(vjp: _vjpAdding(real:)
-        where T: Differentiable,
-              T.TangentVector == T)
+    @differentiable(vjp: _vjpAdding(real:) where T: Differentiable, T.TangentVector == T)
     func adding(real: T) -> Complex {
         var c = self
         c.real += real
         return c
     }
 
-    @differentiable(vjp: _vjpSubtracting(real:)
-        where T: Differentiable,
-              T.TangentVector == T)
+    @differentiable(vjp: _vjpSubtracting(real:) where T: Differentiable, T.TangentVector == T)
     func subtracting(real: T) -> Complex {
         var c = self
         c.real -= real
         return c
     }
 
-    @differentiable(vjp: _vjpAdding(imaginary:)
-        where T: Differentiable,
-              T.TangentVector == T)
+    @differentiable(vjp: _vjpAdding(imaginary:) where T: Differentiable, T.TangentVector == T)
     func adding(imaginary: T) -> Complex {
         var c = self
         c.imaginary += imaginary
         return c
     }
 
-    @differentiable(vjp: _vjpSubtracting(imaginary:)
-        where T: Differentiable,
-              T.TangentVector == T)
+    @differentiable(vjp: _vjpSubtracting(imaginary:) where T: Differentiable, T.TangentVector == T)
     func subtracting(imaginary: T) -> Complex {
         var c = self
         c.imaginary -= imaginary
@@ -259,66 +301,55 @@ extension Complex {
 }
 
 extension Complex where T: Differentiable, T.TangentVector == T {
-    @usableFromInline
     static func _vjpInit(real: T, imaginary: T) -> (Complex, (Complex) -> (T, T)) {
         return (Complex(real: real, imaginary: imaginary), { ($0.real, $0.imaginary) })
     }
 }
 
 extension Complex where T: Differentiable {
-    @usableFromInline
     static func _vjpAdd(lhs: Complex, rhs: Complex)
-    -> (Complex, (Complex) -> (Complex, Complex)) {
+        -> (Complex, (Complex) -> (Complex, Complex)) {
         return (lhs + rhs, { v in (v, v) })
     }
 
-    @usableFromInline
     static func _vjpSubtract(lhs: Complex, rhs: Complex)
-    -> (Complex, (Complex) -> (Complex, Complex)) {
+        -> (Complex, (Complex) -> (Complex, Complex)) {
         return (lhs - rhs, { v in (v, -v) })
     }
 
-    @usableFromInline
     static func _vjpMultiply(lhs: Complex, rhs: Complex)
-    -> (Complex, (Complex) -> (Complex, Complex)) {
+        -> (Complex, (Complex) -> (Complex, Complex)) {
         return (lhs * rhs, { v in (rhs * v, lhs * v) })
     }
 
-    @usableFromInline
     static func _vjpDivide(lhs: Complex, rhs: Complex)
-    -> (Complex, (Complex) -> (Complex, Complex)) {
+        -> (Complex, (Complex) -> (Complex, Complex)) {
         return (lhs / rhs, { v in (v / rhs, -lhs / (rhs * rhs) * v) })
     }
 
-    @usableFromInline
     static func _vjpNegate(operand: Complex)
-    -> (Complex, (Complex) -> Complex) {
+        -> (Complex, (Complex) -> Complex) {
         return (-operand, { -$0 })
     }
 
-    @usableFromInline
     func _vjpComplexConjugate() -> (Complex, (Complex) -> Complex) {
         return (complexConjugate(), { v in v.complexConjugate() })
     }
 }
 
 extension Complex where T: Differentiable, T.TangentVector == T {
-    @usableFromInline
     func _vjpAdding(real: T) -> (Complex, (Complex) -> (Complex, T)) {
         return (self.adding(real: real), { ($0, $0.real) })
     }
 
-    @usableFromInline
     func _vjpSubtracting(real: T) -> (Complex, (Complex) -> (Complex, T)) {
         return (self.subtracting(real: real), { ($0, -$0.real) })
     }
 
-    @usableFromInline
     func _vjpAdding(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
         return (self.adding(real: real), { ($0, $0.imaginary) })
     }
 
-    @usableFromInline
     func _vjpSubtracting(imaginary: T) -> (Complex, (Complex) -> (Complex, T)) {
         return (self.subtracting(real: real), { ($0, -$0.imaginary) })
     }

From 09c53d0ec9275e97042ba006bb48a1e7764cafcd Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Fri, 24 May 2019 17:51:04 -0700
Subject: [PATCH 20/28] Spacing and license update.

---
 .../third_party/Experimental/Complex.swift    | 17 ++++++---
 Tests/ExperimentalTests/ComplexTests.swift    | 36 ++++++++++++-------
 2 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/Sources/third_party/Experimental/Complex.swift b/Sources/third_party/Experimental/Complex.swift
index 1eb113364..5520faa59 100644
--- a/Sources/third_party/Experimental/Complex.swift
+++ b/Sources/third_party/Experimental/Complex.swift
@@ -1,8 +1,16 @@
+// Copyright 2017-2019 Xiaodi Wu and The TensorFlow Authors. All Rights Reserved.
 //
-//  Complex.swift
-//  NumericAnnex
+// Licensed under the MIT License (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
 //
-//  Created by Xiaodi Wu on 3/25/17.
+//     https://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 //
 //  Note
 //  ====
@@ -12,7 +20,7 @@
 //  functions, and hyperbolic functions are adapted from libc++.
 //
 //  Code in libc++ is dual-licensed under the MIT and UIUC/NCSA licenses.
-//  Copyright © 2009-2017 contributors to the LLVM/libc++ project, Google LLC.
+//  Copyright © 2009-2017 contributors to the LLVM/libc++ project.
 /// A type to represent a complex value in Cartesian form.
 ///
 /// - Note: `Complex64` is a type alias for `Complex<Float>` and `Complex128` is
@@ -253,7 +261,6 @@ extension Complex {
         return Complex(real: x, imaginary: y)
     }
 
-
     static func /= (lhs: inout Complex, rhs: Complex) {
         lhs = lhs / rhs
     }
diff --git a/Tests/ExperimentalTests/ComplexTests.swift b/Tests/ExperimentalTests/ComplexTests.swift
index 3f95ac15e..aa72c5c97 100644
--- a/Tests/ExperimentalTests/ComplexTests.swift
+++ b/Tests/ExperimentalTests/ComplexTests.swift
@@ -153,21 +153,24 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpAdd() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+            Complex<Float>(real: 2, imaginary: 3)) { x in
             return x + Complex<Float>(real: 5, imaginary: 6)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
     }
     
     func testVjpSubtract() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+            Complex<Float>(real: 2, imaginary: 3)) { x in
             return Complex<Float>(real: 5, imaginary: 6) - x
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
     }
     
     func testVjpMultiply() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+            Complex<Float>(real: 2, imaginary: 3)) { x in
             return x * x
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 4, imaginary: 6))
@@ -176,15 +179,19 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpDivide() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+            Complex<Float>(real: 20, imaginary: -4)) { x in
             return x / Complex<Float>(real: 2, imaginary: 2)
         }
-        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 0.25, imaginary: -0.25))
-        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)), Complex<Float>(real: 0.25, imaginary: 0.25))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)),
+                       Complex<Float>(real: 0.25, imaginary: -0.25))
+        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)),
+                       Complex<Float>(real: 0.25, imaginary: 0.25))
     }
     
     func testVjpNegate() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+            Complex<Float>(real: 20, imaginary: -4)) { x in
             return -x
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
@@ -193,7 +200,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpComplexConjugate() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+            Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.complexConjugate()
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
@@ -202,7 +210,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpAddingReal() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+            Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.adding(real: 5)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
@@ -211,7 +220,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpAddingImaginary() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+            Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.adding(imaginary: 5)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
@@ -220,7 +230,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpSubtractingReal() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+            Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.subtracting(real: 5)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
@@ -229,7 +240,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpSubtractingImaginary() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
+        Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.subtracting(imaginary: 5)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))

From 1856a2f9841ad616f888d78fa3f2baa417a3b8dc Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Fri, 24 May 2019 18:18:29 -0700
Subject: [PATCH 21/28] Fix line wrapping.

---
 .../third_party/Experimental/Complex.swift    |  3 +-
 Tests/ExperimentalTests/ComplexTests.swift    | 50 ++++++++++---------
 2 files changed, 27 insertions(+), 26 deletions(-)

diff --git a/Sources/third_party/Experimental/Complex.swift b/Sources/third_party/Experimental/Complex.swift
index 5520faa59..2b4afa832 100644
--- a/Sources/third_party/Experimental/Complex.swift
+++ b/Sources/third_party/Experimental/Complex.swift
@@ -89,8 +89,7 @@ extension Complex {
     }
 
     var isNaN: Bool {
-        return (real.isNaN && !imaginary.isInfinite) ||
-        (imaginary.isNaN && !real.isInfinite)
+        return (real.isNaN && !imaginary.isInfinite) || (imaginary.isNaN && !real.isInfinite)
     }
 
     var isZero: Bool {
diff --git a/Tests/ExperimentalTests/ComplexTests.swift b/Tests/ExperimentalTests/ComplexTests.swift
index aa72c5c97..a6b197495 100644
--- a/Tests/ExperimentalTests/ComplexTests.swift
+++ b/Tests/ExperimentalTests/ComplexTests.swift
@@ -153,24 +153,24 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpAdd() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-            Complex<Float>(real: 2, imaginary: 3)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
             return x + Complex<Float>(real: 5, imaginary: 6)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
     }
     
     func testVjpSubtract() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-            Complex<Float>(real: 2, imaginary: 3)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
             return Complex<Float>(real: 5, imaginary: 6) - x
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: -1, imaginary: -1))
     }
     
     func testVjpMultiply() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-            Complex<Float>(real: 2, imaginary: 3)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
             return x * x
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 4, imaginary: 6))
@@ -179,19 +179,21 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpDivide() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-            Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
             return x / Complex<Float>(real: 2, imaginary: 2)
         }
-        XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)),
-                       Complex<Float>(real: 0.25, imaginary: -0.25))
-        XCTAssertEqual(pb(Complex(real: 0, imaginary: 1)),
-                       Complex<Float>(real: 0.25, imaginary: 0.25))
+        XCTAssertEqual(
+            pb(Complex(real: 1, imaginary: 0)),
+            Complex<Float>(real: 0.25, imaginary: -0.25))
+        XCTAssertEqual(
+            pb(Complex(real: 0, imaginary: 1)),
+            Complex<Float>(real: 0.25, imaginary: 0.25))
     }
     
     func testVjpNegate() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-            Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
             return -x
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: -1, imaginary: 0))
@@ -200,8 +202,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpComplexConjugate() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-            Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.complexConjugate()
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
@@ -210,8 +212,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpAddingReal() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-            Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.adding(real: 5)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
@@ -220,8 +222,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpAddingImaginary() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-            Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.adding(imaginary: 5)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
@@ -230,8 +232,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpSubtractingReal() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-            Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.subtracting(real: 5)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))
@@ -240,8 +242,8 @@ final class ComplexTests: XCTestCase {
     }
     
     func testVjpSubtractingImaginary() {
-        let pb: (Complex<Float>) -> Complex<Float> = pullback(at:
-        Complex<Float>(real: 20, imaginary: -4)) { x in
+        let pb: (Complex<Float>) -> Complex<Float> =
+            pullback(at: Complex<Float>(real: 20, imaginary: -4)) { x in
             return x.subtracting(imaginary: 5)
         }
         XCTAssertEqual(pb(Complex(real: 1, imaginary: 0)), Complex<Float>(real: 1, imaginary: 0))

From 90bff2926f3d2f5895a2c8524616445ed0f75962 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Mon, 27 May 2019 19:20:06 -0700
Subject: [PATCH 22/28] Clean up documentation, slight code cleanup

- TODO: see if I can clean up multipication more
---
 .../third_party/Experimental/Complex.swift    | 39 +++----------------
 1 file changed, 5 insertions(+), 34 deletions(-)

diff --git a/Sources/third_party/Experimental/Complex.swift b/Sources/third_party/Experimental/Complex.swift
index 2b4afa832..42943b097 100644
--- a/Sources/third_party/Experimental/Complex.swift
+++ b/Sources/third_party/Experimental/Complex.swift
@@ -15,25 +15,15 @@
 //  Note
 //  ====
 //
-//  For maximum consistency with corresponding functions in C/C++, checks for
-//  special values in `naturalExponential()`, `squareRoot()`, trigonometric
-//  functions, and hyperbolic functions are adapted from libc++.
-//
 //  Code in libc++ is dual-licensed under the MIT and UIUC/NCSA licenses.
 //  Copyright © 2009-2017 contributors to the LLVM/libc++ project.
 /// A type to represent a complex value in Cartesian form.
 ///
-/// - Note: `Complex64` is a type alias for `Complex<Float>` and `Complex128` is
-///   a type alias for `Complex<Double>`.
-///
 /// Create new instances of `Complex<T>` using integer or floating-point
 /// literals and the imaginary unit `Complex<T>.i`. For example:
 ///
 /// ```swift
-/// let x = 2 + 4 * .i // `x` is of type `Complex<Double>`
-/// let y = 3.5 + 7 * .i // `y` is of type `Complex<Double>`
-///
-/// let z: Complex64 = .e + .pi * .i // `z` is of type `Complex<Float>`
+/// let x: Complex<Double> = 2 + 4 * .i
 /// ```
 ///
 /// Additional Considerations
@@ -42,22 +32,6 @@
 /// Floating-point types have special values that represent infinity or NaN
 /// ("not a number"). Complex functions in different languages may return
 /// different results when working with special values.
-///
-/// Many complex functions have [branch cuts][dfn], which are curves in the
-/// complex plane across which a function is discontinuous. Different languages
-/// may adopt different branch cut structures for the same complex function.
-///
-/// Implementations in `Complex<T>` adhere to the [C standard][std] (Annex G) as
-/// closely as possible with respect to special values and branch cuts.
-///
-/// To users unfamiliar with complex functions, the principal value returned by
-/// some complex functions may be unexpected. For example,
-/// `Double.cbrt(-8) == -2`, which is the __real root__, while
-/// `Complex.cbrt(-8) == 2 * Complex.exp(.i * .pi / 3)`, which is the
-/// __principal root__.
-///
-/// [dfn]: http://mathworld.wolfram.com/BranchCut.html
-/// [std]: http://www.open-std.org/JTC1/SC22/WG14/www/standards.html#9899
 
 struct Complex<T: FloatingPoint> {
     var real: T
@@ -170,16 +144,13 @@ extension Complex: Numeric {
                 if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
                 if d.isNaN { d = T(signOf: d, magnitudeOf: 0) }
                 recalculate = true
-            }
-            if c.isInfinite || d.isInfinite {
-                if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
-                if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
+            } else if c.isInfinite || d.isInfinite {
                 c = T(signOf: c, magnitudeOf: c.isInfinite ? 1 : 0)
                 d = T(signOf: d, magnitudeOf: d.isInfinite ? 1 : 0)
+                if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
+                if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
                 recalculate = true
-            }
-            if !recalculate &&
-                (ac.isInfinite || bd.isInfinite || ad.isInfinite || bc.isInfinite) {
+            } else if ac.isInfinite || bd.isInfinite || ad.isInfinite || bc.isInfinite {
                 if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
                 if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
                 if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }

From a7d88bbf40b2ddb38662a2653ce477e8f360a7fe Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Mon, 27 May 2019 19:46:46 -0700
Subject: [PATCH 23/28] Pull more multiplication code out to helper func.

---
 .../third_party/Experimental/Complex.swift    | 34 +++++++++++--------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/Sources/third_party/Experimental/Complex.swift b/Sources/third_party/Experimental/Complex.swift
index 42943b097..9869d55b3 100644
--- a/Sources/third_party/Experimental/Complex.swift
+++ b/Sources/third_party/Experimental/Complex.swift
@@ -128,6 +128,24 @@ extension Complex: Numeric {
         self.real = t
         self.imaginary = 0
     }
+    
+    static private func handleMultiplyNaN(infiniteA: T, infiniteB: T, nanA: T, nanB: T) -> Complex {
+        var a = infiniteA
+        var b = infiniteB
+        var c = nanA
+        var d = nanB
+        
+        a = T(signOf: infiniteA, magnitudeOf: infiniteA.isInfinite ? 1 : 0)
+        b = T(signOf: infiniteB, magnitudeOf: infiniteB.isInfinite ? 1 : 0)
+        
+        if nanA.isNaN { c = T(signOf: nanA, magnitudeOf: 0) }
+        if nanB.isNaN { d = T(signOf: nanB, magnitudeOf: 0) }
+        
+        return Complex(
+            real: .infinity * (a * c - b * d),
+            imaginary: .infinity * (a * d + b * c)
+        )
+    }
 
     @differentiable(vjp: _vjpMultiply(lhs:rhs:) where T: Differentiable)
     static func * (lhs: Complex, rhs: Complex) -> Complex {
@@ -137,27 +155,15 @@ extension Complex: Numeric {
         let y = ad + bc
 
         if x.isNaN && y.isNaN {
-            var recalculate = false
             if a.isInfinite || b.isInfinite {
-                a = T(signOf: a, magnitudeOf: a.isInfinite ? 1 : 0)
-                b = T(signOf: b, magnitudeOf: b.isInfinite ? 1 : 0)
-                if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
-                if d.isNaN { d = T(signOf: d, magnitudeOf: 0) }
-                recalculate = true
+                return handleMultiplyNaN(infiniteA: a, infiniteB: b, nanA: c, nanB: d)
             } else if c.isInfinite || d.isInfinite {
-                c = T(signOf: c, magnitudeOf: c.isInfinite ? 1 : 0)
-                d = T(signOf: d, magnitudeOf: d.isInfinite ? 1 : 0)
-                if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
-                if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
-                recalculate = true
+                return handleMultiplyNaN(infiniteA: c, infiniteB: d, nanA: a, nanB: b)
             } else if ac.isInfinite || bd.isInfinite || ad.isInfinite || bc.isInfinite {
                 if a.isNaN { a = T(signOf: a, magnitudeOf: 0) }
                 if b.isNaN { b = T(signOf: b, magnitudeOf: 0) }
                 if c.isNaN { c = T(signOf: c, magnitudeOf: 0) }
                 if d.isNaN { d = T(signOf: d, magnitudeOf: 0) }
-                recalculate = true
-            }
-            if recalculate {
                 return Complex(
                     real: .infinity * (a * c - b * d),
                     imaginary: .infinity * (a * d + b * c)

From 57bca41854a96f5907f7d3d166f3debd387b465b Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Wed, 29 May 2019 19:38:41 -0700
Subject: [PATCH 24/28] Update licensing and mention Autograd.

---
 .../third_party/Experimental/Complex.swift    | 28 +++++++++++++------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/Sources/third_party/Experimental/Complex.swift b/Sources/third_party/Experimental/Complex.swift
index 9869d55b3..c09f3dcf3 100644
--- a/Sources/third_party/Experimental/Complex.swift
+++ b/Sources/third_party/Experimental/Complex.swift
@@ -1,10 +1,10 @@
-// Copyright 2017-2019 Xiaodi Wu and The TensorFlow Authors. All Rights Reserved.
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
 //
-// Licensed under the MIT License (the "License");
+// Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     https://opensource.org/licenses/MIT
+//     http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -12,12 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 //
-//  Note
-//  ====
-//
-//  Code in libc++ is dual-licensed under the MIT and UIUC/NCSA licenses.
-//  Copyright © 2009-2017 contributors to the LLVM/libc++ project.
-/// A type to represent a complex value in Cartesian form.
+/// Note
+/// ----
+///
+/// This implementation uses a modified implementation from the
+/// xwu/NumericAnnex Swift numeric library repo vy Xiaodi Wu. To view the
+/// original code, see the implementation here
+///
+///    https://github.com/xwu/NumericAnnex/blob/master/Sources/Complex.swift
 ///
 /// Create new instances of `Complex<T>` using integer or floating-point
 /// literals and the imaginary unit `Complex<T>.i`. For example:
@@ -29,6 +31,14 @@
 /// Additional Considerations
 /// -------------------------
 ///
+/// Our implementation of complex number differentiation follows the same
+/// convention as Autograd. In short, we can get the derivative of a
+/// holomorphic function, functions whose codomain are the Reals, and
+/// functions whose codomain and domain are the Reals. You can read more about
+/// Autograd at
+///
+///    https://github.com/HIPS/autograd/blob/master/docs/tutorial.md#complex-numbers
+///
 /// Floating-point types have special values that represent infinity or NaN
 /// ("not a number"). Complex functions in different languages may return
 /// different results when working with special values.

From 4e2053abd5790952025f6691fc350d1b1d886670 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Thu, 30 May 2019 09:12:54 -0700
Subject: [PATCH 25/28] Update third party LICENSE file with tensorflow apache
 license.

---
 Sources/third_party/Experimental/LICENSE | 32 ++++++++++--------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/Sources/third_party/Experimental/LICENSE b/Sources/third_party/Experimental/LICENSE
index a3c663297..8362daf4d 100644
--- a/Sources/third_party/Experimental/LICENSE
+++ b/Sources/third_party/Experimental/LICENSE
@@ -1,19 +1,13 @@
-Copyright © 2017 Xiaodi Wu (https://github.com/xwu).
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+// Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.

From 80cab5cf9fd242b4fbd39b96d916b633b7c38f23 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Mon, 3 Jun 2019 08:56:37 -0700
Subject: [PATCH 26/28] WIP: add extra tests.

---
 Sources/third_party/Experimental/Complex.swift | 12 +++++++++++-
 Tests/ExperimentalTests/ComplexTests.swift     | 16 +++++++++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/Sources/third_party/Experimental/Complex.swift b/Sources/third_party/Experimental/Complex.swift
index 9869d55b3..f5aa1a745 100644
--- a/Sources/third_party/Experimental/Complex.swift
+++ b/Sources/third_party/Experimental/Complex.swift
@@ -4,7 +4,7 @@
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
-//     https://opensource.org/licenses/MIT
+//   https://opensource.org/licenses/MIT
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -29,12 +29,22 @@
 /// Additional Considerations
 /// -------------------------
 ///
+/// Our implementation of complex number differentiation follows the same
+/// convention as Autograd. In short, we can get the derivative of a
+/// holomorphic function, functions whose codomain are the Reals, and
+/// functions whose codomain and domain are the Reals. You can read more about
+/// Autograd at
+///
+///   https://github.com/HIPS/autograd/blob/master/docs/tutorial.md#complex-numbers
+///
 /// Floating-point types have special values that represent infinity or NaN
 /// ("not a number"). Complex functions in different languages may return
 /// different results when working with special values.
 
 struct Complex<T: FloatingPoint> {
+    @differentiable
     var real: T
+    @differentiable
     var imaginary: T
 
     @differentiable(vjp: _vjpInit where T: Differentiable, T.TangentVector == T)
diff --git a/Tests/ExperimentalTests/ComplexTests.swift b/Tests/ExperimentalTests/ComplexTests.swift
index a6b197495..c421c1213 100644
--- a/Tests/ExperimentalTests/ComplexTests.swift
+++ b/Tests/ExperimentalTests/ComplexTests.swift
@@ -300,6 +300,19 @@ final class ComplexTests: XCTestCase {
         XCTAssertEqual(expectedVector, pbComplex(Complex(real: 1, imaginary: 1)))
     }
     
+    func testImplicitDifferentiation() {
+        func addRealComponents(lhs: Complex<Float>, rhs: Complex<Float>) -> Float {
+            return lhs.real + rhs.real
+        }
+        
+        let (result, pbComplex) = valueWithPullback(at: Complex(real: 2, imaginary: -3)) { x in
+            return addRealComponents(lhs: x, rhs: Complex(real: -4, imaginary: 1))
+        }
+        
+        XCTAssertEqual(-2, result)
+        XCTAssertEqual(Complex(real: 1, imaginary: 1), pbComplex(1))
+    }
+    
     static var allTests = [
         ("testInitializer", testInitializer),
         ("testStaticImaginary", testStaticImaginary),
@@ -326,6 +339,7 @@ final class ComplexTests: XCTestCase {
         ("testVjpAddingImaginary", testVjpAddingImaginary),
         ("testVjpSubtractingReal", testVjpSubtractingReal),
         ("testVjpSubtractingImaginary", testVjpSubtractingImaginary),
-        ("testJvpDotProduct", testJvpDotProduct)
+        ("testJvpDotProduct", testJvpDotProduct),
+        ("testImplicitDifferentiation", testImplicitDifferentiation)
     ]
 }

From 33b178f38b0235dade93fd3831d7fc72add353d6 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Mon, 3 Jun 2019 12:31:29 -0700
Subject: [PATCH 27/28] Add vjpInit test.

---
 Sources/third_party/Experimental/Complex.swift | 2 --
 Tests/ExperimentalTests/ComplexTests.swift     | 9 ++++++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/Sources/third_party/Experimental/Complex.swift b/Sources/third_party/Experimental/Complex.swift
index 300a0a278..c09f3dcf3 100644
--- a/Sources/third_party/Experimental/Complex.swift
+++ b/Sources/third_party/Experimental/Complex.swift
@@ -44,9 +44,7 @@
 /// different results when working with special values.
 
 struct Complex<T: FloatingPoint> {
-    @differentiable
     var real: T
-    @differentiable
     var imaginary: T
 
     @differentiable(vjp: _vjpInit where T: Differentiable, T.TangentVector == T)
diff --git a/Tests/ExperimentalTests/ComplexTests.swift b/Tests/ExperimentalTests/ComplexTests.swift
index c421c1213..262d42bfa 100644
--- a/Tests/ExperimentalTests/ComplexTests.swift
+++ b/Tests/ExperimentalTests/ComplexTests.swift
@@ -151,6 +151,13 @@ final class ComplexTests: XCTestCase {
         expected = Complex<Float>(real: 2, imaginary: -5)
         XCTAssertEqual(expected, input.subtracting(imaginary: 1))
     }
+  
+    func testVjpInit() {
+        let pb = pullback(at: 4, -3) { r, i in
+            return Complex<Float>(real: r, imaginary: i)
+        }
+        XCTAssertEqual((-1, 2), pb(Complex<Float>(real: -1, imaginary: 2)))
+    }
     
     func testVjpAdd() {
         let pb: (Complex<Float>) -> Complex<Float> =
@@ -253,7 +260,6 @@ final class ComplexTests: XCTestCase {
     
     func testJvpDotProduct() {
         struct ComplexVector : Differentiable & AdditiveArithmetic {
-            @differentiable
             var w: Complex<Float>
             var x: Complex<Float>
             var y: Complex<Float>
@@ -329,6 +335,7 @@ final class ComplexTests: XCTestCase {
         ("testComplexConjugate", testComplexConjugate),
         ("testAdding", testAdding),
         ("testSubtracting", testSubtracting),
+        ("testVjpInit", testVjpInit),
         ("testVjpAdd", testVjpAdd),
         ("testVjpSubtract", testVjpSubtract),
         ("testVjpMultiply", testVjpMultiply),

From 4e01fed9c3bbef26257cfe72c537e1571907e786 Mon Sep 17 00:00:00 2001
From: Bart Chrzaszcz <bart.chr@gmail.com>
Date: Fri, 2 Aug 2019 15:39:47 -0700
Subject: [PATCH 28/28] Add another init test and fix test typos.

---
 Tests/ExperimentalTests/ComplexTests.swift | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/Tests/ExperimentalTests/ComplexTests.swift b/Tests/ExperimentalTests/ComplexTests.swift
index 262d42bfa..1f85894c5 100644
--- a/Tests/ExperimentalTests/ComplexTests.swift
+++ b/Tests/ExperimentalTests/ComplexTests.swift
@@ -153,10 +153,19 @@ final class ComplexTests: XCTestCase {
     }
   
     func testVjpInit() {
-        let pb = pullback(at: 4, -3) { r, i in
+        var pb = pullback(at: 4, -3) { r, i in
             return Complex<Float>(real: r, imaginary: i)
         }
-        XCTAssertEqual((-1, 2), pb(Complex<Float>(real: -1, imaginary: 2)))
+        var tanTuple = pb(Complex<Float>(real: -1, imaginary: 2))
+        XCTAssertEqual(-1, tanTuple.0)
+        XCTAssertEqual(2, tanTuple.1)
+
+      pb = pullback(at: 4, -3) { r, i in
+          return Complex<Float>(real: r * r, imaginary: i + i)
+      }
+      tanTuple = pb(Complex<Float>(real: -1, imaginary: 1))
+      XCTAssertEqual(-8, tanTuple.0)
+      XCTAssertEqual(2, tanTuple.1)
     }
     
     func testVjpAdd() {
@@ -164,7 +173,8 @@ final class ComplexTests: XCTestCase {
             pullback(at: Complex<Float>(real: 2, imaginary: 3)) { x in
             return x + Complex<Float>(real: 5, imaginary: 6)
         }
-        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)), Complex<Float>(real: 1, imaginary: 1))
+        XCTAssertEqual(pb(Complex(real: 1, imaginary: 1)),
+                       Complex<Float>(real: 1, imaginary: 1))
     }
     
     func testVjpSubtract() {
@@ -316,7 +326,7 @@ final class ComplexTests: XCTestCase {
         }
         
         XCTAssertEqual(-2, result)
-        XCTAssertEqual(Complex(real: 1, imaginary: 1), pbComplex(1))
+        XCTAssertEqual(Complex(real: 1, imaginary: 0), pbComplex(1))
     }
     
     static var allTests = [