Skip to content

Commit

Permalink
Fix undefined symbols, Migrate more ints -> sd::LongType (#9963)
Browse files Browse the repository at this point in the history 
* Remove extra op context changes due to performance regressions
Fix javacpp hack removals due to unsatisfied link on mac
Remove extra nd4j-profiler dependency in tests

* Fix up unsatisfied links by cleaning up definitions
Add new compiler flags support for nd4j-native
Add validation for missing method definitions
More clean up on ints/unsigned values to sd::Longs

* More cuda int -> sd::LongType changes

* Fix cpu builds

* Fix cpu/cuda linker flags

* Java side int -> long changes

* Fix debugging left overs, jackson versions

* Fix jackson versions in tests
  • Loading branch information
@agibsonccc
agibsonccc committed Apr 21, 2023
1 parent e756652 commit f8bb11d
Show file tree
Hide file tree
Showing 398 changed files with 3,759 additions and 4,957 deletions.
2 changes: 1 addition & 1 deletion codegen/op-codegen/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-kotlin</artifactId>
<version>2.9.9</version>
<version>${jackson.version}</version>
</dependency>

<dependency>
Expand Down
8 changes: 4 additions & 4 deletions codegen/op-codegen/src/main/ops/org/nd4j/codegen/mixins/Mixins.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ val scalar = Mixin("scalar"){
val reduce = Mixin("reduce"){
Input(DataType.NUMERIC, "in") { description = "Input variable" }
Arg(DataType.BOOL,"keepDims"){ description = "Whether to keep the original dimensions or produce a shrunk array with less dimensions"; defaultValue = false}
Arg(DataType.INT, "dimensions"){ count = AtLeast(0); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
Arg(DataType.LONG, "dimensions"){ count = AtLeast(0); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
Output(DataType.NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" }
}

Expand Down Expand Up @@ -148,7 +148,7 @@ val reduce3 = Mixin("reduce3"){
Input(DataType.NUMERIC, "y") { description = "Input variable y" }
Arg(DataType.BOOL,"keepDims",{description = "Whether to preserve original dimensions or not"; defaultValue = false})
Arg(DataType.BOOL,"isComplex",{description = "Depending on the implementation, such as distance calculations, this can determine whether all distance calculations for all points should be done."; defaultValue = false})
val dims = Arg(DataType.INT, "dimensions"){ count = AtLeast(0); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
val dims = Arg(DataType.LONG, "dimensions"){ count = AtLeast(0); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
Output(DataType.NUMERIC, "output"){ description = "Output variable" }
}

Expand All @@ -169,7 +169,7 @@ val indexAccum = Mixin("indexAccum"){
javaPackage = "org.nd4j.linalg.api.ops.impl.indexaccum"
val input = Input(DataType.NUMERIC, "in") { description = "Input variable" }
val keepDims = Arg(DataType.BOOL, "keepDims") { description = "If true: keep the dimensions that are reduced on (as length 1). False: remove the reduction dimensions"; defaultValue = false }
val dims = Arg(DataType.INT, "dimensions"){ count = AtLeast(1); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
val dims = Arg(DataType.LONG, "dimensions"){ count = AtLeast(1); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
Output(DataType.NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" }

Signature(input, dims)
Expand All @@ -194,7 +194,7 @@ val indexAccumCustom = Mixin("indexAccumCustom"){
javaPackage = "org.nd4j.linalg.api.ops.impl.indexaccum.custom"
val input = Input(DataType.NUMERIC, "in") { description = "Input variable" }
val keepDims = Arg(DataType.BOOL, "keepDims") { description = "If true: keep the dimensions that are reduced on (as length 1). False: remove the reduction dimensions"; defaultValue = false }
val dims = Arg(DataType.INT, "dimensions"){ count = AtLeast(1); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
val dims = Arg(DataType.LONG, "dimensions"){ count = AtLeast(1); isVargarg = true; description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed" }
Output(DataType.NUMERIC, "output"){ description = "Reduced array of rank (input rank - num dimensions)" }

Signature(input, dims)
Expand Down
15 changes: 7 additions & 8 deletions codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/Math.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,11 +226,10 @@ fun Math() = Namespace("Math") {
javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.clip"
val x = Input(NUMERIC, "x") { description = "Input variable" }
val clipValue = Arg(NUMERIC, "clipValue") { description = "Clipping value (maximum l2 norm)" }
Arg(INT, "dimensions"){ count = AtLeast(0); description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed"} //; defaultValue = intArrayOf(0) } //TODO
Arg(LONG, "dimensions"){ count = AtLeast(0); description = "Dimensions to reduce over. If dimensions are not specified, full array reduction is performed"} //; defaultValue = intArrayOf(0) } //TODO
Output(NUMERIC, "output"){ description = "Output variable" }

// AllParamSignature(withOutput = false)
// Signature(x, clipValue)

Doc(Language.ANY, DocScope.ALL){
"""
Clipping by L2 norm, optionally along dimension(s)
Expand Down Expand Up @@ -264,7 +263,7 @@ fun Math() = Namespace("Math") {
javaOpClass = "ClipByAvgNorm"
Input(NUMERIC, "x") { description = "Input variable" }
Arg(NUMERIC, "clipValue") { description = "Value for clipping" }
Arg(INT, "dimensions"){ count = AtLeast(0); description = "Dimensions to reduce over"}
Arg(LONG, "dimensions"){ count = AtLeast(0); description = "Dimensions to reduce over"}
Output(NUMERIC, "output"){ description = "Output variable" }
Doc(Language.ANY, DocScope.ALL){
"""
Expand Down Expand Up @@ -557,7 +556,7 @@ fun Math() = Namespace("Math") {
Arg(INT, "rows") { description = "Number of rows" }
Arg(INT, "cols") { description = "Number of columns" }
Arg(DATA_TYPE, "dataType") { description = "Data type" } //TODO: Mapped DataType to INT.
Arg(DataType.INT, "dimensions"){ count = AtLeast(0)}
Arg(DataType.LONG, "dimensions"){ count = AtLeast(0)}
Output(NUMERIC, "output"){ description = "Identity matrix" }
Doc(Language.ANY, DocScope.ALL){
"""
Expand Down Expand Up @@ -819,7 +818,7 @@ fun Math() = Namespace("Math") {
Op("logSumExp") {
javaPackage = "org.nd4j.linalg.api.ops.impl.reduce.custom"
Input(NUMERIC, "input") { description = "Input variable" }
Arg(INT, "dimensions"){ count = AtLeast(0); description = "Optional dimensions to reduce along" }
Arg(LONG, "dimensions"){ count = AtLeast(0); description = "Optional dimensions to reduce along" }
Output(NUMERIC, "output"){ description = "Output variable" }
Doc(Language.ANY, DocScope.ALL){
"""
Expand Down Expand Up @@ -972,7 +971,7 @@ fun Math() = Namespace("Math") {
Op("moments") {
javaPackage = "org.nd4j.linalg.api.ops.impl.reduce"
Input(NUMERIC, "input") { description = "Input to calculate moments for" }
Arg(INT, "axes"){ count = AtLeast(0); description = "Dimensions to perform calculation over" }
Arg(LONG, "axes"){ count = AtLeast(0); description = "Dimensions to perform calculation over" }
Arg(BOOL,"keepDims") { description = "Whether to keep dimensions during reduction or not. "}
Output(NUMERIC, "output_mean"){ description = "Mean variable" }
Output(NUMERIC, "output_variance"){ description = "Variance variable" }
Expand Down Expand Up @@ -1335,7 +1334,7 @@ fun Math() = Namespace("Math") {
Op("standardize") {
javaPackage = "org.nd4j.linalg.api.ops.impl.transforms.custom"
Input(NUMERIC, "x") { description = "Input variable" }
Arg(INT, "dimensions"){ count = AtLeast(1); description = "" } //TODO: Missing description for dimension.
Arg(LONG, "dimensions"){ count = AtLeast(1); description = "" } //TODO: Missing description for dimension.
Output(NUMERIC, "output"){ description = "Output variable" }
Doc(Language.ANY, DocScope.ALL){
"""
Expand Down
2 changes: 1 addition & 1 deletion codegen/op-codegen/src/main/ops/org/nd4j/codegen/ops/NeuralNetwork.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,7 @@ fun NN() = Namespace("NN") {
val g = Input(NUMERIC, "gain") { description = "Gain" }
Input(NUMERIC, "bias") { description = "Bias"; defaultValue = null}
val ch = Arg(BOOL, "channelsFirst") { description = "For 2D input - unused. True for NCHW (minibatch, channels, height, width), false for NHWC data" }
val dim = Arg(INT, "dimensions") { count = AtLeast(1); description = "Dimensions to perform layer norm over - dimension=1 for 2d/MLP data, dimension=1,2,3 for CNNs" }
val dim = Arg(LONG, "dimensions") { count = AtLeast(1); description = "Dimensions to perform layer norm over - dimension=1 for 2d/MLP data, dimension=1,2,3 for CNNs" }

Output(NUMERIC, "output") { description = "Output variable" }

Expand Down

0 comments on commit f8bb11d

Please sign in to comment.