From a484d34d753ad1a1ae39151392bfa42dd207a299 Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Thu, 19 Jan 2017 16:27:25 -0800 Subject: [PATCH 001/540] WIP. --- .../main/java/org/apache/ignite/IgniteMath.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 modules/core/src/main/java/org/apache/ignite/IgniteMath.java diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteMath.java b/modules/core/src/main/java/org/apache/ignite/IgniteMath.java new file mode 100644 index 0000000000000..ceef51d6b1ff8 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/IgniteMath.java @@ -0,0 +1,16 @@ +// @java.file.header + +/* _________ _____ __________________ _____ + * __ ____/___________(_)______ /__ ____/______ ____(_)_______ + * _ / __ __ ___/__ / _ __ / _ / __ _ __ `/__ / __ __ \ + * / /_/ / _ / _ / / /_/ / / /_/ / / /_/ / _ / _ / / / + * \____/ /_/ /_/ \_,__/ \____/ \__,_/ /_/ /_/ /_/ + */ + +package org.apache.ignite; + +/** + * TODO: add description. + */ +public interface IgniteMath { +} From b2f2fe2eecfec7aaff2df368aa5b1da42cde80f5 Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Thu, 19 Jan 2017 16:28:11 -0800 Subject: [PATCH 002/540] WIP. --- .../java/org/apache/ignite/IgniteMath.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteMath.java b/modules/core/src/main/java/org/apache/ignite/IgniteMath.java index ceef51d6b1ff8..7b021e0dea259 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteMath.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteMath.java @@ -1,16 +1,26 @@ -// @java.file.header - -/* _________ _____ __________________ _____ - * __ ____/___________(_)______ /__ ____/______ ____(_)_______ - * _ / __ __ ___/__ / _ __ / _ / __ _ __ `/__ / __ __ \ - * / /_/ / _ / _ / / /_/ / / /_/ / / /_/ / _ / _ / / / - * \____/ /_/ /_/ \_,__/ \____/ \__,_/ /_/ /_/ /_/ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.ignite; +import org.apache.ignite.lang.*; + /** - * TODO: add description. + * Defines distributed core algebra support for dense and sparse data sets. */ -public interface IgniteMath { +public interface IgniteMath extends IgniteAsyncSupport { } From 0e7305d81431ec5311768ef18a997f9a386e4b3d Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Thu, 19 Jan 2017 16:31:49 -0800 Subject: [PATCH 003/540] WIP. --- .../java/org/apache/ignite/math/Matrix.java | 24 +++++++++++++++++++ .../java/org/apache/ignite/math/Vector.java | 24 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 modules/core/src/main/java/org/apache/ignite/math/Matrix.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/Vector.java diff --git a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java new file mode 100644 index 0000000000000..80a904cfe2dc7 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math; + +/** + * TODO: add description. + */ +public interface Matrix { +} diff --git a/modules/core/src/main/java/org/apache/ignite/math/Vector.java b/modules/core/src/main/java/org/apache/ignite/math/Vector.java new file mode 100644 index 0000000000000..89ae169b6334d --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/Vector.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math; + +/** + * TODO: add description. + */ +public interface Vector { +} From dabda0c6c87812c2e0dd0fc99f1e999736ba45d6 Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Fri, 20 Jan 2017 18:15:45 -0800 Subject: [PATCH 004/540] WIP. --- .../ignite/math/CardinalityException.java | 35 +++ .../apache/ignite/math/IndexException.java | 33 +++ .../java/org/apache/ignite/math/Matrix.java | 210 +++++++++++++++++- .../ignite/math/UnknownProviderException.java | 33 +++ .../math/UnsupportedOperationException.java | 35 +++ 5 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/math/CardinalityException.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/IndexException.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/UnknownProviderException.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java diff --git a/modules/core/src/main/java/org/apache/ignite/math/CardinalityException.java b/modules/core/src/main/java/org/apache/ignite/math/CardinalityException.java new file mode 100644 index 0000000000000..4be87cc349e05 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/CardinalityException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math; + +import org.apache.ignite.*; + +/** + * Indicates a cardinality mismatch in matrix or vector operations. + */ +public class CardinalityException extends IgniteException { + /** + * Creates new cardinality violation exception. + * + * @param exp Expected cardinality. + * @param act Actual cardinality. + */ + public CardinalityException(int exp, int act) { + super("Cardinality violation [expected=" + exp + ", actual=" + act + "]"); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/math/IndexException.java b/modules/core/src/main/java/org/apache/ignite/math/IndexException.java new file mode 100644 index 0000000000000..79260b98d1fd8 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/IndexException.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math; + +import org.apache.ignite.*; + +/** + * Indicates an invalid, i.e. out of bound, index on matrix or vector operations. + */ +public class IndexException extends IgniteException { + /** + * + * @param idx Index value that caused this exception. + */ + public IndexException(int idx) { + super("Invalid (out of bound) index: " + idx); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java index 80a904cfe2dc7..5e89d80d1fb52 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java @@ -17,8 +17,216 @@ package org.apache.ignite.math; +import org.apache.ignite.cluster.*; +import org.apache.ignite.lang.*; +import java.util.*; +import java.util.function.*; + /** - * TODO: add description. + * A matrix interface. + * + * Based on its flavor it can have vastly different implementations tailored for + * for different types of data (e.g. dense vs. sparse), different sizes of data or different operation + * optimizations. + * + * Note also that not all operations can be supported by all underlying implementations. If an operation is not + * supported a {@link UnsupportedOperationException} is thrown. This exception can also be thrown in partial cases + * where an operation is unsupported only in special cases, e.g. where a given operation cannot be deterministically + * completed in polynomial time. */ public interface Matrix { + /** + * Assigns given value to all elements of this matrix. + * + * @param val Value to assign to all elements. + * @return This matrix. + */ + Matrix assign(double val); + + /** + * Assigns given values to this matrix. + * + * @param vals Values to assign. + * @return This matrix. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix assign(double[][] vals); + + /** + * Assigns values from given matrix to this matrix. + * + * @param mtx Matrix to assign to this matrix. + * @return This matrix. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix assign(Matrix mtx); + + /** + * Maps all values in this matrix through a given function. + * + * @param fun Mapping function. + * @return This matrix. + */ + Matrix map(DoubleFunction fun); + + /** + * Maps all values in this matrix through a given function. + * + * For this matrix A, argument matrix B and the + * function F this method maps every cell x, y as: + * A(x,y) = fun(A(x,y), B(x,y)) + * + * @param mtx Argument matrix. + * @param fun Mapping function. + * @return This function. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix map(Matrix mtx, BiFunction fun); + + /** + * Assigns values from given vector to the specified column in this matrix. + * + * @param col Column index. + * @param vec Vector to get values from. + * @return This matrix. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix assignColumn(int col, Vector vec); + + /** + * Assigns values from given vector to the specified row in this matrix. + * + * @param row Row index. + * @param vec Vector to get values from. + * @return This matrix. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix assignRow(int row, Vector vec); + + /** + * Collects the results of applying a given function to all rows in this matrix. + * + * @param fun Aggregating function. + * @return Vector of row aggregates. + */ + Vector aggregateRows(Function fun); + + /** + * Collects the results of applying a given function to all columns in this matrix. + * + * @param fun Aggregating function. + * @return Vector of column aggregates. + */ + Vector aggregateColumns(Function fun); + + /** + * Folds matrix into a single value. + * + * @param foldFun Folding function that takes two parameters: accumulator and the current value. + * @param mapFun Mapping function that is called on each matrix cell before its passed to the accumulator + * (as its second parameter). + * @return Folded value of this matrix. + */ + double foldMap(BiFunction foldFun, DoubleFunction mapFun); + + /** + * Gets number of columns in this matrix. + * + * @return The number of columns in this matrix. + */ + int columnSize(); + + /** + * Gets number of rows in this matrix. + * + * @return The number of rows in this matrix. + */ + int rowSize(); + + /** + * Returns matrix determinator using Laplace theorem. + * + * @return A determinator for this matrix. + */ + double determinant(); + + /** + * Divides each value in this matrix by the argument. + * + * @param x Divider value. + * @return This matrix. + */ + Matrix divide(double x); + + /** + * Gets the matrix value at the provided location. + * + * @param row Row index. + * @param col Column index. + * @return Matrix value. + * @throws IndexException Thrown in case of index is out of bound. + */ + double get(int row, int col); + + /** + * Gets the matrix value at the provided location without checking boundaries. + * This method is marginally quicker than its {@link #get(int, int)} sibling. + * + * @param row Row index. + * @param col Column index. + * @return Matrix value. + */ + double getX(int row, int col); + + /** + * Creates new empty matrix of the same underlying class and the same size as this matrix. + * + * NOTE: new matrix will have the same flavor as the this matrix but a different ID. + * + * @return New matrix of the same size using the same underlying class. + */ + Matrix cloneEmpty(); + + /** + * Clones this matrix. + * + * NOTE: new matrix will have the same flavor as the this matrix but a different ID. + * + * @return New matrix of the same underlying class, the same size and the same values. + */ + Matrix clone(); + + /** + * Creates new empty matrix of the same underlying class but of different size. + * + * NOTE: new matrix will have the same flavor as the this matrix but a different ID. + * + * @param rows Number of rows for new matrix. + * @param cols Number of columns for new matrix. + * @return New matrix. + */ + Matrix cloneEmpty(int rows, int cols); + + Matrix minus(Matrix x); + Matrix plus(double x); + Matrix plus(Matrix x); + + /** + * Auto-generated globally unique matrix ID. + * + * @return Matrix GUID. + */ + IgniteUuid guid(); + + /** + * + * @return Matrix flavor. + */ + String flavor(); + + /** + * + * @return Optional cluster group for this matrix to be stored on. + */ + Optional clusterGroup(); } diff --git a/modules/core/src/main/java/org/apache/ignite/math/UnknownProviderException.java b/modules/core/src/main/java/org/apache/ignite/math/UnknownProviderException.java new file mode 100644 index 0000000000000..f2afd8e3fb16f --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/UnknownProviderException.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math; + +import org.apache.ignite.*; + +/** + * Indicates that no provider has been found for a given vector or matrix flavor. + */ +public class UnknownProviderException extends IgniteException { + /** + * + * @param flavor Flavor (a.k.a. operation performance hints) that has no registered provider for. + */ + public UnknownProviderException(String flavor) { + super("No provider has been found for the flavor: " + flavor); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java b/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java new file mode 100644 index 0000000000000..d8ff64c15cabd --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math; + +import org.apache.ignite.*; + +/** + * Thrown by matrices and vectors to indicate that a specific operation is not + * supported by the used implementation. In some cases, an operation may be unsupported + * only in certain cases where, for example, it could not be performed in polynomial time. + */ +public class UnsupportedOperationException extends IgniteException { + /** + * + * @param errMsg Error message. + */ + public UnsupportedOperationException(String errMsg) { + super(errMsg); + } +} From efc6d30ca3058bff595d0283d1698b7f448d33fd Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Fri, 20 Jan 2017 19:39:31 -0800 Subject: [PATCH 005/540] WIP. --- .../java/org/apache/ignite/math/Matrix.java | 26 +++++++++++++++++-- .../java/org/apache/ignite/math/Vector.java | 8 ++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java index 5e89d80d1fb52..c88b600a9a610 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java @@ -207,9 +207,31 @@ public interface Matrix { */ Matrix cloneEmpty(int rows, int cols); - Matrix minus(Matrix x); + /** + * Decrements each value in this matrix by corresponding value from the argument matrix. + * + * @param mtx Argument matrix. + * @return This matrix. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix minus(Matrix mtx); + + /** + * Adds given value to each element of this matrix. + * + * @param x Value to add. + * @return This matrix. + */ Matrix plus(double x); - Matrix plus(Matrix x); + + /** + * Increase each element in this matrix by corresponding value from the argument matrix. + * + * @param mtx Argument matrix. + * @return This matrix. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix plus(Matrix mtx); /** * Auto-generated globally unique matrix ID. diff --git a/modules/core/src/main/java/org/apache/ignite/math/Vector.java b/modules/core/src/main/java/org/apache/ignite/math/Vector.java index 89ae169b6334d..4b6de33a9244a 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Vector.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Vector.java @@ -17,8 +17,16 @@ package org.apache.ignite.math; +import org.apache.ignite.lang.*; + /** * TODO: add description. */ public interface Vector { + /** + * Auto-generated globally unique vectgor ID. + * + * @return Vector GUID. + */ + IgniteUuid guid(); } From 57c83b7c1cf291678a3c08919e2d12a0c6f319e1 Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Fri, 20 Jan 2017 21:50:43 -0800 Subject: [PATCH 006/540] WIP. --- .../ignite/math/AssociativeFunction.java | 26 +++++++++++++++++++ .../ignite/math/CommutativeFunction.java | 25 ++++++++++++++++++ .../java/org/apache/ignite/math/Matrix.java | 17 +++++++----- 3 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/math/AssociativeFunction.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/CommutativeFunction.java diff --git a/modules/core/src/main/java/org/apache/ignite/math/AssociativeFunction.java b/modules/core/src/main/java/org/apache/ignite/math/AssociativeFunction.java new file mode 100644 index 0000000000000..868ed934b062f --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/AssociativeFunction.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math; + +/** + * Marker interface for functions that have associative property, + * i.e f(x, f(y, z)) = f(f(x, y), z) for any x, y, z. + */ +public interface AssociativeFunction { + // No-op. +} diff --git a/modules/core/src/main/java/org/apache/ignite/math/CommutativeFunction.java b/modules/core/src/main/java/org/apache/ignite/math/CommutativeFunction.java new file mode 100644 index 0000000000000..0e02c50d00c63 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/CommutativeFunction.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math; + +/** + * Marker interface for functions with commutative property, i.e. f(x, y) = f(y, x) for any x, y. + */ +public interface CommutativeFunction { + // No-op.. +} diff --git a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java index c88b600a9a610..15bdba7d560cc 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java @@ -203,32 +203,35 @@ public interface Matrix { * * @param rows Number of rows for new matrix. * @param cols Number of columns for new matrix. - * @return New matrix. + * @return New matrix of the same underlying class and size. */ Matrix cloneEmpty(int rows, int cols); /** - * Decrements each value in this matrix by corresponding value from the argument matrix. + * Creates new matrix where each value is a difference between corresponding value of this matrix and + * passed in argument matrix. * * @param mtx Argument matrix. - * @return This matrix. + * @return New matrix of the same underlying class and size. * @throws CardinalityException Thrown if cardinalities mismatch. */ Matrix minus(Matrix mtx); /** - * Adds given value to each element of this matrix. + * Creates new matrix where each value is a sum of the corresponding value of this matrix and + * argument value. * * @param x Value to add. - * @return This matrix. + * @return New matrix of the same underlying class and size. */ Matrix plus(double x); /** - * Increase each element in this matrix by corresponding value from the argument matrix. + * Creates new matrix where each value is a sum of corresponding values of this matrix and + * passed in argument matrix. * * @param mtx Argument matrix. - * @return This matrix. + * @return New matrix of the same underlying class and size. * @throws CardinalityException Thrown if cardinalities mismatch. */ Matrix plus(Matrix mtx); From 7d290a601a259a0867ab3daba5892790f90a7172 Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Fri, 20 Jan 2017 22:50:07 -0800 Subject: [PATCH 007/540] WIP. --- .../main/java/org/apache/ignite/math/Matrix.java | 2 ++ .../main/java/org/apache/ignite/math/Vector.java | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java index 15bdba7d560cc..a682d4bf28ad5 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java @@ -33,6 +33,8 @@ * supported a {@link UnsupportedOperationException} is thrown. This exception can also be thrown in partial cases * where an operation is unsupported only in special cases, e.g. where a given operation cannot be deterministically * completed in polynomial time. + * + * Based on ideas from Apache Mahout. */ public interface Matrix { /** diff --git a/modules/core/src/main/java/org/apache/ignite/math/Vector.java b/modules/core/src/main/java/org/apache/ignite/math/Vector.java index 4b6de33a9244a..dd1f18c6267bc 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Vector.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Vector.java @@ -20,11 +20,22 @@ import org.apache.ignite.lang.*; /** - * TODO: add description. + * A vector interface. + * + * Based on its flavor it can have vastly different implementations tailored for + * for different types of data (e.g. dense vs. sparse), different sizes of data or different operation + * optimizations. + * + * Note also that not all operations can be supported by all underlying implementations. If an operation is not + * supported a {@link UnsupportedOperationException} is thrown. This exception can also be thrown in partial cases + * where an operation is unsupported only in special cases, e.g. where a given operation cannot be deterministically + * completed in polynomial time. + * + * Based on ideas from Apache Mahout. */ public interface Vector { /** - * Auto-generated globally unique vectgor ID. + * Auto-generated globally unique vector ID. * * @return Vector GUID. */ From 829275a49fbb6371b2651170d55a1b935e1e9ad6 Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Sat, 21 Jan 2017 12:11:40 -0800 Subject: [PATCH 008/540] WIP. --- .../java/org/apache/ignite/math/Matrix.java | 114 +++++++++++++++++- 1 file changed, 113 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java index a682d4bf28ad5..619ade59acc05 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java @@ -128,8 +128,9 @@ public interface Matrix { * @param mapFun Mapping function that is called on each matrix cell before its passed to the accumulator * (as its second parameter). * @return Folded value of this matrix. + * @param Type of the folded value. */ - double foldMap(BiFunction foldFun, DoubleFunction mapFun); + T foldMap(BiFunction foldFun, DoubleFunction mapFun); /** * Gets number of columns in this matrix. @@ -245,6 +246,117 @@ public interface Matrix { */ IgniteUuid guid(); + /** + * Sets given value. + * + * @param row Row index. + * @param col Column index. + * @param val Value to set. + * @return This matrix. + * @throws IndexException Thrown in case of either index is out of bound. + */ + Matrix set(int row, int col, double val); + + /** + * Sets values for given row. + * + * @param row Row index. + * @param data Row data to set. + * @return This matrix. + * @throws IndexException Thrown in case of index is out of bound. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix setRow(int row, double[] data); + + /** + * Sets values for given column. + * + * @param col Column index. + * @param data Column data to set. + * @return This matrix. + * @throws IndexException Thrown in case of index is out of bound. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix setColumn(int col, double[] data); + + /** + * Sets given value without checking for index bounds. This method is marginally faster + * than its {@link #set(int, int, double)} sibling. + * + * @param row Row index. + * @param col Column index. + * @param val Value to set. + * @return This matrix. + */ + Matrix setX(int row, int col, double val); + + /** + * Creates new matrix containing the product of given value and values in this matrix. + * + * @param x Value to multiply. + * @return New matrix. + */ + Matrix times(double x); + + /** + * Creates new matrix that is the product of multiplying this matrix and the argument matrix. + * + * @param mtx Argument matrix. + * @return New matrix. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Matrix times(Matrix mtx); + + /** + * Gets sum of all elements in the matrix. + * + * @return Sum of all elements in this matrix. + */ + double sum(); + + /** + * Creates new matrix that is transpose of this matrix. + * + * @return New transposed matrix. + */ + Matrix transpose(); + + /** + * Creates new view into this matrix. Changes to the view will be propagated to this matrix. + * + * @param offset View offset as int[x,y]. + * @param size View size as int[rows, cols] + * @return New view. + * @throws CardinalityException Thrown if cardinalities mismatch. + * @throws IndexException Thrown in case of offset is out of bound. + */ + Matrix viewPart(int[] offset, int[] size); + + /** + * Creates new view into matrix row. Changes to the view will be propagated to this matrix. + * + * @param row Row index. + * @return New view. + * @throws IndexException Thrown in case of index is out of bound. + */ + Vector viewRow(int row); + + /** + * Creates new view into matrix column . Changes to the view will be propagated to this matrix. + * + * @param col Column index. + * @return New view. + * @throws IndexException Thrown in case of index is out of bound. + */ + Vector viewColumn(int col); + + /** + * Creates new view into matrix diagonal. Changes to the view will be propagated to this matrix. + * + * @return New view. + */ + Vector viewDiagonal(); + /** * * @return Matrix flavor. From 49cc25e6bda4d036bb005b703f704898f8f86d99 Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Sat, 21 Jan 2017 14:17:38 -0800 Subject: [PATCH 009/540] WIP. --- .../apache/ignite/math/IndexException.java | 3 + .../java/org/apache/ignite/math/Matrix.java | 2 + .../ignite/math/UnknownProviderException.java | 3 + .../math/UnsupportedOperationException.java | 3 + .../java/org/apache/ignite/math/Vector.java | 69 ++++++ .../math/impls/DenseLocalOnHeapMatrix.java | 218 ++++++++++++++++++ .../math/impls/DenseLocalOnHeapVector.java | 24 ++ .../ignite/math/impls/package-info.java | 22 ++ .../org/apache/ignite/math/package-info.java | 22 ++ 9 files changed, 366 insertions(+) create mode 100644 modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapMatrix.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapVector.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/impls/package-info.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/package-info.java diff --git a/modules/core/src/main/java/org/apache/ignite/math/IndexException.java b/modules/core/src/main/java/org/apache/ignite/math/IndexException.java index 79260b98d1fd8..e03e35798d644 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/IndexException.java +++ b/modules/core/src/main/java/org/apache/ignite/math/IndexException.java @@ -23,6 +23,9 @@ * Indicates an invalid, i.e. out of bound, index on matrix or vector operations. */ public class IndexException extends IgniteException { + /** */ + private static final long serialVersionUID = 0L; + /** * * @param idx Index value that caused this exception. diff --git a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java index 619ade59acc05..fe1d8d7bd0494 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java @@ -364,6 +364,8 @@ public interface Matrix { String flavor(); /** + * Gets optional cluster group this matrix is stored on. In case of local JVM storage it may + * return an empty option or a cluster group consisting of only the local Ignite node. * * @return Optional cluster group for this matrix to be stored on. */ diff --git a/modules/core/src/main/java/org/apache/ignite/math/UnknownProviderException.java b/modules/core/src/main/java/org/apache/ignite/math/UnknownProviderException.java index f2afd8e3fb16f..730be1da91bf5 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/UnknownProviderException.java +++ b/modules/core/src/main/java/org/apache/ignite/math/UnknownProviderException.java @@ -23,6 +23,9 @@ * Indicates that no provider has been found for a given vector or matrix flavor. */ public class UnknownProviderException extends IgniteException { + /** */ + private static final long serialVersionUID = 0L; + /** * * @param flavor Flavor (a.k.a. operation performance hints) that has no registered provider for. diff --git a/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java b/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java index d8ff64c15cabd..b02d4911d2aeb 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java +++ b/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java @@ -25,6 +25,9 @@ * only in certain cases where, for example, it could not be performed in polynomial time. */ public class UnsupportedOperationException extends IgniteException { + /** */ + private static final long serialVersionUID = 0L; + /** * * @param errMsg Error message. diff --git a/modules/core/src/main/java/org/apache/ignite/math/Vector.java b/modules/core/src/main/java/org/apache/ignite/math/Vector.java index dd1f18c6267bc..cb0498afec1c0 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Vector.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Vector.java @@ -17,7 +17,10 @@ package org.apache.ignite.math; +import org.apache.ignite.cluster.*; import org.apache.ignite.lang.*; +import java.util.*; +import java.util.function.*; /** * A vector interface. @@ -34,6 +37,72 @@ * Based on ideas from Apache Mahout. */ public interface Vector { + /** + * Assigns given value to all elements of this vector. + * + * @param val Value to assign. + * @return This vector. + */ + Vector assign(double val); + + /** + * Assigns values from given array to this vector. + * + * @param vals Values to assign. + * @throws CardinalityException Thrown if cardinalities mismatch. + * @return This vector. + */ + Vector assign(double[] vals); + + /** + * Copies values from the argument vector to this one. + * + * @param vec Argument vector. + * @return This vector. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Vector assign(Vector vec); + + /** + * Maps all values in this vector through a given function. + * + * @param fun Mapping function. + * @return This vector. + */ + Vector map(DoubleFunction fun); + + /** + * Maps all values in this vector through a given function. + * + * For this vector A, argument vector B and the + * function F this method maps every element x as: + * A(x) = fun(A(x), B(x)) + * + * @param vec Argument vector. + * @param fun Mapping function. + * @return This function. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Vector map(Vector vec, DoubleFunction fun); + + /** + * Maps all elements of this vector by applying given function to each element with a constant + * second parameter y. + * + * @param fun Mapping function. + * @param y Second parameter for mapping function. + * @return This vector. + */ + Vector map(BiFunction fun, double y); + + /** + * Gets optional cluster group this vector is stored on. In case of local JVM storage it may + * return an empty option or a cluster group consisting of only the local Ignite node. + * + * @return Optional cluster group for this vector to be stored on. + */ + Optional clusterGroup(); + /** * Auto-generated globally unique vector ID. * diff --git a/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapMatrix.java b/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapMatrix.java new file mode 100644 index 0000000000000..f1106a64ca412 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapMatrix.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math.impls; + +import org.apache.ignite.cluster.*; +import org.apache.ignite.lang.*; +import org.apache.ignite.math.*; +import org.apache.ignite.math.Vector; +import java.util.*; +import java.util.function.*; + +/** + * This is a trivial implementation for matrix assuming dense logic, local on-heap JVM storage + * based on double[][] array. It is only suitable for data sets where + * local, non-distributed execution is satisfactory and on-heap JVM storage is enough + * to keep the entire data set. + */ +public class DenseLocalOnHeapMatrix implements Matrix { + @Override + public Matrix assign(double val) { + return null; // TODO + } + + @Override + public Matrix assign(double[][] vals) { + return null; // TODO + } + + @Override + public Matrix assign(Matrix mtx) { + return null; // TODO + } + + @Override + public Matrix map(DoubleFunction fun) { + return null; // TODO + } + + @Override + public Matrix map(Matrix mtx, BiFunction fun) { + return null; // TODO + } + + @Override + public Matrix assignColumn(int col, Vector vec) { + return null; // TODO + } + + @Override + public Matrix assignRow(int row, Vector vec) { + return null; // TODO + } + + @Override + public Vector aggregateRows(Function fun) { + return null; // TODO + } + + @Override + public Vector aggregateColumns(Function fun) { + return null; // TODO + } + + @Override + public T foldMap(BiFunction foldFun, DoubleFunction mapFun) { + return null; // TODO + } + + @Override + public int columnSize() { + return 0; // TODO + } + + @Override + public int rowSize() { + return 0; // TODO + } + + @Override + public double determinant() { + return 0; // TODO + } + + @Override + public Matrix divide(double x) { + return null; // TODO + } + + @Override + public double get(int row, int col) { + return 0; // TODO + } + + @Override + public double getX(int row, int col) { + return 0; // TODO + } + + @Override + public Matrix cloneEmpty() { + return null; // TODO + } + + @Override + public Matrix clone() { + return null; // TODO + } + + @Override + public Matrix cloneEmpty(int rows, int cols) { + return null; // TODO + } + + @Override + public Matrix minus(Matrix mtx) { + return null; // TODO + } + + @Override + public Matrix plus(double x) { + return null; // TODO + } + + @Override + public Matrix plus(Matrix mtx) { + return null; // TODO + } + + @Override + public IgniteUuid guid() { + return null; // TODO + } + + @Override + public Matrix set(int row, int col, double val) { + return null; // TODO + } + + @Override + public Matrix setRow(int row, double[] data) { + return null; // TODO + } + + @Override + public Matrix setColumn(int col, double[] data) { + return null; // TODO + } + + @Override + public Matrix setX(int row, int col, double val) { + return null; // TODO + } + + @Override + public Matrix times(double x) { + return null; // TODO + } + + @Override + public Matrix times(Matrix mtx) { + return null; // TODO + } + + @Override + public double sum() { + return 0; // TODO + } + + @Override + public Matrix transpose() { + return null; // TODO + } + + @Override + public Matrix viewPart(int[] offset, int[] size) { + return null; // TODO + } + + @Override + public Vector viewRow(int row) { + return null; // TODO + } + + @Override + public Vector viewColumn(int col) { + return null; // TODO + } + + @Override + public Vector viewDiagonal() { + return null; // TODO + } + + @Override + public String flavor() { + return null; // TODO + } + + @Override + public Optional clusterGroup() { + return null; // TODO + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapVector.java b/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapVector.java new file mode 100644 index 0000000000000..03cf747df1d81 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapVector.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.math.impls; + +/** + * TODO: add description. + */ +public class DenseLocalOnHeapVector { +} diff --git a/modules/core/src/main/java/org/apache/ignite/math/impls/package-info.java b/modules/core/src/main/java/org/apache/ignite/math/impls/package-info.java new file mode 100644 index 0000000000000..f1b2e36560587 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/impls/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains specific implementations for core algebra. + */ +package org.apache.ignite.math.impls; \ No newline at end of file diff --git a/modules/core/src/main/java/org/apache/ignite/math/package-info.java b/modules/core/src/main/java/org/apache/ignite/math/package-info.java new file mode 100644 index 0000000000000..05ce651a0b100 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/math/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains main APIs for distributed code algebra. + */ +package org.apache.ignite.math; \ No newline at end of file From 43453ee714f9c84c75f04cf00fb3d8aef2fdc4f1 Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Sat, 21 Jan 2017 19:46:07 -0800 Subject: [PATCH 010/540] WIP. --- .../ignite/math/CardinalityException.java | 3 + .../java/org/apache/ignite/math/Matrix.java | 2 +- .../math/UnsupportedOperationException.java | 6 +- .../java/org/apache/ignite/math/Vector.java | 310 ++++++++++++++++++ .../commands/alert/VisorAlertCommand.scala | 4 +- .../visor/commands/gc/VisorGcCommand.scala | 2 +- .../commands/tasks/VisorTasksCommand.scala | 18 +- .../scala/org/apache/ignite/visor/visor.scala | 2 +- 8 files changed, 329 insertions(+), 18 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/math/CardinalityException.java b/modules/core/src/main/java/org/apache/ignite/math/CardinalityException.java index 4be87cc349e05..567598a7cf8f8 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/CardinalityException.java +++ b/modules/core/src/main/java/org/apache/ignite/math/CardinalityException.java @@ -23,6 +23,9 @@ * Indicates a cardinality mismatch in matrix or vector operations. */ public class CardinalityException extends IgniteException { + /** */ + private static final long serialVersionUID = 0L; + /** * Creates new cardinality violation exception. * diff --git a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java index fe1d8d7bd0494..775973ad8307e 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Matrix.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Matrix.java @@ -122,7 +122,7 @@ public interface Matrix { Vector aggregateColumns(Function fun); /** - * Folds matrix into a single value. + * Folds this matrix into a single value. * * @param foldFun Folding function that takes two parameters: accumulator and the current value. * @param mapFun Mapping function that is called on each matrix cell before its passed to the accumulator diff --git a/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java b/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java index b02d4911d2aeb..2a40665906226 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java +++ b/modules/core/src/main/java/org/apache/ignite/math/UnsupportedOperationException.java @@ -20,9 +20,9 @@ import org.apache.ignite.*; /** - * Thrown by matrices and vectors to indicate that a specific operation is not - * supported by the used implementation. In some cases, an operation may be unsupported - * only in certain cases where, for example, it could not be performed in polynomial time. + * Indicate that a specific operation is not supported by the underlying implementation. + * In some cases, an operation may be unsupported only in certain cases where, for example, + * it could not be deterministically completed in polynomial time. */ public class UnsupportedOperationException extends IgniteException { /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/math/Vector.java b/modules/core/src/main/java/org/apache/ignite/math/Vector.java index cb0498afec1c0..a03ed8da3981a 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/Vector.java +++ b/modules/core/src/main/java/org/apache/ignite/math/Vector.java @@ -37,6 +37,95 @@ * Based on ideas from Apache Mahout. */ public interface Vector { + /** + * Holder for vector's element. + */ + interface Element { + /** + * Gets element's value. + * + * @return The value of this vector element. + */ + double get(); + + /** + * Gets element's index in the vector. + * + * @return The index of this vector element. + */ + int index(); + + /** + * Sets element's value. + * + * @param value Value to set. + */ + void set(double value); + } + + /** + * Gets cardinality of this vector (maximum number of the elements). + * + * @return This vector's cardinality. + */ + int size(); + + /** + * Checks if this implementation should be considered dense so that it explicitly + * represents every value. + * + * @return Density flag. + */ + boolean isDense(); + + /** + * Checks if this implementation should be considered to be iterable in index order in an efficient way. + * This implies that methods like {@link #all()} and {@link #nonZeroes()} return elements + * in ascending order by index. + * + * @return Sequential access flag. + */ + boolean isSequentialAccess(); + + /** + * Creates new copy of this vector. + * + * @return New copy vector. + */ + Vector clone(); + + /** + * Gets iterator over all elements in this vector. + * + * NOTE: implementation can choose to reuse {@link Element} instance so you need to copy it + * if you want to retain it outside of iteration. + * + * @return Iterator. + */ + Iterable all(); + + /** + * Iterates ove all non-zero elements in this vector. + * + * NOTE: implementation can choose to reuse {@link Element} instance so you need to copy it + * if you want to retain it outside of iteration. + * + * @return Iterator. + */ + Iterable nonZeroes(); + + /** + * Gets element at the given index. + * + * NOTE: implementation can choose to reuse {@link Element} instance so you need to copy it + * if you want to retain it outside of iteration. + * + * @param idx Element's index. + * @return Vector's element at the given index. + * @throws IndexException Throw if index is out of bounds. + */ + Element getElement(int idx); + /** * Assigns given value to all elements of this vector. * @@ -95,6 +184,227 @@ public interface Vector { */ Vector map(BiFunction fun, double y); + /** + * Creates new vector containing values from this vector divided by the argument. + * + * @param x Division argument. + * @return New vector. + */ + Vector divide(double x); + + /** + * Gets dot product of two vectors. + * + * @param vec Argument vector. + * @return Dot product of two vectors. + */ + double dot(Vector vec); + + /** + * Gets the value at specified index. + * + * @param idx Vector index. + * @return Vector value. + * @throws IndexException Throw if index is out of bounds. + */ + double get(int idx); + + /** + * Gets the value at specified index without checking for index boundaries. + * + * @param idx Vector index. + * @return Vector value. + */ + double getX(int idx); + + /** + * Creates new empty vector of the same underlying class but of different cardinality. + * + * @param crd Cardinality for new vector. + * @return New vector. + */ + Vector like(int crd); + + /** + * Creates new vector containing element by element difference between this vector and the argument one. + * + * @param vec Argument vector. + * @return New vector. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + Vector minus(Vector vec); + + /** + * Creates new vector containing the normalized (L_2 norm) values of this vector. + * + * @return New vector. + */ + Vector normalize(); + + /** + * Creates new vector containing the normalized (L_power norm) values of this vector. + * See http://en.wikipedia.org/wiki/Lp_space for details. + * + * @param power The power to use. Must be >= 0. May also be {@link Double#POSITIVE_INFINITY}. + * @return New vector {@code x} such that {@code norm(x, power) == 1} + */ + Vector normalize(double power); + + /** + * Creates new vector containing the {@code log(1 + entry) / L_2 norm} values of this vector. + * + * @return New vector. + */ + Vector logNormalize(); + + /** + * Creates new vector with a normalized value calculated as {@code log_power(1 + entry) / L_power norm}. + * + * @param power The power to use. Must be > 1. Cannot be {@link Double#POSITIVE_INFINITY}. + * @return New vector + */ + Vector logNormalize(double power); + + /** + * Gets the k-norm of the vector. See http://en.wikipedia.org/wiki/Lp_space for more details. + * + * @param power The power to use. + * @see #normalize(double) + */ + double norm(double power); + + /** + * Gets minimal element in this vector. + * + * @return Minimal element. + */ + Element minValue(); + + /** + * Gets maximum element in this vector. + * + * @return Maximum element. + */ + double maxValue(); + + /** + * Creates new vector containing sum of each element in this vector and argument. + * + * @param x Argument value. + * @return New vector. + */ + Vector plus(double x); + + /** + * Creates new vector containing element by element sum from both vectors. + * + * @param vec Other argument vector to add. + * @throws CardinalityException Thrown if cardinalities mismatch. + * @return New vector. + */ + Vector plus(Vector vec); + + /** + * Sets value. + * + * @param idx Vector index to set value at. + * @param val Value to set. + * @throws IndexException Throw if index is out of bounds. + * @return This vector. + */ + Vector set(int idx, double val); + + /** + * Sets value without checking for index boundaries. + * + * @param idx Vector index to set value at. + * @param val Value to set. + * @return This vector. + */ + Vector setX(int idx, double val); + + /** + * Increments value at given index without checking for index boundaries. + * + * @param idx Vector index. + * @param val Increment value. + */ + void incrementX(int idx, double val); + + /** + * Gets number of non-default elements in this vector. + * + * @return Number of non-default elements in this vector. + */ + int nonDefaultElements(); + + /** + * Gets number of non-zero elements in this vector. + * + * @return Number of non-zero elements in this vector. + */ + int nonZeroElements(); + + Vector times(double x); + Vector times(Vector x); + Vector viewPart(int offset, int length); + + /** + * Gets the sum of all elements in this vector. + * + * @return Vector's sum + */ + double sum(); + + /** + * Gets the cross product of this vector and the other vector. + * + * @param vec Second vector. + * @return New matrix as a cross product of two vectors. + */ + Matrix cross(Vector vec); + + /** + * Folds this vector into a single value. + * + * @param foldFun Folding function that takes two parameters: accumulator and the current value. + * @param mapFun Mapping function that is called on each vector element before its passed to the accumulator + * (as its second parameter). + * @param Type of the folded value. + * @return Folded value of this vector. + */ + T foldMap(BiFunction foldFun, DoubleFunction mapFun); + + /** + * Gets the sum of squares of all elements in this vector. + * + * @return Length squared value. + */ + double getLengthSquared(); + + /** + * Get the square of the distance between this vector and the argument vector. + * + * @param vec Another vector. + * @return Distance squared. + * @throws CardinalityException Thrown if cardinalities mismatch. + */ + double getDistanceSquared(Vector vec); + + /** + * Gets an estimate of the cost *in number of ops* it takes to lookup a random element in this vector. + * + * @return Lookup cost in number of ops. + */ + double getLookupCost(); + + /** + * Checks if adding a non-zero element to this vector is done in a constant time. + * + * @return Add constant time flag. + */ + boolean isAddConstantTime(); + /** * Gets optional cluster group this vector is stored on. In case of local JVM storage it may * return an empty option or a cluster group consisting of only the local Ignite node. diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/alert/VisorAlertCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/alert/VisorAlertCommand.scala index fb60a1e5bbb47..991dff9de5c5b 100644 --- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/alert/VisorAlertCommand.scala +++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/alert/VisorAlertCommand.scala @@ -564,8 +564,8 @@ class VisorAlertCommand extends VisorConsoleCommand { val sum = new VisorTextTable() - val firstSnd = (-1L /: stats.values)((b, a) => if (b == -1) a.firstSnd else math.min(b, a.firstSnd)) - val lastSnd = (0L /: stats.values)((b, a) => math.max(b, a.lastSnd)) + val firstSnd = (-1L /: stats.values)((b, a) => if (b == -1) a.firstSnd else scala.math.min(b, a.firstSnd)) + val lastSnd = (0L /: stats.values)((b, a) => scala.math.max(b, a.lastSnd)) sum += ("Total alerts", alerts.size) sum += ("Total sends", (0 /: stats.values)((b, a) => b + a.cnt)) diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/gc/VisorGcCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/gc/VisorGcCommand.scala index 990fd00fd5b62..aa4e2cf984018 100644 --- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/gc/VisorGcCommand.scala +++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/gc/VisorGcCommand.scala @@ -109,7 +109,7 @@ class VisorGcCommand extends VisorConsoleCommand { val sign = if (roundHa > roundHb) "+" else "" - val deltaPercent = math.round(roundHa * 100d / roundHb - 100) + val deltaPercent = scala.math.round(roundHa * 100d / roundHb - 100) t += (nodeId8(nid), roundHb + "mb", roundHa + "mb", sign + deltaPercent + "%") } diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/tasks/VisorTasksCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/tasks/VisorTasksCommand.scala index 0468e80c7f19c..f24abccdf8a1b 100644 --- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/tasks/VisorTasksCommand.scala +++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/tasks/VisorTasksCommand.scala @@ -222,7 +222,7 @@ private case class VisorTask( lazy val avgNodes: Int = (0 /: execs)((b, a) => a.nodeIds.size + b) / execs.size - override def equals(r: Any) = + override def equals(r: Any): Boolean = if (this eq r.asInstanceOf[AnyRef]) true else if (r == null || !r.isInstanceOf[VisorTask]) @@ -230,7 +230,7 @@ private case class VisorTask( else r.asInstanceOf[VisorTask].taskName == taskName - override def hashCode() = + override def hashCode(): Int = taskName.hashCode() } @@ -420,7 +420,7 @@ class VisorTasksCommand extends VisorConsoleCommand { else if (hasArgName("n", argLst)) { val n = argValue("n", argLst) - if (!n.isDefined) + if (n.isEmpty) scold("Invalid arguments: " + args) else task(n.get, hasArgFlag("r", argLst), hasArgFlag("a", argLst)) @@ -428,7 +428,7 @@ class VisorTasksCommand extends VisorConsoleCommand { else if (hasArgName("e", argLst)) { val s = argValue("e", argLst) - if (!s.isDefined) + if (s.isEmpty) scold("Invalid arguments: " + args) else exec(s.get, hasArgFlag("r", argLst)) @@ -550,8 +550,8 @@ class VisorTasksCommand extends VisorConsoleCommand { s.evts = s.evts :+ te s.nodeIds = s.nodeIds + te.nid() - s.startTs = math.min(s.startTs, te.timestamp()) - s.endTs = math.max(s.endTs, te.timestamp()) + s.startTs = scala.math.min(s.startTs, te.timestamp()) + s.endTs = scala.math.max(s.endTs, te.timestamp()) te.typeId() match { case EVT_TASK_STARTED => @@ -585,8 +585,8 @@ class VisorTasksCommand extends VisorConsoleCommand { s.evts = s.evts :+ je s.nodeIds = s.nodeIds + je.nid() - s.startTs = math.min(s.startTs, je.timestamp()) - s.endTs = math.max(s.endTs, je.timestamp()) + s.startTs = scala.math.min(s.startTs, je.timestamp()) + s.endTs = scala.math.max(s.endTs, je.timestamp()) case _ => } @@ -807,8 +807,6 @@ class VisorTasksCommand extends VisorConsoleCommand { assert(taskName != null) try { - val prj = ignite.cluster.forRemotes() - val evts = executeMulti(classOf[VisorNodeEventsCollectorTask], VisorNodeEventsCollectorTaskArg.createTasksArg(null, taskName, null)) diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala index 8673b9cc637f9..784fe047f379b 100644 --- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala +++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/visor.scala @@ -2678,7 +2678,7 @@ object visor extends VisorTag { var dec = BigDecimal.valueOf(0L) - for (i <- octets.indices) dec += octets(i).toLong * math.pow(256, octets.length - 1 - i).toLong + for (i <- octets.indices) dec += octets(i).toLong * scala.math.pow(256, octets.length - 1 - i).toLong dec } From d54f4404748ed02d735dff618e6d86d8a1370a38 Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Sat, 21 Jan 2017 19:48:24 -0800 Subject: [PATCH 011/540] WIP. --- .../math/impls/DenseLocalOnHeapMatrix.java | 2 + .../math/impls/DenseLocalOnHeapVector.java | 240 +++++++++++++++++- 2 files changed, 240 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapMatrix.java b/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapMatrix.java index f1106a64ca412..b3f46d38eb7ff 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapMatrix.java +++ b/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapMatrix.java @@ -25,6 +25,8 @@ import java.util.function.*; /** + * Basic implementation for matrix. + * * This is a trivial implementation for matrix assuming dense logic, local on-heap JVM storage * based on double[][] array. It is only suitable for data sets where * local, non-distributed execution is satisfactory and on-heap JVM storage is enough diff --git a/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapVector.java b/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapVector.java index 03cf747df1d81..503cfcf74993e 100644 --- a/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapVector.java +++ b/modules/core/src/main/java/org/apache/ignite/math/impls/DenseLocalOnHeapVector.java @@ -17,8 +17,244 @@ package org.apache.ignite.math.impls; +import org.apache.ignite.cluster.*; +import org.apache.ignite.lang.*; +import org.apache.ignite.math.*; +import org.apache.ignite.math.Vector; +import java.util.*; +import java.util.function.*; + /** - * TODO: add description. + * Basic implementation for vector. + * + * This is a trivial implementation for vector assuming dense logic, local on-heap JVM storage + * based on double[] array. It is only suitable for data sets where + * local, non-distributed execution is satisfactory and on-heap JVM storage is enough + * to keep the entire data set. */ -public class DenseLocalOnHeapVector { +public class DenseLocalOnHeapVector implements Vector { + @Override + public int size() { + return 0; // TODO + } + + @Override + public boolean isDense() { + return false; // TODO + } + + @Override + public boolean isSequentialAccess() { + return false; // TODO + } + + @Override + public Vector clone() { + return null; // TODO + } + + @Override + public Iterable all() { + return null; // TODO + } + + @Override + public Iterable nonZeroes() { + return null; // TODO + } + + @Override + public Element getElement(int idx) { + return null; // TODO + } + + @Override + public Vector assign(double val) { + return null; // TODO + } + + @Override + public Vector assign(double[] vals) { + return null; // TODO + } + + @Override + public Vector assign(Vector vec) { + return null; // TODO + } + + @Override + public Vector map(DoubleFunction fun) { + return null; // TODO + } + + @Override + public Vector map(Vector vec, DoubleFunction fun) { + return null; // TODO + } + + @Override + public Vector map(BiFunction fun, double y) { + return null; // TODO + } + + @Override + public Vector divide(double x) { + return null; // TODO + } + + @Override + public double dot(Vector vec) { + return 0; // TODO + } + + @Override + public double get(int idx) { + return 0; // TODO + } + + @Override + public double getX(int idx) { + return 0; // TODO + } + + @Override + public Vector like(int crd) { + return null; // TODO + } + + @Override + public Vector minus(Vector vec) { + return null; // TODO + } + + @Override + public Vector normalize() { + return null; // TODO + } + + @Override + public Vector normalize(double power) { + return null; // TODO + } + + @Override + public Vector logNormalize() { + return null; // TODO + } + + @Override + public Vector logNormalize(double power) { + return null; // TODO + } + + @Override + public double norm(double power) { + return 0; // TODO + } + + @Override + public Element minValue() { + return null; // TODO + } + + @Override + public double maxValue() { + return 0; // TODO + } + + @Override + public Vector plus(double x) { + return null; // TODO + } + + @Override + public Vector plus(Vector vec) { + return null; // TODO + } + + @Override + public Vector set(int idx, double val) { + return null; // TODO + } + + @Override + public Vector setX(int idx, double val) { + return null; // TODO + } + + @Override + public void incrementX(int idx, double val) { + // TODO + } + + @Override + public int nonDefaultElements() { + return 0; // TODO + } + + @Override + public int nonZeroElements() { + return 0; // TODO + } + + @Override + public Vector times(double x) { + return null; // TODO + } + + @Override + public Vector times(Vector x) { + return null; // TODO + } + + @Override + public Vector viewPart(int offset, int length) { + return null; // TODO + } + + @Override + public double sum() { + return 0; // TODO + } + + @Override + public Matrix cross(Vector vec) { + return null; // TODO + } + + @Override + public T foldMap(BiFunction foldFun, DoubleFunction mapFun) { + return null; // TODO + } + + @Override + public double getLengthSquared() { + return 0; // TODO + } + + @Override + public double getDistanceSquared(Vector vec) { + return 0; // TODO + } + + @Override + public double getLookupCost() { + return 0; // TODO + } + + @Override + public boolean isAddConstantTime() { + return false; // TODO + } + + @Override + public Optional clusterGroup() { + return null; // TODO + } + + @Override + public IgniteUuid guid() { + return null; // TODO + } } From e8cfbca52059a09a9246dc4f196b145c430f8bec Mon Sep 17 00:00:00 2001 From: Nikita Ivanov Date: Mon, 23 Jan 2017 15:22:31 -0800 Subject: [PATCH 012/540] WIP (doesn't compile for now). --- .../main/java/org/apache/ignite/Ignite.java | 35 ++++---- .../java/org/apache/ignite/IgniteMath.java | 66 +++++++++++++++ .../ignite/math/DefaultMathProvider.java | 83 +++++++++++++++++++ .../apache/ignite/math/IgniteMathImpl.java | 80 ++++++++++++++++++ .../org/apache/ignite/math/MathProvider.java | 49 +++++++++++ .../math/impls/DenseLocalOnHeapMatrix.java | 25 ++++++ .../math/impls/DenseLocalOnHeapVector.java | 32 +++++++ 7 files changed, 354 insertions(+), 16 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/math/DefaultMathProvider.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/IgniteMathImpl.java create mode 100644 modules/core/src/main/java/org/apache/ignite/math/MathProvider.java diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java index 0de08d56584a8..ae2a108673ec1 100644 --- a/modules/core/src/main/java/org/apache/ignite/Ignite.java +++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java @@ -17,22 +17,17 @@ package org.apache.ignite; -import java.util.Collection; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import javax.cache.CacheException; -import org.apache.ignite.cache.CacheMode; -import org.apache.ignite.cache.affinity.Affinity; -import org.apache.ignite.cluster.ClusterGroup; -import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.CollectionConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; -import org.apache.ignite.configuration.NearCacheConfiguration; -import org.apache.ignite.internal.util.typedef.G; -import org.apache.ignite.lang.IgniteProductVersion; -import org.apache.ignite.plugin.IgnitePlugin; -import org.apache.ignite.plugin.PluginNotFoundException; -import org.jetbrains.annotations.Nullable; +import org.apache.ignite.cache.*; +import org.apache.ignite.cache.affinity.*; +import org.apache.ignite.cluster.*; +import org.apache.ignite.configuration.*; +import org.apache.ignite.internal.util.typedef.*; +import org.apache.ignite.lang.*; +import org.apache.ignite.plugin.*; +import org.jetbrains.annotations.*; +import javax.cache.*; +import java.util.*; +import java.util.concurrent.*; /** * Main entry-point for all Ignite APIs. @@ -44,6 +39,7 @@ * Ignite provides the following functionality: *