From 412e2e6b8fcd208ded929df100d2ee80ee7a0281 Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Fri, 17 Oct 2014 18:53:13 +0900 Subject: [PATCH 01/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- .../tajo/validation/AbstractValidator.java | 60 ++++ .../tajo/validation/ConstraintViolation.java | 47 +++ .../ConstraintViolationException.java | 51 +++ .../tajo/validation/GroupValidator.java | 49 +++ .../tajo/validation/LengthValidator.java | 57 ++++ .../apache/tajo/validation/MaxValidator.java | 79 +++++ .../apache/tajo/validation/MinValidator.java | 84 +++++ .../tajo/validation/NotNullValidator.java | 47 +++ .../tajo/validation/PatternValidator.java | 58 ++++ .../tajo/validation/RangeValidator.java | 53 +++ .../org/apache/tajo/validation/Validator.java | 29 ++ .../apache/tajo/validation/Validators.java | 53 +++ .../tajo/conf/TestValidatorsForTajoConf.java | 26 ++ .../tajo/validation/TestValidators.java | 302 ++++++++++++++++++ 14 files changed, 995 insertions(+) create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/AbstractValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolation.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/GroupValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/LengthValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/MinValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/NotNullValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/RangeValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/Validator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/Validators.java create mode 100644 tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java create mode 100644 tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/AbstractValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/AbstractValidator.java new file mode 100644 index 0000000000..0f7082bc01 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/AbstractValidator.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; + +import org.apache.tajo.util.TUtil; + +public abstract class AbstractValidator implements Validator { + + protected abstract String getErrorMessage(T object); + protected abstract boolean validateInternal(T object); + protected abstract Collection getDependantValidators(); + + @Override + public Collection validate(T object) { + Collection violations = TUtil.newHashSet(); + + if (!validateInternal(object)) { + ConstraintViolation violation = new ConstraintViolation(); + violation.setMessage(getErrorMessage(object)); + violation.setValidatorClazz(this.getClass()); + violations.add(violation); + } + + for (Validator dependantValidator: getDependantValidators()) { + violations.addAll(dependantValidator.validate(object)); + } + + return violations; + } + + @Override + public void validate(T object, boolean generateThrow) { + Collection violations = validate(object); + + if (violations.size() > 0) { + if (generateThrow) { + throw new ConstraintViolationException(violations); + } + } + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolation.java b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolation.java new file mode 100644 index 0000000000..8d84aa7636 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolation.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +public class ConstraintViolation { + + private String message; + private Class validatorClazz; + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public Class getValidatorClazz() { + return validatorClazz; + } + + public void setValidatorClazz(Class validatorClazz) { + this.validatorClazz = validatorClazz; + } + + @Override + public String toString() { + return "ConstraintViolation [message=" + message + ", validatorClazz=" + validatorClazz + "]"; + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java new file mode 100644 index 0000000000..664ec09771 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; + +public class ConstraintViolationException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final Collection violations; + + public ConstraintViolationException(Collection violations) { + this.violations = violations; + } + + public Collection getViolations() { + return violations; + } + + @Override + public String getMessage() { + if (violations != null) { + String errorMessage = "ConstraintViolationException ["; + for (ConstraintViolation violation: violations) { + errorMessage += violation.getMessage() + ","; + } + errorMessage += "]"; + return errorMessage; + } else { + return super.getMessage(); + } + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/GroupValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/GroupValidator.java new file mode 100644 index 0000000000..db2ba51e1e --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/GroupValidator.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; + +public class GroupValidator extends AbstractValidator { + + private final Collection dependants; + + public GroupValidator(Collection validators) { + if (validators.size() == 0) { + throw new IllegalArgumentException("Needs at least 1 or more validators."); + } + this.dependants = validators; + } + + @Override + protected String getErrorMessage(T object) { + return ""; + } + + @Override + protected boolean validateInternal(T object) { + return true; + } + + @Override + protected Collection getDependantValidators() { + return dependants; + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/LengthValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/LengthValidator.java new file mode 100644 index 0000000000..e53634b167 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/LengthValidator.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class LengthValidator extends AbstractValidator { + + private final int maxLength; + + public LengthValidator(int maxLen) { + this.maxLength = maxLen; + } + + @Override + protected String getErrorMessage(T object) { + return "Length of " + object + " is greater than " + maxLength; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof CharSequence) { + result = ((CharSequence)object).length() <= maxLength; + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java new file mode 100644 index 0000000000..63c2352b09 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collection; +import java.util.Collections; + +import org.apache.commons.lang.math.NumberUtils; + +public class MaxValidator extends AbstractValidator { + + private final String maxValue; + + public MaxValidator(String maxValue) { + if (!NumberUtils.isNumber(maxValue)) { + throw new IllegalArgumentException(maxValue + " is not a Java number."); + } + + this.maxValue = maxValue; + } + + @Override + protected String getErrorMessage(T object) { + return object + " should be less than " + maxValue; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if ((object instanceof Byte) || (object instanceof Short) || (object instanceof Integer)) { + Integer objInteger = Integer.decode(object.toString()); + Integer maxInteger = Integer.decode(maxValue); + result = objInteger.compareTo(maxInteger) <= 0; + } else if (object instanceof Long) { + Long objLong = Long.decode(object.toString()); + Long maxLong = Long.decode(maxValue); + result = objLong.compareTo(maxLong) <= 0; + } else if (object instanceof BigInteger) { + BigInteger objInteger = (BigInteger) object; + BigInteger maxInteger = new BigInteger(maxValue); + result = objInteger.compareTo(maxInteger) <= 0; + } else if (object instanceof BigDecimal) { + BigDecimal objDecimal = (BigDecimal) object; + BigDecimal maxDecimal = new BigDecimal(maxValue); + result = objDecimal.compareTo(maxDecimal) <= 0; + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/MinValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/MinValidator.java new file mode 100644 index 0000000000..431fe9b26f --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/MinValidator.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collection; +import java.util.Collections; + +import org.apache.commons.lang.math.NumberUtils; + +public class MinValidator extends AbstractValidator { + + private final String minValue; + + public MinValidator(String minValue) { + if (!NumberUtils.isNumber(minValue)) { + throw new IllegalArgumentException(minValue + " is not a Java Number."); + } + + this.minValue = minValue; + } + + @Override + protected String getErrorMessage(T object) { + return object + " should be greater than " + minValue; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if ((object instanceof Byte) || (object instanceof Short) || (object instanceof Integer)) { + Integer objInteger = Integer.decode(object.toString()); + Integer minInteger = Integer.decode(minValue); + result = objInteger.compareTo(minInteger) >= 0; + } else if (object instanceof Long) { + Long objLong = Long.decode(object.toString()); + Long minLong = Long.decode(minValue); + result = objLong.compareTo(minLong) >= 0; + } else if ((object instanceof Float) || (object instanceof Double)) { + Double objDouble = Double.valueOf(object.toString()); + Double minDouble = Double.valueOf(minValue); + result = objDouble.compareTo(minDouble) >= 0; + } else if (object instanceof BigInteger) { + BigInteger objInteger = (BigInteger) object; + BigInteger minInteger = new BigInteger(minValue); + result = objInteger.compareTo(minInteger) >= 0; + } else if (object instanceof BigDecimal) { + BigDecimal objDecimal = (BigDecimal) object; + BigDecimal minDecimal = new BigDecimal(minValue); + result = objDecimal.compareTo(minDecimal) >= 0; + } + } + else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/NotNullValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/NotNullValidator.java new file mode 100644 index 0000000000..d0ace1ffa3 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/NotNullValidator.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class NotNullValidator extends AbstractValidator { + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + + @Override + protected String getErrorMessage(T object) { + return "Object is null"; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + result = true; + } + + return result; + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java new file mode 100644 index 0000000000..0c3a99a2a2 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; +import java.util.regex.Pattern; + +public class PatternValidator extends AbstractValidator { + + private final Pattern pattern; + + public PatternValidator(String patternString) { + pattern = Pattern.compile(patternString); + } + + @Override + protected String getErrorMessage(T object) { + return object + " does not match to the " + pattern.pattern(); + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof CharSequence) { + result = pattern.matcher((CharSequence) object).find(); + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/RangeValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/RangeValidator.java new file mode 100644 index 0000000000..ae6fd76a06 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/RangeValidator.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class RangeValidator extends AbstractValidator { + + private final String minValue; + private final AbstractValidator minValidator; + private final String maxValue; + private final AbstractValidator maxValidator; + + public RangeValidator(String minValue, String maxValue) { + this.minValue = minValue; + this.minValidator = new MinValidator(minValue); + this.maxValue = maxValue; + this.maxValidator = new MaxValidator(maxValue); + } + + @Override + protected String getErrorMessage(T object) { + return object + " is not a range of " + minValue + " and " + maxValue; + } + + @Override + protected boolean validateInternal(T object) { + return this.minValidator.validateInternal(object) & this.maxValidator.validateInternal(object); + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validator.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validator.java new file mode 100644 index 0000000000..8f16fda2aa --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validator.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; + +public interface Validator { + + public Collection validate(T object); + + public void validate(T object, boolean generateThrow); + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java new file mode 100644 index 0000000000..26056ad9ce --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import org.apache.tajo.util.TUtil; + +public class Validators { + + public static Validator groups(Validator...validators) { + return new GroupValidator(TUtil.newHashSet(validators)); + } + + public static Validator length(int maxLength) { + return new LengthValidator(maxLength); + } + + public static Validator max(String maxValue) { + return new MaxValidator(maxValue); + } + + public static Validator min(String minValue) { + return new MinValidator(minValue); + } + + public static Validator notNull() { + return new NotNullValidator(); + } + + public static Validator patternMatch(String regex) { + return new PatternValidator(regex); + } + + public static Validator range(String minValue, String maxValue) { + return new RangeValidator(minValue, maxValue); + } + +} diff --git a/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java b/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java new file mode 100644 index 0000000000..db5aae77c0 --- /dev/null +++ b/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java @@ -0,0 +1,26 @@ +package org.apache.tajo.conf; + +import static org.junit.Assert.*; + +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.validation.ConstraintViolationException; +import org.junit.Test; + +public class TestValidatorsForTajoConf { + + @Test(expected=ConstraintViolationException.class) + public void testPathValidatorsForTajoConf_setVar() { + TajoConf conf = new TajoConf(); + + TajoConf.setVar(conf, ConfVars.ROOT_DIR, "Invalid path"); + fail(); + } + + @Test + public void testValidPathForTajoConf() { + TajoConf conf = new TajoConf(); + + TajoConf.setVar(conf, ConfVars.ROOT_DIR, "file:///tmp/tajo-${user.name}/"); + } + +} diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java new file mode 100644 index 0000000000..facb8c89c5 --- /dev/null +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -0,0 +1,302 @@ +package org.apache.tajo.validation; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; +import java.util.Collection; +import java.util.Random; +import java.util.UUID; + +import org.apache.tajo.util.TUtil; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.junit.Test; + +public class TestValidators { + + private class ValidatorClazzMatcher extends org.hamcrest.TypeSafeDiagnosingMatcher { + + private final org.hamcrest.Matcher> matcher; + + public ValidatorClazzMatcher(Matcher> matcher) { + this.matcher = matcher; + } + + @Override + public void describeTo(Description description) { + description + .appendText("validatorClazz property of ConstraintViolation class containing ") + .appendDescriptionOf(matcher); + } + + @Override + protected boolean matchesSafely(ConstraintViolation item, Description mismatchDescription) { + if (matcher.matches(item.getValidatorClazz())) { + return true; + } + matcher.describeMismatch(item, mismatchDescription); + return false; + } + + } + + private org.hamcrest.Matcher + hasAClass(Matcher> matcher) { + return new ValidatorClazzMatcher(matcher); + } + + @Test + public void testNotNullValidator() { + Object testValue = null; + + assertThat(new NotNullValidator().validateInternal(testValue), is(false)); + assertThat(new NotNullValidator().validate(testValue).size(), is(1)); + assertThat(new NotNullValidator().validate(testValue), + hasItem(new ValidatorClazzMatcher(equalTo(NotNullValidator.class)))); + } + + @Test + public void testMinValidator() { + byte byteValue; + short shortValue; + int intValue; + long longValue; + float floatValue; + double doubleValue; + BigInteger bigIntegerValue; + BigDecimal bigDecimalValue; + + byteValue = 2; + assertThat(new MinValidator(Byte.toString(Byte.MIN_VALUE)).validateInternal(byteValue), is(true)); + assertThat(new MinValidator(Byte.toString(Byte.MIN_VALUE)).validate(byteValue).size(), is(0)); + assertThat(new MinValidator(Byte.toString(Byte.MAX_VALUE)).validateInternal(byteValue), is(false)); + assertThat(new MinValidator(Byte.toString(Byte.MAX_VALUE)).validate(byteValue).size(), is(1)); + assertThat(new MinValidator(Byte.toString(Byte.MAX_VALUE)).validate(byteValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + shortValue = 3; + assertThat(new MinValidator(Short.toString(Short.MIN_VALUE)).validateInternal(shortValue), is(true)); + assertThat(new MinValidator(Short.toString(Short.MIN_VALUE)).validate(shortValue).size(), is(0)); + assertThat(new MinValidator(Short.toString(Short.MAX_VALUE)).validateInternal(shortValue), is(false)); + assertThat(new MinValidator(Short.toString(Short.MAX_VALUE)).validate(shortValue).size(), is(1)); + assertThat(new MinValidator(Short.toString(Short.MAX_VALUE)).validate(shortValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + intValue = 4; + assertThat(new MinValidator(Integer.toString(Integer.MIN_VALUE)).validateInternal(intValue), is(true)); + assertThat(new MinValidator(Integer.toString(Integer.MIN_VALUE)).validate(intValue).size(), is(0)); + assertThat(new MinValidator(Integer.toString(Integer.MAX_VALUE)).validateInternal(intValue), is(false)); + assertThat(new MinValidator(Integer.toString(Integer.MAX_VALUE)).validate(intValue).size(), is(1)); + assertThat(new MinValidator(Integer.toString(Integer.MAX_VALUE)).validate(intValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + longValue = 5; + assertThat(new MinValidator(Long.toString(Long.MIN_VALUE)).validateInternal(longValue), is(true)); + assertThat(new MinValidator(Long.toString(Long.MIN_VALUE)).validate(longValue).size(), is(0)); + assertThat(new MinValidator(Long.toString(Long.MAX_VALUE)).validateInternal(longValue), is(false)); + assertThat(new MinValidator(Long.toString(Long.MAX_VALUE)).validate(longValue).size(), is(1)); + assertThat(new MinValidator(Long.toString(Long.MAX_VALUE)).validate(longValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + floatValue = 4.7f; + assertThat(new MinValidator(Float.toString(Float.MIN_VALUE)).validateInternal(floatValue), is(true)); + assertThat(new MinValidator(Float.toString(Float.MIN_VALUE)).validate(floatValue).size(), is(0)); + assertThat(new MinValidator(Float.toString(Float.MAX_VALUE)).validateInternal(floatValue), is(false)); + assertThat(new MinValidator(Float.toString(Float.MAX_VALUE)).validate(floatValue).size(), is(1)); + assertThat(new MinValidator(Float.toString(Float.MAX_VALUE)).validate(floatValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + doubleValue = 7.5e10; + assertThat(new MinValidator(Double.toString(Double.MIN_VALUE)).validateInternal(doubleValue), is(true)); + assertThat(new MinValidator(Double.toString(Double.MIN_VALUE)).validate(doubleValue).size(), is(0)); + assertThat(new MinValidator(Double.toString(Double.MAX_VALUE)).validateInternal(doubleValue), is(false)); + assertThat(new MinValidator(Double.toString(Double.MAX_VALUE)).validate(doubleValue).size(), is(1)); + assertThat(new MinValidator(Double.toString(Double.MAX_VALUE)).validate(doubleValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + bigIntegerValue = new BigInteger(10, new Random()); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validateInternal(bigIntegerValue), is(true)); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validate(bigIntegerValue).size(), is(0)); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validateInternal(bigIntegerValue), is(false)); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validate(bigIntegerValue).size(), is(1)); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validate(bigIntegerValue), hasItem(hasAClass(equalTo(MinValidator.class)))); + + bigDecimalValue = new BigDecimal(new BigInteger(10, new Random()), MathContext.DECIMAL64); + assertThat(new MinValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validateInternal(bigDecimalValue), is(true)); + assertThat(new MinValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validate(bigDecimalValue).size(), is(0)); + assertThat(new MinValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validateInternal(bigDecimalValue), is(false)); + assertThat(new MinValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validate(bigDecimalValue).size(), is(1)); + assertThat(new MinValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validate(bigDecimalValue), hasItem(hasAClass(equalTo(MinValidator.class)))); + } + + @Test + public void testMaxValidator() { + byte byteValue; + short shortValue; + int intValue; + long longValue; + float floatValue; + double doubleValue; + BigInteger bigIntegerValue; + BigDecimal bigDecimalValue; + + byteValue = 2; + assertThat(new MaxValidator(Byte.toString(Byte.MAX_VALUE)).validateInternal(byteValue), is(true)); + assertThat(new MaxValidator(Byte.toString(Byte.MAX_VALUE)).validate(byteValue).size(), is(0)); + assertThat(new MaxValidator(Byte.toString(Byte.MIN_VALUE)).validateInternal(byteValue), is(false)); + assertThat(new MaxValidator(Byte.toString(Byte.MIN_VALUE)).validate(byteValue).size(), is(1)); + assertThat(new MaxValidator(Byte.toString(Byte.MIN_VALUE)).validate(byteValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + shortValue = 3; + assertThat(new MaxValidator(Short.toString(Short.MAX_VALUE)).validateInternal(shortValue), is(true)); + assertThat(new MaxValidator(Short.toString(Short.MAX_VALUE)).validate(shortValue).size(), is(0)); + assertThat(new MaxValidator(Short.toString(Short.MIN_VALUE)).validateInternal(shortValue), is(false)); + assertThat(new MaxValidator(Short.toString(Short.MIN_VALUE)).validate(shortValue).size(), is(1)); + assertThat(new MaxValidator(Short.toString(Short.MIN_VALUE)).validate(shortValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + intValue = 4; + assertThat(new MaxValidator(Integer.toString(Integer.MAX_VALUE)).validateInternal(intValue), is(true)); + assertThat(new MaxValidator(Integer.toString(Integer.MAX_VALUE)).validate(intValue).size(), is(0)); + assertThat(new MaxValidator(Integer.toString(Integer.MIN_VALUE)).validateInternal(intValue), is(false)); + assertThat(new MaxValidator(Integer.toString(Integer.MIN_VALUE)).validate(intValue).size(), is(1)); + assertThat(new MaxValidator(Integer.toString(Integer.MIN_VALUE)).validate(intValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + longValue = 5; + assertThat(new MaxValidator(Long.toString(Long.MAX_VALUE)).validateInternal(longValue), is(true)); + assertThat(new MaxValidator(Long.toString(Long.MAX_VALUE)).validate(longValue).size(), is(0)); + assertThat(new MaxValidator(Long.toString(Long.MIN_VALUE)).validateInternal(longValue), is(false)); + assertThat(new MaxValidator(Long.toString(Long.MIN_VALUE)).validate(longValue).size(), is(1)); + assertThat(new MaxValidator(Long.toString(Long.MIN_VALUE)).validate(longValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + floatValue = 4.7f; + assertThat(new MaxValidator(Float.toString(Float.MAX_VALUE)).validateInternal(floatValue), is(true)); + assertThat(new MaxValidator(Float.toString(Float.MAX_VALUE)).validate(floatValue).size(), is(0)); + assertThat(new MaxValidator(Float.toString(Float.MIN_VALUE)).validateInternal(floatValue), is(false)); + assertThat(new MaxValidator(Float.toString(Float.MIN_VALUE)).validate(floatValue).size(), is(1)); + assertThat(new MaxValidator(Float.toString(Float.MIN_VALUE)).validate(floatValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + doubleValue = 7.5e10; + assertThat(new MaxValidator(Double.toString(Double.MAX_VALUE)).validateInternal(doubleValue), is(true)); + assertThat(new MaxValidator(Double.toString(Double.MAX_VALUE)).validate(doubleValue).size(), is(0)); + assertThat(new MaxValidator(Double.toString(Double.MIN_VALUE)).validateInternal(doubleValue), is(false)); + assertThat(new MaxValidator(Double.toString(Double.MIN_VALUE)).validate(doubleValue).size(), is(1)); + assertThat(new MaxValidator(Double.toString(Double.MIN_VALUE)).validate(doubleValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + bigIntegerValue = new BigInteger(10, new Random()); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validateInternal(bigIntegerValue), is(true)); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validate(bigIntegerValue).size(), is(0)); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validateInternal(bigIntegerValue), is(false)); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validate(bigIntegerValue).size(), is(1)); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validate(bigIntegerValue), hasItem(hasAClass(equalTo(MaxValidator.class)))); + + bigDecimalValue = new BigDecimal(new BigInteger(10, new Random()), MathContext.DECIMAL64); + assertThat(new MaxValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validateInternal(bigDecimalValue), is(true)); + assertThat(new MaxValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validate(bigDecimalValue).size(), is(0)); + assertThat(new MaxValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validateInternal(bigDecimalValue), is(false)); + assertThat(new MaxValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validate(bigDecimalValue).size(), is(1)); + assertThat(new MaxValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validate(bigDecimalValue), hasItem(hasAClass(equalTo(MaxValidator.class)))); + } + + @Test + public void testPatternValidator() { + String schemeString = "http://tajo.apache.org"; + assertThat(new PatternValidator("^([a-zA-Z])+://").validateInternal(schemeString), is(true)); + assertThat(new PatternValidator("^([a-zA-Z])+://").validate(schemeString).size(), is(0)); + assertThat(new PatternValidator("([a-zA-Z])+://$").validateInternal(schemeString), is(false)); + assertThat(new PatternValidator("([a-zA-Z])+://$").validate(schemeString).size(), is(1)); + assertThat(new PatternValidator("([a-zA-Z])+://$").validate(schemeString), + hasItem(hasAClass(equalTo(PatternValidator.class)))); + } + + @Test + public void testLengthValidator() { + String shortString = "12345"; + String longString = UUID.randomUUID().toString(); + + assertThat(new LengthValidator(10).validateInternal(shortString), is(true)); + assertThat(new LengthValidator(10).validate(shortString).size(), is(0)); + assertThat(new LengthValidator(3).validateInternal(shortString), is(false)); + assertThat(new LengthValidator(3).validate(shortString).size(), is(1)); + assertThat(new LengthValidator(3).validate(shortString), + hasItem(hasAClass(equalTo(LengthValidator.class)))); + + assertThat(new LengthValidator(40).validateInternal(longString), is(true)); + assertThat(new LengthValidator(40).validate(longString).size(), is(0)); + assertThat(new LengthValidator(10).validateInternal(longString), is(false)); + assertThat(new LengthValidator(10).validate(longString).size(), is(1)); + assertThat(new LengthValidator(10).validate(longString), + hasItem(hasAClass(equalTo(LengthValidator.class)))); + } + + @Test + public void testGroupValidator() { + String httpUrl = "http://tajo.apache.org"; + Collection validators = null; + + validators = TUtil.newHashSet(); + validators.add(new PatternValidator("^[a-zA-Z]+://")); + validators.add(new LengthValidator(255)); + assertThat(new GroupValidator(validators).validate(httpUrl).size(), is(0)); + assertThat(new GroupValidator(validators).validate("tajo").size(), is(2)); + assertThat(new GroupValidator(validators).validate("tajo"), + hasItem(hasAClass(equalTo(PatternValidator.class)))); + + validators = TUtil.newHashSet(); + validators.add(new PatternValidator("^[a-zA-Z]+://")); + validators.add(new LengthValidator(7)); + assertThat(new GroupValidator(validators).validate(httpUrl).size(), is(1)); + assertThat(new GroupValidator(validators).validate(httpUrl), + hasItem(hasAClass(equalTo(LengthValidator.class)))); + } + + @Test + public void testRangeValidator() { + int intValue = 5; + + assertThat(new RangeValidator("0", Integer.toString(Integer.MAX_VALUE)).validateInternal(intValue), + is(true)); + assertThat(new RangeValidator("1024", "2048").validateInternal(intValue), is(false)); + } + + @Test(expected=ConstraintViolationException.class) + public void testExceptionThrow() { + Collection validators = null; + + validators = TUtil.newHashSet(); + validators.add(new PatternValidator("^[a-zA-Z]+://")); + validators.add(new LengthValidator(255)); + new GroupValidator(validators).validate("tajo", true); + + fail(); + } + +} From 76720f853dd22b245207d2de8e49fcc9554562f5 Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Mon, 20 Oct 2014 16:13:01 +0900 Subject: [PATCH 02/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- tajo-common/pom.xml | 5 + .../main/java/org/apache/tajo/ConfigKey.java | 6 + .../java/org/apache/tajo/OverridableConf.java | 15 ++ .../java/org/apache/tajo/SessionVars.java | 21 +++ .../java/org/apache/tajo/conf/TajoConf.java | 154 +++++++++++++++--- .../java/org/apache/tajo/util/NumberUtil.java | 21 +++ .../apache/tajo/validation/MaxValidator.java | 4 + .../tajo/validation/PatternValidator.java | 7 +- .../apache/tajo/validation/Validators.java | 16 ++ .../tajo/conf/TestValidatorsForTajoConf.java | 81 ++++++++- .../tajo/validation/TestValidators.java | 152 ++++++++++++++++- .../tajo/engine/query/QueryContext.java | 11 ++ 12 files changed, 466 insertions(+), 27 deletions(-) diff --git a/tajo-common/pom.xml b/tajo-common/pom.xml index d857e0e716..820220c8f1 100644 --- a/tajo-common/pom.xml +++ b/tajo-common/pom.xml @@ -108,6 +108,11 @@ + + + src/test/resources/org/apache/tajo/conf/** + + org.apache.maven.plugins diff --git a/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java index b9d51ec76d..6329e77930 100644 --- a/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java +++ b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java @@ -18,6 +18,8 @@ package org.apache.tajo; +import org.apache.tajo.validation.Validator; + public interface ConfigKey { // Client can set or change variables of this mode. @@ -49,4 +51,8 @@ public String getPrefix() { public String keyname(); public ConfigType type(); + + public Class valueClass(); + + public Validator validator(); } diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java index c9cf7fac2c..288d5775db 100644 --- a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java @@ -73,6 +73,9 @@ public TajoConf getConf() { } public void setBool(ConfigKey key, boolean val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } setBool(key.keyname(), val); } @@ -100,6 +103,9 @@ public boolean getBool(ConfigKey key) { } public void setInt(ConfigKey key, int val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } setInt(key.keyname(), val); } @@ -125,6 +131,9 @@ public int getInt(ConfigKey key) { } public void setLong(ConfigKey key, long val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } setLong(key.keyname(), val); } @@ -150,6 +159,9 @@ public long getLong(ConfigKey key) { } public void setFloat(ConfigKey key, float val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } setFloat(key.keyname(), val); } @@ -175,6 +187,9 @@ public float getFloat(ConfigKey key) { } public void put(ConfigKey key, String val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } set(key.keyname(), val); } diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index 1229849819..fb6b75ec36 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -22,6 +22,8 @@ import java.util.Map; +import org.apache.tajo.validation.Validator; + import static org.apache.tajo.SessionVars.VariableMode.*; import static org.apache.tajo.conf.TajoConf.ConfVars; @@ -143,6 +145,9 @@ public enum SessionVars implements ConfigKey { private final ConfVars key; private final String description; private final VariableMode mode; + + private Class valClass; + private Validator validator; public static enum VariableMode { DEFAULT, // Client can set or change variables of this mode.. @@ -157,6 +162,12 @@ public static enum VariableMode { this.description = description; this.mode = mode; } + + SessionVars(ConfVars key, String description, VariableMode mode, Class valueClass, Validator validator) { + this(key, description, mode); + this.valClass = valueClass; + this.validator = validator; + } public String keyname() { return name(); @@ -213,4 +224,14 @@ public static SessionVars get(String keyname) { public static String handleDeprecatedName(String keyname) { return SessionVars.exists(keyname) ? SessionVars.get(keyname).keyname() : keyname; } + + @Override + public Class valueClass() { + return valClass; + } + + @Override + public Validator validator() { + return validator; + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 66d3030f43..334c50fe45 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -19,30 +19,41 @@ package org.apache.tajo.conf; import com.google.common.base.Preconditions; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.ConfigKey; +import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.NumberUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.datetime.DateTimeConstants; +import org.apache.tajo.validation.ConstraintViolationException; +import org.apache.tajo.validation.Validator; +import org.apache.tajo.validation.Validators; import java.io.IOException; import java.io.PrintStream; import java.net.InetSocketAddress; import java.util.Map; +import java.util.Properties; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; public class TajoConf extends Configuration { - + private static TimeZone CURRENT_TIMEZONE; private static int DATE_ORDER = -1; private static final ReentrantReadWriteLock confLock = new ReentrantReadWriteLock(); private static final Lock writeLock = confLock.writeLock(); private static final Lock readLock = confLock.readLock(); + private final AtomicBoolean isReloaded = new AtomicBoolean(true); + + private static final Map vars = TUtil.newHashMap(); static { Configuration.addDefaultResource("catalog-default.xml"); @@ -53,12 +64,14 @@ public class TajoConf extends Configuration { Configuration.addDefaultResource("tajo-site.xml"); confStaticInit(); + + for (ConfVars confVars: ConfVars.values()) { + vars.put(confVars.keyname(), confVars); + } } private static final String EMPTY_VALUE = ""; - private static final Map vars = TUtil.newHashMap(); - public TajoConf() { super(); } @@ -143,8 +156,8 @@ public static enum ConfVars implements ConfigKey { /////////////////////////////////////////////////////////////////////////////////////// // a username for a running Tajo cluster - ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/"), - USERNAME("tajo.username", "${user.name}"), + ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/", Validators.pathUrl()), + USERNAME("tajo.username", "${user.name}", Validators.shellVar()), // Configurable System Directories WAREHOUSE_DIR("tajo.warehouse.directory", EMPTY_VALUE), @@ -154,16 +167,19 @@ public static enum ConfVars implements ConfigKey { SYSTEM_CONF_REPLICA_COUNT("tajo.system-conf.replica-count", 20), // Tajo Master Service Addresses - TAJO_MASTER_UMBILICAL_RPC_ADDRESS("tajo.master.umbilical-rpc.address", "localhost:26001"), - TAJO_MASTER_CLIENT_RPC_ADDRESS("tajo.master.client-rpc.address", "localhost:26002"), - TAJO_MASTER_INFO_ADDRESS("tajo.master.info-http.address", "0.0.0.0:26080"), + TAJO_MASTER_UMBILICAL_RPC_ADDRESS("tajo.master.umbilical-rpc.address", "localhost:26001", + Validators.networkAddr()), + TAJO_MASTER_CLIENT_RPC_ADDRESS("tajo.master.client-rpc.address", "localhost:26002", + Validators.networkAddr()), + TAJO_MASTER_INFO_ADDRESS("tajo.master.info-http.address", "0.0.0.0:26080", Validators.networkAddr()), // Tajo Master HA Configurations - TAJO_MASTER_HA_ENABLE("tajo.master.ha.enable", false), + TAJO_MASTER_HA_ENABLE("tajo.master.ha.enable", false, Validators.bool()), TAJO_MASTER_HA_MONITOR_INTERVAL("tajo.master.ha.monitor.interval", 5 * 1000), // 5 sec // Resource tracker service - RESOURCE_TRACKER_RPC_ADDRESS("tajo.resource-tracker.rpc.address", "localhost:26003"), + RESOURCE_TRACKER_RPC_ADDRESS("tajo.resource-tracker.rpc.address", "localhost:26003", + Validators.networkAddr()), RESOURCE_TRACKER_HEARTBEAT_TIMEOUT("tajo.resource-tracker.heartbeat.timeout-secs", 120 * 1000), // seconds // QueryMaster resource @@ -171,25 +187,26 @@ public static enum ConfVars implements ConfigKey { TAJO_QUERYMASTER_MEMORY_MB("tajo.qm.resource.memory-mb", 512), // Tajo Worker Service Addresses - WORKER_INFO_ADDRESS("tajo.worker.info-http.address", "0.0.0.0:28080"), - WORKER_QM_INFO_ADDRESS("tajo.worker.qm-info-http.address", "0.0.0.0:28081"), - WORKER_PEER_RPC_ADDRESS("tajo.worker.peer-rpc.address", "0.0.0.0:28091"), - WORKER_CLIENT_RPC_ADDRESS("tajo.worker.client-rpc.address", "0.0.0.0:28092"), - WORKER_QM_RPC_ADDRESS("tajo.worker.qm-rpc.address", "0.0.0.0:28093"), + WORKER_INFO_ADDRESS("tajo.worker.info-http.address", "0.0.0.0:28080", Validators.networkAddr()), + WORKER_QM_INFO_ADDRESS("tajo.worker.qm-info-http.address", "0.0.0.0:28081", Validators.networkAddr()), + WORKER_PEER_RPC_ADDRESS("tajo.worker.peer-rpc.address", "0.0.0.0:28091", Validators.networkAddr()), + WORKER_CLIENT_RPC_ADDRESS("tajo.worker.client-rpc.address", "0.0.0.0:28092", Validators.networkAddr()), + WORKER_QM_RPC_ADDRESS("tajo.worker.qm-rpc.address", "0.0.0.0:28093", Validators.networkAddr()), // Tajo Worker Temporal Directories - WORKER_TEMPORAL_DIR("tajo.worker.tmpdir.locations", "/tmp/tajo-${user.name}/tmpdir"), - WORKER_TEMPORAL_DIR_CLEANUP("tajo.worker.tmpdir.cleanup-at-startup", false), + WORKER_TEMPORAL_DIR("tajo.worker.tmpdir.locations", "/tmp/tajo-${user.name}/tmpdir", + Validators.pathUrl()), + WORKER_TEMPORAL_DIR_CLEANUP("tajo.worker.tmpdir.cleanup-at-startup", false, Validators.bool()), // Tajo Worker Resources WORKER_RESOURCE_AVAILABLE_CPU_CORES("tajo.worker.resource.cpu-cores", 1), WORKER_RESOURCE_AVAILABLE_MEMORY_MB("tajo.worker.resource.memory-mb", 1024), WORKER_RESOURCE_AVAILABLE_DISKS("tajo.worker.resource.disks", 1.0f), WORKER_EXECUTION_MAX_SLOTS("tajo.worker.parallel-execution.max-num", 2), - WORKER_RESOURCE_DFS_DIR_AWARE("tajo.worker.resource.dfs-dir-aware", false), + WORKER_RESOURCE_DFS_DIR_AWARE("tajo.worker.resource.dfs-dir-aware", false, Validators.bool()), // Tajo Worker Dedicated Resources - WORKER_RESOURCE_DEDICATED("tajo.worker.resource.dedicated", false), + WORKER_RESOURCE_DEDICATED("tajo.worker.resource.dedicated", false, Validators.bool()), WORKER_RESOURCE_DEDICATED_MEMORY_RATIO("tajo.worker.resource.dedicated-memory-ratio", 0.8f), // Tajo Worker History @@ -201,7 +218,7 @@ public static enum ConfVars implements ConfigKey { RESOURCE_MANAGER_CLASS("tajo.resource.manager", "org.apache.tajo.master.rm.TajoWorkerResourceManager"), // Catalog - CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005"), + CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005", Validators.networkAddr()), // for Yarn Resource Manager ---------------------------------------------- @@ -217,7 +234,7 @@ public static enum ConfVars implements ConfigKey { // Shuffle Configuration -------------------------------------------------- PULLSERVER_PORT("tajo.pullserver.port", 0), - SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false), + SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false, Validators.bool()), SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"), SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM("tajo.shuffle.fetcher.parallel-execution.max-num", 2), SHUFFLE_FETCHER_CHUNK_MAX_SIZE("tajo.shuffle.fetcher.chunk.max-size", 8192), @@ -231,10 +248,10 @@ public static enum ConfVars implements ConfigKey { ROWFILE_SYNC_INTERVAL("rowfile.sync.interval", 100), MINIMUM_SPLIT_SIZE("tajo.min.split.size", (long) 1), // for RCFile - HIVEUSEEXPLICITRCFILEHEADER("tajo.exec.rcfile.use.explicit.header", true), + HIVEUSEEXPLICITRCFILEHEADER("tajo.exec.rcfile.use.explicit.header", true, Validators.bool()), // for Storage Manager v2 - STORAGE_MANAGER_VERSION_2("tajo.storage-manager.v2", false), + STORAGE_MANAGER_VERSION_2("tajo.storage-manager.v2", false, Validators.bool()), STORAGE_MANAGER_DISK_SCHEDULER_MAX_READ_BYTES_PER_SLOT("tajo.storage-manager.max-read-bytes", 8 * 1024 * 1024), STORAGE_MANAGER_DISK_SCHEDULER_REPORT_INTERVAL("tajo.storage-manager.disk-scheduler.report-interval", 60 * 1000), STORAGE_MANAGER_CONCURRENCY_PER_DISK("tajo.storage-manager.disk-scheduler.per-disk-concurrency", 2), @@ -373,6 +390,7 @@ public static enum ConfVars implements ConfigKey { public final boolean defaultBoolVal; private final VarType type; + private Validator validator; ConfVars(String varname, String defaultVal) { this.varname = varname; @@ -384,6 +402,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = false; this.type = VarType.STRING; } + + ConfVars(String varname, String defaultVal, Validator validator) { + this(varname, defaultVal); + this.validator = validator; + } ConfVars(String varname, int defaultIntVal) { this.varname = varname; @@ -395,6 +418,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = false; this.type = VarType.INT; } + + ConfVars(String varname, int defaultIntVal, Validator validator) { + this(varname, defaultIntVal); + this.validator = validator; + } ConfVars(String varname, long defaultLongVal) { this.varname = varname; @@ -406,6 +434,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = false; this.type = VarType.LONG; } + + ConfVars(String varname, long defaultLongVal, Validator validator) { + this(varname, defaultLongVal); + this.validator = validator; + } ConfVars(String varname, float defaultFloatVal) { this.varname = varname; @@ -417,6 +450,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = false; this.type = VarType.FLOAT; } + + ConfVars(String varname, float defaultFloatVal, Validator validator) { + this(varname, defaultFloatVal); + this.validator = validator; + } ConfVars(String varname, boolean defaultBoolVal) { this.varname = varname; @@ -428,6 +466,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = defaultBoolVal; this.type = VarType.BOOLEAN; } + + ConfVars(String varname, boolean defaultBoolVal, Validator validator) { + this(varname, defaultBoolVal); + this.validator = validator; + } enum VarType { STRING { void checkType(String value) throws Exception { } }, @@ -453,6 +496,16 @@ public String keyname() { public ConfigType type() { return ConfigType.SYSTEM; } + + @Override + public Class valueClass() { + return valClass; + } + + @Override + public Validator validator() { + return validator; + } } public static int getIntVar(Configuration conf, ConfVars var) { @@ -642,4 +695,59 @@ public static Path getSystemConfPath(TajoConf conf) { return new Path(systemConfPathStr); } } + + private void validateProperty(String name, String value) throws ConstraintViolationException { + ConfigKey configKey = null; + + configKey = TajoConf.getConfVars(name); + if (configKey == null) { + configKey = SessionVars.get(name); + } + + if (configKey != null && configKey.validator() != null && configKey.valueClass() != null) { + Object valueObj = value; + if (Number.class.isAssignableFrom(configKey.valueClass())) { + valueObj = NumberUtil.numberValue(configKey.valueClass(), value); + if (valueObj == null) { + return; + } + } + + configKey.validator().validate(valueObj, true); + } + } + + @Override + protected synchronized Properties getProps() { + Properties properties = super.getProps(); + + if (isReloaded.getAndSet(false)) { + + for (String key: properties.stringPropertyNames()) { + String value = properties.getProperty(key); + + validateProperty(key, value); + } + } + + return properties; + } + + @Override + public synchronized void reloadConfiguration() { + super.reloadConfiguration(); + isReloaded.set(true); + } + + @Override + public void set(String name, String value, String source) { + validateProperty(name, value); + super.set(name, value, source); + } + + @Override + public void set(String name, String value) { + set(name, value, null); + } + } diff --git a/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java index d52b80497c..375a2e45ac 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java @@ -18,6 +18,9 @@ package org.apache.tajo.util; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + public class NumberUtil { public static final double[] powersOf10 = { /* Table giving binary powers of 10. Entry */ @@ -408,4 +411,22 @@ private static int parseIntInternal(byte[] bytes, int start, int length, int off } return result; } + + public static Number numberValue(Class numberClazz, String value) { + Number returnNumber = null; + + if (numberClazz == null && value == null) { + return returnNumber; + } + + if (Number.class.isAssignableFrom(numberClazz)) { + try { + Constructor constructor = numberClazz.getConstructor(String.class); + returnNumber = (Number) constructor.newInstance(value); + } catch (Exception ignored) { + } + } + + return returnNumber; + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java index 63c2352b09..300c5ea7bb 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java @@ -55,6 +55,10 @@ protected boolean validateInternal(T object) { Long objLong = Long.decode(object.toString()); Long maxLong = Long.decode(maxValue); result = objLong.compareTo(maxLong) <= 0; + } else if ((object instanceof Float) || (object instanceof Double)) { + Double objDouble = Double.valueOf(object.toString()); + Double maxDouble = Double.valueOf(maxValue); + result = objDouble.compareTo(maxDouble) <= 0; } else if (object instanceof BigInteger) { BigInteger objInteger = (BigInteger) object; BigInteger maxInteger = new BigInteger(maxValue); diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java index 0c3a99a2a2..2a29671891 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java @@ -41,7 +41,12 @@ protected boolean validateInternal(T object) { if (object != null) { if (object instanceof CharSequence) { - result = pattern.matcher((CharSequence) object).find(); + String valueString = object.toString(); + if (valueString.isEmpty()) { + result = true; + } else { + result = pattern.matcher((CharSequence) object).find(); + } } } else { result = true; diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java index 26056ad9ce..4184c9757e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java @@ -50,4 +50,20 @@ public static Validator range(String minValue, String maxValue) { return new RangeValidator(minValue, maxValue); } + public static Validator pathUrl() { + return new PathValidator(); + } + + public static Validator shellVar() { + return new ShellVariableValidator(); + } + + public static Validator networkAddr() { + return new NetworkAddressValidator(); + } + + public static Validator bool() { + return new BooleanValidator(); + } + } diff --git a/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java b/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java index db5aae77c0..cdf4588861 100644 --- a/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java +++ b/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java @@ -1,26 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.conf; import static org.junit.Assert.*; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.hadoop.conf.Configuration; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.validation.ConstraintViolationException; +import org.junit.Before; import org.junit.Test; public class TestValidatorsForTajoConf { + + @Before + public void setUp() throws Exception { + CopyOnWriteArrayList defaultResources = new CopyOnWriteArrayList( + new String[] {"catalog-default.xml", "catalog-site.xml", + "storage-default.xml", "storage-site.xml", + "tajo-default.xml", "tajo-site.xml"}); + Field defaultResourcesField = Configuration.class.getDeclaredField("defaultResources"); + defaultResourcesField.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(defaultResourcesField, defaultResourcesField.getModifiers() & ~Modifier.FINAL); + defaultResourcesField.set(null, defaultResources); + } @Test(expected=ConstraintViolationException.class) public void testPathValidatorsForTajoConf_setVar() { TajoConf conf = new TajoConf(); TajoConf.setVar(conf, ConfVars.ROOT_DIR, "Invalid path"); - fail(); + fail("ConstraintViolationException is expected but it is not occurred."); } @Test public void testValidPathForTajoConf() { TajoConf conf = new TajoConf(); - TajoConf.setVar(conf, ConfVars.ROOT_DIR, "file:///tmp/tajo-${user.name}/"); + try { + TajoConf.setVar(conf, ConfVars.ROOT_DIR, "file:///tmp/tajo-${user.name}/"); + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected ConstraintViolationException has been occurred."); + } + } + + @Test(expected=ConstraintViolationException.class) + public void testAddDefaultResourceValidation1() { + TajoConf.addDefaultResource("org/apache/tajo/conf/InvalidConf1.xml"); + TajoConf conf = new TajoConf(); + + conf.getVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS); + fail("Validation check has failed"); + } + + @Test(expected=ConstraintViolationException.class) + public void testAddDefaultResourceValidation2() { + TajoConf.addDefaultResource("org/apache/tajo/conf/InvalidConf2.xml"); + TajoConf conf = new TajoConf(); + + conf.getBoolVar(ConfVars.TAJO_MASTER_HA_ENABLE); + fail("Validation check has failed"); + } + + @Test(expected=ConstraintViolationException.class) + public void testSetConf1() { + TajoConf conf = new TajoConf(); + + conf.setVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, "9440"); + fail("Validation check has failed"); + } + + @Test(expected=ConstraintViolationException.class) + public void testSetConf2() { + TajoConf conf = new TajoConf(); + + conf.set(ConfVars.TAJO_MASTER_HA_ENABLE.keyname(), "yes"); + fail("Validation check has failed"); } } diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java index facb8c89c5..4d79b6bb39 100644 --- a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.validation; import static org.junit.Assert.*; @@ -266,7 +284,7 @@ public void testGroupValidator() { validators.add(new PatternValidator("^[a-zA-Z]+://")); validators.add(new LengthValidator(255)); assertThat(new GroupValidator(validators).validate(httpUrl).size(), is(0)); - assertThat(new GroupValidator(validators).validate("tajo").size(), is(2)); + assertThat(new GroupValidator(validators).validate("tajo").size(), is(1)); assertThat(new GroupValidator(validators).validate("tajo"), hasItem(hasAClass(equalTo(PatternValidator.class)))); @@ -298,5 +316,137 @@ public void testExceptionThrow() { fail(); } + + @Test + public void testPathValidator() { + String validUrl = "file:///tmp/tajo-$root/"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + validUrl = "file:///tmp/tajo-${user.name}/"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + validUrl = "file:///C:/Windows/System32"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + validUrl = "/C:/Windows/system32/driver"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + validUrl = "/tmp/tajo-root/"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + String invalidUrl = "t!ef:///tmp/tajo-root"; + assertThat(new PathValidator().validateInternal(invalidUrl), is(false)); + assertThat(new PathValidator().validate(invalidUrl).size(), is(1)); + assertThat(new PathValidator().validate(invalidUrl), + hasItem(hasAClass(equalTo(PathValidator.class)))); + + invalidUrl = "This is not a valid url."; + assertThat(new PathValidator().validateInternal(invalidUrl), is(false)); + assertThat(new PathValidator().validate(invalidUrl).size(), is(1)); + assertThat(new PathValidator().validate(invalidUrl), + hasItem(hasAClass(equalTo(PathValidator.class)))); + } + + @Test + public void testShellVariableValidator() { + String validVariable = "${user.name}"; + assertThat(new ShellVariableValidator().validateInternal(validVariable), is(true)); + assertThat(new ShellVariableValidator().validate(validVariable).size(), is(0)); + + validVariable = "$SHELL"; + assertThat(new ShellVariableValidator().validateInternal(validVariable), is(true)); + assertThat(new ShellVariableValidator().validate(validVariable).size(), is(0)); + + validVariable = "STRING"; + assertThat(new ShellVariableValidator().validateInternal(validVariable), is(true)); + assertThat(new ShellVariableValidator().validate(validVariable).size(), is(0)); + + String invalidVariable = "Invalid Shell Variable Name"; + assertThat(new ShellVariableValidator().validateInternal(invalidVariable), is(false)); + assertThat(new ShellVariableValidator().validate(invalidVariable).size(), is(1)); + assertThat(new ShellVariableValidator().validate(invalidVariable), + hasItem(hasAClass(equalTo(ShellVariableValidator.class)))); + } + + @Test + public void testNetworkAddressValidator() { + String validNetworkAddress = "localhost:5000"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "192.168.0.1:5000"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "0.0.0.0:28094"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "tajo.apache.org"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "192.168.122.1"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + String invalidNetAddr = "5000"; + assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr), + hasItem(hasAClass(equalTo(NetworkAddressValidator.class)))); + + invalidNetAddr = "192.168."; + assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr), + hasItem(hasAClass(equalTo(NetworkAddressValidator.class)))); + + invalidNetAddr = "localhost:98765"; + assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr), + hasItem(hasAClass(equalTo(NetworkAddressValidator.class)))); + } + + @Test + public void testBooleanValidator() { + String validBoolean = "true"; + assertThat(new BooleanValidator().validateInternal(validBoolean), is(true)); + assertThat(new BooleanValidator().validate(validBoolean).size(), is(0)); + + validBoolean = "false"; + assertThat(new BooleanValidator().validateInternal(validBoolean), is(true)); + assertThat(new BooleanValidator().validate(validBoolean).size(), is(0)); + + assertThat(new BooleanValidator().validateInternal(true), is(true)); + assertThat(new BooleanValidator().validate(true).size(), is(0)); + + assertThat(new BooleanValidator().validateInternal(false), is(true)); + assertThat(new BooleanValidator().validate(false).size(), is(0)); + + String invalidBoolean = "yes"; + assertThat(new BooleanValidator().validateInternal(invalidBoolean), is(false)); + assertThat(new BooleanValidator().validate(invalidBoolean).size(), is(1)); + assertThat(new BooleanValidator().validate(invalidBoolean), + hasItem(hasAClass(equalTo(BooleanValidator.class)))); + + invalidBoolean = "nope"; + assertThat(new BooleanValidator().validateInternal(invalidBoolean), is(false)); + assertThat(new BooleanValidator().validate(invalidBoolean).size(), is(1)); + assertThat(new BooleanValidator().validate(invalidBoolean), + hasItem(hasAClass(equalTo(BooleanValidator.class)))); + + invalidBoolean = "invalid"; + assertThat(new BooleanValidator().validateInternal(invalidBoolean), is(false)); + assertThat(new BooleanValidator().validate(invalidBoolean).size(), is(1)); + assertThat(new BooleanValidator().validate(invalidBoolean), + hasItem(hasAClass(equalTo(BooleanValidator.class)))); + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index f4160e495c..3af70f5a09 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -26,6 +26,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.master.session.Session; +import org.apache.tajo.validation.Validator; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; @@ -56,6 +57,16 @@ public String keyname() { public ConfigType type() { return ConfigType.QUERY; } + + @Override + public Class valueClass() { + return null; + } + + @Override + public Validator validator() { + return null; + } } public QueryContext(TajoConf conf) { From f99e7edcfa51c2955da7a54af4a2edf7b11c6068 Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Mon, 20 Oct 2014 16:13:53 +0900 Subject: [PATCH 03/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- .../tajo/validation/BooleanValidator.java | 57 +++++++++++++++ .../validation/NetworkAddressValidator.java | 73 +++++++++++++++++++ .../apache/tajo/validation/PathValidator.java | 32 ++++++++ .../validation/ShellVariableValidator.java | 32 ++++++++ .../org/apache/tajo/conf/InvalidConf1.xml | 7 ++ .../org/apache/tajo/conf/InvalidConf2.xml | 7 ++ 6 files changed, 208 insertions(+) create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/BooleanValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/ShellVariableValidator.java create mode 100644 tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf1.xml create mode 100644 tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf2.xml diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/BooleanValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/BooleanValidator.java new file mode 100644 index 0000000000..b72a5fde03 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/BooleanValidator.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class BooleanValidator extends AbstractValidator { + + @Override + protected String getErrorMessage(T object) { + return object + " is not a valid boolean representation."; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof Boolean) { + result = true; + } else if (object instanceof CharSequence) { + String valueString = object.toString(); + if (Boolean.FALSE.toString().equalsIgnoreCase(valueString) || + Boolean.TRUE.toString().equalsIgnoreCase(valueString)) { + result = true; + } + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java new file mode 100644 index 0000000000..31565646fc --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; +import java.util.regex.Pattern; + +public class NetworkAddressValidator extends AbstractValidator { + + private final Pattern hostnamePattern; + private final Pattern portNumberPattern; + + public NetworkAddressValidator() { + hostnamePattern = Pattern.compile( + "^(?:[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}|[a-zA-Z][a-zA-Z0-9.-_]+)$"); + portNumberPattern = Pattern.compile("^[1-6]?[0-9]{1,4}$"); + } + + @Override + protected String getErrorMessage(T object) { + return object + " is not a valid network address representation."; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof CharSequence) { + String valueString = object.toString(); + if (valueString.isEmpty()) { + + } else { + int separatorIdx = valueString.indexOf(':'); + + if (separatorIdx > -1) { + result = (hostnamePattern.matcher(valueString.substring(0, separatorIdx)).find() & + portNumberPattern.matcher(valueString.substring(separatorIdx + 1)).find()); + } else { + result = hostnamePattern.matcher(valueString).find(); + } + } + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java new file mode 100644 index 0000000000..aad66a2fe6 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +public class PathValidator extends PatternValidator { + + public PathValidator() { + super("^(?:[a-zA-Z][a-zA-Z0-9+-.]+://[a-zA-Z-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$"); + } + + @Override + protected String getErrorMessage(T object) { + return object + " is not valid path."; + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ShellVariableValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/ShellVariableValidator.java new file mode 100644 index 0000000000..cfc0cc93ff --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ShellVariableValidator.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +public class ShellVariableValidator extends PatternValidator { + + public ShellVariableValidator() { + super("^[\\\\$]?[\\\\{]?[a-zA-Z0-9.]+[\\\\}]?$"); + } + + @Override + protected String getErrorMessage(T object) { + return object + " is not a valid shell variable"; + } + +} diff --git a/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf1.xml b/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf1.xml new file mode 100644 index 0000000000..c9a7c01e3d --- /dev/null +++ b/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf1.xml @@ -0,0 +1,7 @@ + + + + tajo.master.umbilical-rpc.address + localhost:98475 + + \ No newline at end of file diff --git a/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf2.xml b/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf2.xml new file mode 100644 index 0000000000..674a16887b --- /dev/null +++ b/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf2.xml @@ -0,0 +1,7 @@ + + + + tajo.master.ha.enable + yes + + \ No newline at end of file From ce0fb6d44e5782c28618b735d95b10c57d3f4692 Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Mon, 20 Oct 2014 21:47:27 +0900 Subject: [PATCH 04/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- .../java/org/apache/tajo/SessionVars.java | 42 ++++++++------ .../java/org/apache/tajo/conf/TajoConf.java | 40 +++++++------ .../tajo/validation/ClassValidator.java | 57 +++++++++++++++++++ .../ConstraintViolationException.java | 7 ++- .../validation/NetworkAddressValidator.java | 42 ++++++++++++-- .../apache/tajo/validation/PathValidator.java | 2 +- .../apache/tajo/validation/Validators.java | 4 ++ .../tajo/validation/TestValidators.java | 42 ++++++++++++-- 8 files changed, 190 insertions(+), 46 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/ClassValidator.java diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index fb6b75ec36..dbe949acc4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.tajo.validation.Validator; +import org.apache.tajo.validation.Validators; import static org.apache.tajo.SessionVars.VariableMode.*; import static org.apache.tajo.conf.TajoConf.ConfVars; @@ -42,8 +43,8 @@ public enum SessionVars implements ConfigKey { //------------------------------------------------------------------------------- // Server Side Only Variables //------------------------------------------------------------------------------- - SESSION_ID(ConfVars.$EMPTY, "session variable", SERVER_SIDE_VAR), - SESSION_LAST_ACCESS_TIME(ConfVars.$EMPTY, "last access time", SERVER_SIDE_VAR), + SESSION_ID(ConfVars.$EMPTY, "session variable", SERVER_SIDE_VAR, String.class, Validators.notNull()), + SESSION_LAST_ACCESS_TIME(ConfVars.$EMPTY, "last access time", SERVER_SIDE_VAR, Long.class, Validators.min("0")), USERNAME(ConfVars.USERNAME, "username", SERVER_SIDE_VAR), CLIENT_HOST(ConfVars.$EMPTY, "client hostname", SERVER_SIDE_VAR), @@ -55,7 +56,8 @@ public enum SessionVars implements ConfigKey { //------------------------------------------------------------------------------- // Client -------------------------------------------------------- - SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "session expiry time (secs)", DEFAULT), + SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "session expiry time (secs)", DEFAULT, + Integer.class, Validators.min("0")), // Command line interface and its behavior -------------------------------- CLI_COLUMNS(ConfVars.$CLI_MAX_COLUMN, "Sets the width for the wrapped format", CLI_SIDE_VAR), @@ -89,29 +91,36 @@ public enum SessionVars implements ConfigKey { // for distributed query strategies BROADCAST_TABLE_SIZE_LIMIT(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, "limited size (bytes) of broadcast table", - DEFAULT), + DEFAULT, Long.class, Validators.min("0")), - JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size (mb) ", DEFAULT), + JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size (mb) ", DEFAULT, + Integer.class, Validators.min("1")), SORT_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_SORT_TASK_VOLUME, "sort task input size (mb)", DEFAULT), GROUPBY_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_GROUPBY_TASK_VOLUME, "group by task input size (mb)", DEFAULT), - JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join (mb)", DEFAULT), - GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort (mb)", DEFAULT), + JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join (mb)", DEFAULT, + Integer.class, Validators.min("1")), + GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort (mb)", DEFAULT, + Integer.class, Validators.min("1")), TABLE_PARTITION_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME, - "shuffle output size for partition table write (mb)", DEFAULT), + "shuffle output size for partition table write (mb)", DEFAULT, Long.class, Validators.min("1")), - GROUPBY_MULTI_LEVEL_ENABLED(ConfVars.$GROUPBY_MULTI_LEVEL_ENABLED, "Multiple level groupby enabled", DEFAULT), + GROUPBY_MULTI_LEVEL_ENABLED(ConfVars.$GROUPBY_MULTI_LEVEL_ENABLED, "Multiple level groupby enabled", DEFAULT, + Boolean.class, Validators.bool()), // for physical Executors - EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort (mb)", DEFAULT), - HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash join (mb)", DEFAULT), + EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort (mb)", DEFAULT, + Long.class, Validators.min("0")), + HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash join (mb)", DEFAULT, + Long.class, Validators.min("0")), INNER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD, - "limited size for hash inner join (mb)", DEFAULT), + "limited size for hash inner join (mb)", DEFAULT, Long.class, Validators.min("0")), OUTER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash outer join (mb)", - DEFAULT), + DEFAULT, Long.class, Validators.min("0")), HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD, "limited size for hash groupby (mb)", - DEFAULT), - MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size (mb). 0 means infinite.", DEFAULT), + DEFAULT, Long.class, Validators.min("0")), + MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size (mb). 0 means infinite.", DEFAULT, + Long.class, Validators.min("0")), NULL_CHAR(ConfVars.$CSVFILE_NULL, "null char of text file output", DEFAULT), CODEGEN(ConfVars.$CODEGEN, "Runtime code generation enabled (experiment)", DEFAULT), @@ -120,7 +129,8 @@ public enum SessionVars implements ConfigKey { "If true, a running query will be terminated when an overflow or divide-by-zero occurs.", DEFAULT), // ResultSet ---------------------------------------------------------------- - FETCH_ROWNUM(ConfVars.$RESULT_SET_FETCH_ROWNUM, "Sets the number of rows at a time from Master", DEFAULT), + FETCH_ROWNUM(ConfVars.$RESULT_SET_FETCH_ROWNUM, "Sets the number of rows at a time from Master", DEFAULT, + Integer.class, Validators.min("0")), //------------------------------------------------------------------------------- // Only for Unit Testing diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 334c50fe45..281aecd96a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -156,15 +156,17 @@ public static enum ConfVars implements ConfigKey { /////////////////////////////////////////////////////////////////////////////////////// // a username for a running Tajo cluster - ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/", Validators.pathUrl()), - USERNAME("tajo.username", "${user.name}", Validators.shellVar()), + ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/", + Validators.groups(Validators.notNull(), Validators.pathUrl())), + USERNAME("tajo.username", "${user.name}", + Validators.groups(Validators.notNull(), Validators.shellVar())), // Configurable System Directories - WAREHOUSE_DIR("tajo.warehouse.directory", EMPTY_VALUE), - STAGING_ROOT_DIR("tajo.staging.directory", "/tmp/tajo-${user.name}/staging"), + WAREHOUSE_DIR("tajo.warehouse.directory", EMPTY_VALUE, Validators.pathUrl()), + STAGING_ROOT_DIR("tajo.staging.directory", "/tmp/tajo-${user.name}/staging", Validators.pathUrl()), - SYSTEM_CONF_PATH("tajo.system-conf.path", EMPTY_VALUE), - SYSTEM_CONF_REPLICA_COUNT("tajo.system-conf.replica-count", 20), + SYSTEM_CONF_PATH("tajo.system-conf.path", EMPTY_VALUE, Validators.pathUrl()), + SYSTEM_CONF_REPLICA_COUNT("tajo.system-conf.replica-count", 20, Validators.min("1")), // Tajo Master Service Addresses TAJO_MASTER_UMBILICAL_RPC_ADDRESS("tajo.master.umbilical-rpc.address", "localhost:26001", @@ -183,8 +185,8 @@ public static enum ConfVars implements ConfigKey { RESOURCE_TRACKER_HEARTBEAT_TIMEOUT("tajo.resource-tracker.heartbeat.timeout-secs", 120 * 1000), // seconds // QueryMaster resource - TAJO_QUERYMASTER_DISK_SLOT("tajo.qm.resource.disk.slots", 0.0f), - TAJO_QUERYMASTER_MEMORY_MB("tajo.qm.resource.memory-mb", 512), + TAJO_QUERYMASTER_DISK_SLOT("tajo.qm.resource.disk.slots", 0.0f, Validators.min("0.0f")), + TAJO_QUERYMASTER_MEMORY_MB("tajo.qm.resource.memory-mb", 512, Validators.min("64")), // Tajo Worker Service Addresses WORKER_INFO_ADDRESS("tajo.worker.info-http.address", "0.0.0.0:28080", Validators.networkAddr()), @@ -199,15 +201,16 @@ public static enum ConfVars implements ConfigKey { WORKER_TEMPORAL_DIR_CLEANUP("tajo.worker.tmpdir.cleanup-at-startup", false, Validators.bool()), // Tajo Worker Resources - WORKER_RESOURCE_AVAILABLE_CPU_CORES("tajo.worker.resource.cpu-cores", 1), - WORKER_RESOURCE_AVAILABLE_MEMORY_MB("tajo.worker.resource.memory-mb", 1024), + WORKER_RESOURCE_AVAILABLE_CPU_CORES("tajo.worker.resource.cpu-cores", 1, Validators.min("1")), + WORKER_RESOURCE_AVAILABLE_MEMORY_MB("tajo.worker.resource.memory-mb", 1024, Validators.min("64")), WORKER_RESOURCE_AVAILABLE_DISKS("tajo.worker.resource.disks", 1.0f), WORKER_EXECUTION_MAX_SLOTS("tajo.worker.parallel-execution.max-num", 2), WORKER_RESOURCE_DFS_DIR_AWARE("tajo.worker.resource.dfs-dir-aware", false, Validators.bool()), // Tajo Worker Dedicated Resources WORKER_RESOURCE_DEDICATED("tajo.worker.resource.dedicated", false, Validators.bool()), - WORKER_RESOURCE_DEDICATED_MEMORY_RATIO("tajo.worker.resource.dedicated-memory-ratio", 0.8f), + WORKER_RESOURCE_DEDICATED_MEMORY_RATIO("tajo.worker.resource.dedicated-memory-ratio", 0.8f, + Validators.range("0.0f", "1.0f")), // Tajo Worker History WORKER_HISTORY_EXPIRE_PERIOD("tajo.worker.history.expire-interval-minutes", 12 * 60), // 12 hours @@ -215,7 +218,8 @@ public static enum ConfVars implements ConfigKey { WORKER_HEARTBEAT_TIMEOUT("tajo.worker.heartbeat.timeout", 120 * 1000), // 120 sec // Resource Manager - RESOURCE_MANAGER_CLASS("tajo.resource.manager", "org.apache.tajo.master.rm.TajoWorkerResourceManager"), + RESOURCE_MANAGER_CLASS("tajo.resource.manager", "org.apache.tajo.master.rm.TajoWorkerResourceManager", + Validators.groups(Validators.notNull(), Validators.clazz())), // Catalog CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005", Validators.networkAddr()), @@ -224,16 +228,16 @@ public static enum ConfVars implements ConfigKey { // for Yarn Resource Manager ---------------------------------------------- /** how many launching TaskRunners in parallel */ - YARN_RM_QUERY_MASTER_MEMORY_MB("tajo.querymaster.memory-mb", 512), + YARN_RM_QUERY_MASTER_MEMORY_MB("tajo.querymaster.memory-mb", 512, Validators.min("64")), YARN_RM_QUERY_MASTER_DISKS("tajo.yarn-rm.querymaster.disks", 1), YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM("tajo.yarn-rm.parallel-task-runner-launcher-num", 16), YARN_RM_WORKER_NUMBER_PER_NODE("tajo.yarn-rm.max-worker-num-per-node", 8), // Query Configuration - QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60), + QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60, Validators.min("0")), // Shuffle Configuration -------------------------------------------------- - PULLSERVER_PORT("tajo.pullserver.port", 0), + PULLSERVER_PORT("tajo.pullserver.port", 0, Validators.range("0", "65535")), SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false, Validators.bool()), SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"), SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM("tajo.shuffle.fetcher.parallel-execution.max-num", 2), @@ -329,9 +333,9 @@ public static enum ConfVars implements ConfigKey { $DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128), $DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128), - $DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128), - $DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256), - $DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256), + $DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128, Validators.min("1")), + $DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256, Validators.min("1")), + $DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256, Validators.min("1")), $GROUPBY_MULTI_LEVEL_ENABLED("tajo.dist-query.groupby.multi-level-aggr", true), diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ClassValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/ClassValidator.java new file mode 100644 index 0000000000..1b35e6d195 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ClassValidator.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class ClassValidator extends AbstractValidator { + + @Override + protected String getErrorMessage(T object) { + return "ClassLoader cannot find " + object + " class."; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof CharSequence) { + String valueString = object.toString(); + try { + Class.forName(valueString); + result = true; + } catch (ClassNotFoundException e) { + result = false; + } + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java index 664ec09771..cad6184567 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java @@ -38,8 +38,13 @@ public Collection getViolations() { public String getMessage() { if (violations != null) { String errorMessage = "ConstraintViolationException ["; + int elemIdx = 1; + int elemCount = violations.size(); for (ConstraintViolation violation: violations) { - errorMessage += violation.getMessage() + ","; + errorMessage += violation.getMessage(); + if (elemIdx > elemCount) { + errorMessage += ","; + } } errorMessage += "]"; return errorMessage; diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java index 31565646fc..7477c4ca28 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java @@ -22,6 +22,8 @@ import java.util.Collections; import java.util.regex.Pattern; +import org.apache.http.conn.util.InetAddressUtils; + public class NetworkAddressValidator extends AbstractValidator { private final Pattern hostnamePattern; @@ -29,7 +31,7 @@ public class NetworkAddressValidator extends AbstractValidator { public NetworkAddressValidator() { hostnamePattern = Pattern.compile( - "^(?:[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}|[a-zA-Z][a-zA-Z0-9.-_]+)$"); + "^[a-zA-Z][-a-zA-Z0-9\\\\._]+$"); portNumberPattern = Pattern.compile("^[1-6]?[0-9]{1,4}$"); } @@ -46,15 +48,43 @@ protected boolean validateInternal(T object) { if (object instanceof CharSequence) { String valueString = object.toString(); if (valueString.isEmpty()) { - + result = true; } else { int separatorIdx = valueString.indexOf(':'); - + String hostOrIpAddress = null; + if (separatorIdx > -1) { - result = (hostnamePattern.matcher(valueString.substring(0, separatorIdx)).find() & - portNumberPattern.matcher(valueString.substring(separatorIdx + 1)).find()); + if (valueString.indexOf(':', separatorIdx+1) > -1) { + // it is IPV6 representation. + int leftBracketsIdx = valueString.indexOf('['); + int rightBracketsIdx = valueString.indexOf(']'); + int periodIdx = valueString.indexOf('.'); + + if ((leftBracketsIdx > -1) && (rightBracketsIdx > -1) && + (valueString.length() > (rightBracketsIdx+1)) && + valueString.charAt(rightBracketsIdx+1) == ':') { + hostOrIpAddress = valueString.substring(leftBracketsIdx+1, rightBracketsIdx); + separatorIdx = rightBracketsIdx+1; + } else if ((periodIdx > -1)) { + hostOrIpAddress = valueString.substring(0, periodIdx); + separatorIdx = periodIdx; + } else { + separatorIdx = valueString.lastIndexOf(':'); + hostOrIpAddress = valueString.substring(0, separatorIdx); + } + } else { + hostOrIpAddress = valueString.substring(0, separatorIdx); + } } else { - result = hostnamePattern.matcher(valueString).find(); + hostOrIpAddress = valueString; + } + + result = ((hostnamePattern.matcher(hostOrIpAddress).find()) | + InetAddressUtils.isIPv4Address(hostOrIpAddress) | + InetAddressUtils.isIPv6Address(hostOrIpAddress)); + + if (separatorIdx > -1) { + result &= portNumberPattern.matcher(valueString.substring(separatorIdx + 1)).find(); } } } diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java index aad66a2fe6..a76587cd69 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java @@ -21,7 +21,7 @@ public class PathValidator extends PatternValidator { public PathValidator() { - super("^(?:[a-zA-Z][a-zA-Z0-9+-.]+://[a-zA-Z-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$"); + super("^(?:[a-zA-Z][a-zA-Z0-9+-.]+:[/]{1,2}[a-zA-Z-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$"); } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java index 4184c9757e..cc0c82c0f2 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java @@ -66,4 +66,8 @@ public static Validator bool() { return new BooleanValidator(); } + public static Validator clazz() { + return new ClassValidator(); + } + } diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java index 4d79b6bb39..13370d6c02 100644 --- a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -31,13 +31,14 @@ import org.apache.tajo.util.TUtil; import org.hamcrest.Description; import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; import org.junit.Test; public class TestValidators { - private class ValidatorClazzMatcher extends org.hamcrest.TypeSafeDiagnosingMatcher { + private class ValidatorClazzMatcher extends TypeSafeDiagnosingMatcher { - private final org.hamcrest.Matcher> matcher; + private final Matcher> matcher; public ValidatorClazzMatcher(Matcher> matcher) { this.matcher = matcher; @@ -327,6 +328,10 @@ public void testPathValidator() { assertThat(new PathValidator().validateInternal(validUrl), is(true)); assertThat(new PathValidator().validate(validUrl).size(), is(0)); + validUrl = "file:/home/tajo/test-data/TestExternalSortExec"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + validUrl = "file:///C:/Windows/System32"; assertThat(new PathValidator().validateInternal(validUrl), is(true)); assertThat(new PathValidator().validate(validUrl).size(), is(0)); @@ -387,7 +392,7 @@ public void testNetworkAddressValidator() { assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); - validNetworkAddress = "tajo.apache.org"; + validNetworkAddress = "Tajo-Test.apache.org"; assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); @@ -395,13 +400,25 @@ public void testNetworkAddressValidator() { assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + validNetworkAddress = "[2001:db8::ff00:42:8329]:20089"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "2001:db8::ff00:42:8330:20089"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "2001:db8::ff00:42:8331.20089"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + String invalidNetAddr = "5000"; assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); assertThat(new NetworkAddressValidator().validate(invalidNetAddr), hasItem(hasAClass(equalTo(NetworkAddressValidator.class)))); - invalidNetAddr = "192.168."; + invalidNetAddr = "192.168.:"; assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); assertThat(new NetworkAddressValidator().validate(invalidNetAddr), @@ -448,5 +465,22 @@ public void testBooleanValidator() { assertThat(new BooleanValidator().validate(invalidBoolean), hasItem(hasAClass(equalTo(BooleanValidator.class)))); } + + @Test + public void testClassValidator() { + String clazzName = "org.apache.tajo.validation.ClassValidator"; + assertThat(new ClassValidator().validateInternal(clazzName), is(true)); + assertThat(new ClassValidator().validate(clazzName).size(), is(0)); + + clazzName = "org.apache.tajo.ConfigKey"; + assertThat(new ClassValidator().validateInternal(clazzName), is(true)); + assertThat(new ClassValidator().validate(clazzName).size(), is(0)); + + String invalidClazzName = "invalid-.class.name"; + assertThat(new ClassValidator().validateInternal(invalidClazzName), is(false)); + assertThat(new ClassValidator().validate(invalidClazzName).size(), is(1)); + assertThat(new ClassValidator().validate(invalidClazzName), + hasItem(hasAClass(equalTo(ClassValidator.class)))); + } } From 3f3a23a138cc523316cd2b28f1251921a55d404e Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Mon, 20 Oct 2014 22:02:30 +0900 Subject: [PATCH 05/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- .../test/java/org/apache/tajo/validation/TestValidators.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java index 13370d6c02..a104874ee2 100644 --- a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -62,7 +62,7 @@ protected boolean matchesSafely(ConstraintViolation item, Description mismatchDe } - private org.hamcrest.Matcher + private Matcher hasAClass(Matcher> matcher) { return new ValidatorClazzMatcher(matcher); } @@ -74,7 +74,7 @@ public void testNotNullValidator() { assertThat(new NotNullValidator().validateInternal(testValue), is(false)); assertThat(new NotNullValidator().validate(testValue).size(), is(1)); assertThat(new NotNullValidator().validate(testValue), - hasItem(new ValidatorClazzMatcher(equalTo(NotNullValidator.class)))); + hasItem(hasAClass(equalTo(NotNullValidator.class)))); } @Test From 0fb2285fb4ad054db16a18a75d2bbc2bf5317ce5 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 20 Oct 2014 22:19:59 -0700 Subject: [PATCH 06/15] Update Tajo site for 0.9.0 release. --- tajo-project/src/site/apt/index.apt | 2 + tajo-project/src/site/markdown/downloads.md | 12 +- .../markdown/releases/0.9.0/announcement.md | 84 ++ .../resources/releases/0.9.0/relnotes.html | 718 ++++++++++++++++++ tajo-project/src/site/site.xml | 9 +- 5 files changed, 820 insertions(+), 5 deletions(-) create mode 100644 tajo-project/src/site/markdown/releases/0.9.0/announcement.md create mode 100644 tajo-project/src/site/resources/releases/0.9.0/relnotes.html diff --git a/tajo-project/src/site/apt/index.apt b/tajo-project/src/site/apt/index.apt index 04dbad84da..5c2e8dc9ab 100644 --- a/tajo-project/src/site/apt/index.apt +++ b/tajo-project/src/site/apt/index.apt @@ -61,6 +61,8 @@ Features Releases + * <<[2014-10-21]>> {{{./downloads.html}Tajo 0.9.0}} ({{{./releases/0.9.0/announcement.html}Announcement}}, {{{./releases/0.9.0/relnotes.html}Release Notes}}) + * <<[2014-05-01]>> {{{./downloads.html}Tajo 0.8.0}} ({{{./releases/0.8.0/announcement.html}Announcement}}, {{{./releases/0.8.0/relnotes.html}Release Notes}}) * <<[2013-11-20]>> {{{./downloads.html}Tajo 0.2.0-incubating}} ({{{http://s.apache.org/tajo-0.2-release-notes}Release Notes}}) diff --git a/tajo-project/src/site/markdown/downloads.md b/tajo-project/src/site/markdown/downloads.md index cddc38ccbe..0af73ac0da 100644 --- a/tajo-project/src/site/markdown/downloads.md +++ b/tajo-project/src/site/markdown/downloads.md @@ -18,9 +18,19 @@ ## Most Recent Releases -### Release 0.8.0 (May 1, 2014) +### Latest Release 0.9.0 (Oct 21, 2014) + + * [Announcement] (releases/0.9.0/announcement.html) + * [Release Notes] (releases/0.9.0/relnotes.html) + * [User Documentation] (http://tajo.apache.org/docs/0.9.0/index.html) + * Source: [tajo-0.9.0-src.tar.gz](http://apache.org/dyn/closer.cgi/tajo/tajo-0.9.0/tajo-0.9.0-src.tar.gz) ([MD5](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0-src.tar.gz.md5)) ([SHA1](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0-src.tar.gz.sha)) ([SIGN](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0-src.tar.gz.asc)) ([KEYS](http://www.apache.org/dist/tajo/KEYS)) + * Source: [tajo-0.9.0-src.tar.bz2](http://apache.org/dyn/closer.cgi/tajo/tajo-0.9.0/tajo-0.9.0-src.tar.bz2) ([MD5](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0-src.tar.bz2.md5)) ([SHA1](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0-src.tar.bz2.sha)) ([SIGN](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0-src.tar.bz2.asc)) ([KEYS](http://www.apache.org/dist/tajo/KEYS)) + * Source: [tajo-0.9.0-src.zip](http://apache.org/dyn/closer.cgi/tajo/tajo-0.9.0/tajo-0.9.0-src.zip) ([MD5](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0-src.zip.md5)) ([SHA1](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0-src.zip.sha)) ([SIGN](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0-src.zip.asc)) ([KEYS](http://www.apache.org/dist/tajo/KEYS)) + * Binary Tarball: [tajo-0.9.0.tar.gz] (http://apache.org/dyn/closer.cgi/tajo/tajo-0.9.0/tajo-0.9.0.tar.gz) ([MD5](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0.tar.gz.md5)) ([SHA1](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0.tar.gz.sha)) ([SIGN](http://www.apache.org/dist/tajo/tajo-0.9.0/tajo-0.9.0.tar.gz.asc)) ([KEYS](http://www.apache.org/dist/tajo/KEYS)) +### Release 0.8.0 (May 1, 2014) + * [Announcement] (releases/0.8.0/announcement.html) * [Release Notes] (releases/0.8.0/relnotes.html) * [User Documentation] (http://tajo.apache.org/docs/0.8.0/index.html) diff --git a/tajo-project/src/site/markdown/releases/0.9.0/announcement.md b/tajo-project/src/site/markdown/releases/0.9.0/announcement.md new file mode 100644 index 0000000000..fb2633d552 --- /dev/null +++ b/tajo-project/src/site/markdown/releases/0.9.0/announcement.md @@ -0,0 +1,84 @@ + + +# Apache Tajo™ 0.9.0 Release Announcement + +The Apache Tajo team is pleased to announce the release of Apache 0.9.0. +Apache Tajo provides low-latency and scalable SQL query +processing on Hadoop. Due to its advanced design, it is a nice tool at +low-latency ad-hoc queries as well as batch queries. + +The release is available for immediate download: + + * http://tajo.apache.org/downloads.html + +This is a major release. Apache Tajo team resolved 330 issues including +lots of new features, bug fixes, and improvements. Apache Tajo 0.9.0 mainly +focues on advanced SQL features (window function and multiple distinct +aggregation), performance improvements (offheap tuple, skewness handling, +runtime code generation), and scalability on large clusters (improved +shuffle and fetch). + +### Some of highlights + +#### More mature SQL features + * TIMESTAMP, DATE, and TIME Refactoring (TAJO-825) + * INTERVAL type support (TAJO-761) + * WINDOW functions and OVER clause support (TAJO-924) + * ORDER BY and GROUP BY clauses allow column references as well as expressions. + * Multiple distinct aggregation (TAJO-1010) + * ORDER BY NULL FIRST support + * CREATE TABLE LIKE support + * concat() and concat_ws() function support + * to_char() for date time format + * COALESCE() for BOOLEAN, DATE, TIME, and TIMESTAMP + +#### Performance Improvements + * Offheap tuple block and zero-copy tuple (TAJO-907) + * Offheap sort operator for ORDER BY (TAJO-907) + * hash shuffle I/O improvement (TAJO-992) + * Better automatic parallel degree choice for operators involving hash and + range shuffles + * Skewness handling for hash shuffle (TAJO-987) + * Runtime code generation for evaluating expressions (TAJO-906) + * Lots of query optimizer improvements + +#### Hadoop Integration + * Hadoop 2.2.0 or higher (to 2.5.1) support + * Hive Meta Store access support for 0.13.0 and 0.13.1 + * Updated Parquet to 1.5.0. (TAJO-932) + +#### Availability, Reliability, and Stability + * TajoMaster HA (TAJO-704) + * More stable and error-tolerant fetch (TAJO-789, TAJO-953, TAJO-908, TAJO-949, TAJO-991) + +#### Other important improvements + * MariaDB Catalog Store support (TAJO-847) + * Linux and HDFS utility support in tajo shell (TAJO-732) + + +For a complete list of new features and fixed problems, please see the release notes: + + * http://tajo.apache.org/releases/0.9.0/relnotes.html + +### Many Thanks to contributors on the 0.9.0 releases + +Alvin Henrick, DaeMyung Kang, David Chen, Hyoung Jun Kim, Hyunsik Choi, +Ilhyun Suh, JaeHwa Jung, Jae Young Lee, Jihun Kang, Jinho Kim, Jihoon Son, +Jongyoung Park, Mai Hai Thanh, Min Zhou, Keuntae Park, Prafulla T, +Seungun Choe, SeongHwa Ahn, Youngjun Park, and Wan Heo. \ No newline at end of file diff --git a/tajo-project/src/site/resources/releases/0.9.0/relnotes.html b/tajo-project/src/site/resources/releases/0.9.0/relnotes.html new file mode 100644 index 0000000000..01a8570d9a --- /dev/null +++ b/tajo-project/src/site/resources/releases/0.9.0/relnotes.html @@ -0,0 +1,718 @@ + + + + +Release Notes - Tajo - Version 0.9.0 + + + +

Release Notes - Tajo - Version 0.9.0

+ +

Changes since Tajo 0.8.0

+ +

Sub-task +

+
    +
  • [TAJO-215] - Catalog should allow compatible types when finding functions +
    • +
  • [TAJO-218] - HiveQLAnalyzer has to support cast expression. +
    • +
  • [TAJO-517] - Publish Tajo jar to a public maven repository +
    • +
  • [TAJO-529] - Fix warnings in tajo-algebra +
    • +
  • [TAJO-602] - WorkerResourceManager should be broke down into 3 parts +
    • +
  • [TAJO-615] - Implement ADD TABLE RENAME TABLE +
    • +
  • [TAJO-659] - Add Tajo JDBC documentation +
    • +
  • [TAJO-667] - Add math function documentation +
    • +
  • [TAJO-668] - Add datetime function documentation +
    • +
  • [TAJO-669] - Add cluster setup documentation +
    • +
  • [TAJO-696] - Implement ALTER TABLE ADD COLUMN +
    • +
  • [TAJO-697] - Implement ALTER TABLE RENAME COLUMN +
    • +
  • [TAJO-736] - Add table management documentation +
    • +
  • [TAJO-761] - Implements INTERVAL type +
    • +
  • [TAJO-762] - Implements current date/time function +
    • +
  • [TAJO-783] - Remove yarn-related code from tajo-core +
    • +
  • [TAJO-790] - Implements ADD_MONTHS() function +
    • +
  • [TAJO-791] - Implements ADD_DAYS() function +
    • +
  • [TAJO-836] - create index support +
    • +
  • [TAJO-837] - Register index meta information at Catalog +
    • +
  • [TAJO-907] - Implement off-heap tuple block and zero-copy tuple +
    • +
  • [TAJO-924] - Merge the current window_function branch to master +
    • +
  • [TAJO-991] - Running PullServer on a dedicated JVM process which separates from worker. +
    • +
  • [TAJO-992] - Reduce number of hash shuffle output file. +
    • +
  • [TAJO-1008] - Protocol buffer De/Serialization for EvalNode +
    • +
  • [TAJO-1015] - Add executionblock event in worker +
    • +
  • [TAJO-1016] - Refactor worker rpc information +
    • +
  • [TAJO-1038] - Remove use of Builder variable in Schema +
    • +
  • [TAJO-1060] - Apply updated hadoop versions to README and BUILDING files +
    • +
  • [TAJO-1061] - Update build documentation +
    • +
  • [TAJO-1062] - Update TSQL documentation +
    • +
  • [TAJO-1068] - Add SQL Query documentation +
    • +
  • [TAJO-1069] - Add document to explain High Availability support +
    • +
  • [TAJO-1077] - Add Derby configuration documentation +
    • +
  • [TAJO-1096] - Update download source documentation +
    • +
+ +

Bug +

+
    +
  • [TAJO-427] - Empty table makes IndexOutOfBoundsException at LEFT OUTER JOIN clause. +
    • +
  • [TAJO-563] - INSERT OVERWRITE should not remove data before query success +
    • +
  • [TAJO-587] - Query is hanging when OutOfMemoryError occurs in the query master +
    • +
  • [TAJO-590] - Rename HiveConverter to HiveQLAnalyzer +
    • +
  • [TAJO-619] - SELECT count(1) after joins on text keys causes wrong plans +
    • +
  • [TAJO-620] - A join query can cause IndexOutOfBoundsException if one of tables is empty. +
    • +
  • [TAJO-624] - Incorrect progress indication +
    • +
  • [TAJO-628] - The second stage of distinct aggregation can be scheduled to only one node. +
    • +
  • [TAJO-630] - QueryMasterTask never finished when Internal error occurs. +
    • +
  • [TAJO-635] - Improve tests of query semantic verification +
    • +
  • [TAJO-638] - QueryUnitAttempt causes Invalid event error: TA_UPDATE at TA_ASSIGNED +
    • +
  • [TAJO-640] - In inner join clause, empty table can cause a error by order-by clause. +
    • +
  • [TAJO-641] - NPE in HCatalogStore.addTable() +
    • +
  • [TAJO-645] - Task.Reporter can cause NPE during reporting. +
    • +
  • [TAJO-647] - Work unbalance on disk scheduling of DefaultScheduler +
    • +
  • [TAJO-648] - TajoWorker does not send correct QM rpc and client rpc ports via heartbeat. +
    • +
  • [TAJO-650] - Repartitioner::scheduleHashShuffledFetches should adjust the number of tasks +
    • +
  • [TAJO-651] - HcatalogStore should support (de)serialization of RCFile +
    • +
  • [TAJO-652] - logical planner cannot handle alias on partition columns +
    • +
  • [TAJO-653] - RCFileAppender throws IOException +
    • +
  • [TAJO-663] - CREATE TABLE USING RAW doesn't throw ERROR +
    • +
  • [TAJO-672] - Wrong progress status when overwrites to partition table +
    • +
  • [TAJO-674] - ExplainLogicalPlan can cause NPE when a query includes derived tables +
    • +
  • [TAJO-682] - RangePartitionAlgorithm should be improved to handle empty texts +
    • +
  • [TAJO-689] - NoSuchElementException occurs during assigning the leaf tasks +
    • +
  • [TAJO-690] - infinite loop occurs when rack task is assigning +
    • +
  • [TAJO-693] - StatusUpdateTransition in QueryUnitAttempt handles TA_UPDATE incorrectly +
    • +
  • [TAJO-698] - Error occurs when FUNCTION and IN statement are used together. +
    • +
  • [TAJO-701] - Invalid bytes when creating BlobDatum with offset +
    • +
  • [TAJO-705] - CTAS always stores tables with CSV storage type into catalog +
    • +
  • [TAJO-706] - In the case of very quick query, client can't get query status. +
    • +
  • [TAJO-712] - Fix some bugs after database is supported +
    • +
  • [TAJO-713] - Missing INET4 in UniformRangePartition +
    • +
  • [TAJO-716] - Using column names actually aliased in aggregation functions can cause planning error. +
    • +
  • [TAJO-718] - A group-by clause with the same columns but aliased causes planning error. +
    • +
  • [TAJO-719] - JUnit test failures +
    • +
  • [TAJO-729] - PreLogicalPlanVerifier verifies distinct aggregation functions incorrectly. +
    • +
  • [TAJO-738] - NPE occur when failed in QueryMaster's GlobalPlanner.build(). +
    • +
  • [TAJO-739] - A subquery with the same column alias caused planning error. +
    • +
  • [TAJO-741] - GreedyHeuristicJoinOrderAlgorithm removes some join pairs. +
    • +
  • [TAJO-747] - BroadCastJoin omits some data. +
    • +
  • [TAJO-748] - Shuffle output numbers of join may be inconsistent. +
    • +
  • [TAJO-750] - Join orders affects abnormal to the result data. +
    • +
  • [TAJO-754] - failure of INSERT INTO may remove the target table. +
    • +
  • [TAJO-759] - Fix findbug errors added recently. +
    • +
  • [TAJO-763] - Out of range problem in utc_usec_to() +
    • +
  • [TAJO-765] - Incorrect Configuration Classpaths +
    • +
  • [TAJO-766] - Test failures in TestExecExternalShellCommand +
    • +
  • [TAJO-772] - TajoDump cannot dump upper/lower mixed case database names. +
    • +
  • [TAJO-777] - Partition column in function parameter occurs NPE +
    • +
  • [TAJO-778] - TPC-DS Q34 occurs NPE +
    • +
  • [TAJO-779] - TPC-DS Q46 occurs NPE +
    • +
  • [TAJO-786] - TajoDataMetaDatabase::getSchemas creates invalid MetaDataTuple +
    • +
  • [TAJO-787] - FilterPushDownRule::visitSubQuery does not consider aliased columns. +
    • +
  • [TAJO-792] - Insert table with a qualified target table name can cause error. +
    • +
  • [TAJO-795] - PlannerUtil::joinJoinKeyForEachTable need to handle theta-join. +
    • +
  • [TAJO-799] - Local query without FROM throws IllegalArgumentException in CLI +
    • +
  • [TAJO-800] - CLI's meta command should be aware "TABLE_NAME" style. +
    • +
  • [TAJO-802] - No partition columns in WEB catalog page. +
    • +
  • [TAJO-803] - INSERT INTO without FROM throws ClassCastException. +
    • +
  • [TAJO-805] - Multiple constant in selection emits some columns. +
    • +
  • [TAJO-806] - CreateTableNode in CTAS uses a wrong schema as output schema and table schema. +
    • +
  • [TAJO-808] - Fix pre-commit build failure +
    • +
  • [TAJO-812] - Some methods of TajoDatabaseMetaData should result in an empty tuple list instead of SQLFeatureNotSupportedException. +
    • +
  • [TAJO-813] - CLI should support comment character with multi-line query. +
    • +
  • [TAJO-816] - NULL delimiter doesn't apply with HCatalogStore +
    • +
  • [TAJO-819] - KillQuery does not work query occasionally. +
    • +
  • [TAJO-821] - IllegalStateException occurs when a NettyClientBase object is created within single thread. +
    • +
  • [TAJO-823] - Missing INET4 handling in DatumFactory.cast() +
    • +
  • [TAJO-827] - SUM() overflow in the case of INT4 +
    • +
  • [TAJO-829] - Same constants in groupby clause may cause planning error. +
    • +
  • [TAJO-830] - Some filter conditions with a SUBQUERY are removed by optimizer. +
    • +
  • [TAJO-832] - NPE occurs when Exception's message is null in Task. +
    • +
  • [TAJO-833] - NPE occurs when using the column as a alias name in the multiple DISTINCT. +
    • +
  • [TAJO-848] - PreLogicalPlanVerifier::visitInsert need to find smaller expressions than target columns for a partitioned table. +
    • +
  • [TAJO-850] - OUTER JOIN does not properly handle a NULL. +
    • +
  • [TAJO-851] - Timestamp type test of TestSQLExpression::testCastFromTable fails in jenkins CI test +
    • +
  • [TAJO-852] - Integration test using HCatalog as a catalog store is failed +
    • +
  • [TAJO-861] - tajo-dump script are not executable +
    • +
  • [TAJO-862] - Restore failure of dumped relations +
    • +
  • [TAJO-863] - Column order mismatched in the JOIN query with asterisk selection. +
    • +
  • [TAJO-864] - JUnit test failure at TestTimestampDatum.testTimestampConstructor() +
    • +
  • [TAJO-866] - COUNT DISTINCT with other aggregation function throws ClassCastException. +
    • +
  • [TAJO-867] - OUTER JOIN with empty result subquery produces a wrong result. +
    • +
  • [TAJO-868] - TestDateTimeFunctions unit test is occasionally failed +
    • +
  • [TAJO-869] - Sometimes, the unit test of testTaskRunnerHistory is failed. +
    • +
  • [TAJO-870] - FilterPushDown ignores a partitioned column in CASE expression within WHERE clause +
    • +
  • [TAJO-872] - NOW() function has a different value on each task. +
    • +
  • [TAJO-873] - Query status is still RUNNING after session expired. +
    • +
  • [TAJO-874] - Sometimes InvalidOperationException occurs when aggregates TableStat. +
    • +
  • [TAJO-879] - Some data is missing in the case of BROADCAST JOIN and multi-column partition. +
    • +
  • [TAJO-880] - NULL in CASE clause occurs Exception. +
    • +
  • [TAJO-881] - JOIN with union query occurs NPE +
    • +
  • [TAJO-882] - CLI hangs when a error occurs in the GlobalPlanner. +
    • +
  • [TAJO-884] - complex join conditions should be supported in ON clause +
    • +
  • [TAJO-891] - Complex join conditions with UNION or inline should be supported +
    • +
  • [TAJO-894] - Left outer join with partitioned large table and small table returns empty result. +
    • +
  • [TAJO-896] - Full outer join query with empty intermediate data doesn't terminate. +
    • +
  • [TAJO-897] - PartitionedTableRewriter is repeated several times with same table. +
    • +
  • [TAJO-898] - Left outer join with union returns empty result. +
    • +
  • [TAJO-899] - Nested now() has different value for each task +
    • +
  • [TAJO-902] - Unicode delimiter does not work correctly +
    • +
  • [TAJO-904] - ORDER BY Null first support +
    • +
  • [TAJO-905] - When to_date() parses some date without day, the result will be wrong. +
    • +
  • [TAJO-908] - Fetcher does not retry, when pull server connection was closed +
    • +
  • [TAJO-909] - {SortBased, Col}PartitionStoreExec should not write partition keys to files. +
    • +
  • [TAJO-912] - Tsql prints wrong version. +
    • +
  • [TAJO-913] - Add some missed tests for constant value group-by keys +
    • +
  • [TAJO-914] - join queries with constant values can cause schema mismatch in logical plan +
    • +
  • [TAJO-916] - SubQuery::computeStatFromTasks occasionally fail. +
    • +
  • [TAJO-917] - Using alias name which is same to existing column names causes error +
    • +
  • [TAJO-925] - Child ExecutionBlock of JOIN node has different number of shuffle keys. +
    • +
  • [TAJO-926] - Join condition including column references of a row-preserving table in left outer join causes incorrect result +
    • +
  • [TAJO-927] - Broadcast Join with Large, Small, Large, Small tables makes a wrong plan. +
    • +
  • [TAJO-929] - Broadcast join with empty outer join table returns empty result. +
    • +
  • [TAJO-934] - Multiple DISTINCT returns null grouping key value. +
    • +
  • [TAJO-936] - TestStorages::testSplitable is failed occasionally. +
    • +
  • [TAJO-939] - Refactoring the column resolver in LogicalPlan +
    • +
  • [TAJO-945] - Connecting to Tajo by JDBC driver failed with SQL Exception "Invalid JDBC URI" +
    • +
  • [TAJO-947] - ColPartitionStoreExec can cause URISyntaxException due to special characters +
    • +
  • [TAJO-948] - 'INSERT INTO' statement to non existence table casuses NPE. +
    • +
  • [TAJO-949] - PullServer does not release files, when a channel throws an internal exception +
    • +
  • [TAJO-952] - Wrong default partition volume config +
    • +
  • [TAJO-957] - ROUND should be support INT parameter. +
    • +
  • [TAJO-960] - TajoCli's problem does not show the current status +
    • +
  • [TAJO-961] - TajoCli should exit when at least one query faces error while executing a SQL script. +
    • +
  • [TAJO-962] - Column reference used in LIMIT clause incurs NPE. +
    • +
  • [TAJO-965] - Upgrade Bytes class and move some methods to others +
    • +
  • [TAJO-968] - Self-Join query (including partitioned table) doesn't run unexpectedly using auto broadcast join. +
    • +
  • [TAJO-969] - Distributed sort on a large data set may result in incorrect results. +
    • +
  • [TAJO-972] - Broadcast join with left outer join returns duplicated rows. +
    • +
  • [TAJO-974] - Eliminate unexpected case condition in SubQuery +
    • +
  • [TAJO-975] - alias name which is the same to existing column name may cause NPE during PPD +
    • +
  • [TAJO-977] - INSERT into a partitioned table as SELECT statement uses a wrong schema. +
    • +
  • [TAJO-978] - RoundFloat8 should return Float8Datum type. +
    • +
  • [TAJO-979] - Dividing float value by zero should throw "Divide by zero Exception" +
    • +
  • [TAJO-980] - execution page in Web UI broken +
    • +
  • [TAJO-981] - Help command (\?) in tsql takes too long time. +
    • +
  • [TAJO-985] - Client API should be non-blocking +
    • +
  • [TAJO-994] - 'count(distinct x)' function counts first null value. +
    • +
  • [TAJO-995] - HiveMetaStoreClient wrapper should retry the connection +
    • +
  • [TAJO-996] - Sometimes, scheduleFetchesByEvenDistributedVolumes loses some FetchImpls +
    • +
  • [TAJO-999] - SequenceFile key class need to be compatible. +
    • +
  • [TAJO-1000] - TextDatum.asChar() is incorrect, if client charset is different +
    • +
  • [TAJO-1004] - UniformRangePartition cannot deal with unicode ranges +
    • +
  • [TAJO-1006] - Fix wrong storage unit for kilo bytes and others. +
    • +
  • [TAJO-1009] - A binary eval for column references of the same tables should not be recognized as a join condition +
    • +
  • [TAJO-1013] - A complex equality condition including columns of the same table is recognized as a join condition +
    • +
  • [TAJO-1017] - TajoConf misuses read & write locks in some functions +
    • +
  • [TAJO-1020] - TajoContainerProxy::assignExecutionBlock causes NPE by race condition. +
    • +
  • [TAJO-1021] - Remove the member variable Builder from all classes inherited from ProtoObject. +
    • +
  • [TAJO-1022] - tsql does not work as background process +
    • +
  • [TAJO-1024] - RpcConnectionPool::getConnection can cause NPE at initialization +
    • +
  • [TAJO-1025] - Network disconnection during query processing can cause infinite exceptions +
    • +
  • [TAJO-1029] - TAJO_PULLSERVER_STANDALONE should be false in default tajo-env.sh +
    • +
  • [TAJO-1037] - KillQuery hang in subquery init state +
    • +
  • [TAJO-1047] - DefaultTaskScheduler:allocateRackTask is failed occasionally on JDK 1.7 +
    • +
  • [TAJO-1048] - Missed use of session variables in GlobalPlanner +
    • +
  • [TAJO-1050] - RPC client does not retry during connecting +
    • +
  • [TAJO-1056] - Wrong resource release or wrong task scheduling +
    • +
  • [TAJO-1065] - The \admin -cluster argument doesn't run as expected. +
    • +
  • [TAJO-1067] - INSERT OVERWRITE INTO should not remove all partitions. +
    • +
  • [TAJO-1072] - CLI gets stuck when wrong host/port is provided +
    • +
  • [TAJO-1074] - Query calculates wrong progress before subquery init +
    • +
  • [TAJO-1081] - Non-forwarded (simple) query shows wrong rows. +
    • +
  • [TAJO-1097] - IllegalArgumentException: RawFileScanner +
    • +
  • [TAJO-1098] - LogicalPlanVerifier should validate operations within CASE WHEN clauses. +
    • +
  • [TAJO-1099] - LogicalPlanner::convertDataType causes NPE in some cases. +
    • +
  • [TAJO-1101] - Broadcast join with a zero-length file table returns wrong result data. +
    • +
  • [TAJO-1102] - Self-join with a partitioned table returns wrong result data. +
    • +
  • [TAJO-1103] - Insert clause of partitioned table loses some FetchImpls +
    • +
  • [TAJO-1104] - Using asterisk with GROUP BY causes NPE. +
    • +
  • [TAJO-1106] - Missing session check in getFinishedQuery API +
    • +
  • [TAJO-1107] - Broadcast join on non-leaf node scans only first data file. +
    • +
  • [TAJO-1110] - JAVA_PULLSERVER_HEAP_MAX in bin/tajo should be increased +
    • +
  • [TAJO-1111] - TestKillQuery.testKillQueryFromInitState occasionally fails +
    • +
  • [TAJO-1113] - SubQuery in KILLED state should handle unexpected events. +
    • +
+ +

Improvement +

+
    +
  • [TAJO-153] - Proto(Async|Blocking)RpcClient should retry to connect a server when failed. +
    • +
  • [TAJO-196] - Add EngineContext to contain resource information about worker +
    • +
  • [TAJO-356] - Improve TajoClient to directly get query results in the first request +
    • +
  • [TAJO-425] - RAWFILE_SYNC_INTERVAL has not default value. +
    • +
  • [TAJO-589] - Add fine grained progress indicator for each task +
    • +
  • [TAJO-614] - Explaning a logical node should use ExplainLogicalPlanVisitor. +
    • +
  • [TAJO-616] - SequenceFile support +
    • +
  • [TAJO-617] - Rename BIN/tajo_dump BIN/tajo-dump +
    • +
  • [TAJO-634] - ExecutionBlock must be sorted by start time in querydetail.jsp +
    • +
  • [TAJO-644] - Support quoted identifiers +
    • +
  • [TAJO-654] - Separate TajoWorker into yarn worker and standby worker +
    • +
  • [TAJO-662] - The tasks of CTAS on a partitioned table should be fine grained +
    • +
  • [TAJO-665] - sort buffer size must be dealt as long type values. +
    • +
  • [TAJO-670] - Change daemon's hostname to canonical hostname +
    • +
  • [TAJO-673] - Assign proper number of tasks when inserting into partitioned table +
    • +
  • [TAJO-675] - maximum frame size of frameDecoder should be increased +
    • +
  • [TAJO-691] - HashJoin or HashAggregation is too slow if there is many unique keys +
    • +
  • [TAJO-699] - Create a table using LIKE +
    • +
  • [TAJO-709] - Add .reviewboardrc and use rbt instead of post-review +
    • +
  • [TAJO-714] - Enable setting Parquet tuning parameters +
    • +
  • [TAJO-715] - hadoop version upgrade to 2.3.0 +
    • +
  • [TAJO-717] - Improve file splitting for large number of splits +
    • +
  • [TAJO-725] - Broadcast JOIN should supports multiple tables +
    • +
  • [TAJO-728] - Supports expressions in 'IN predicate' +
    • +
  • [TAJO-732] - Support executing LINUX shell command and HDFS command. +
    • +
  • [TAJO-734] - Arrange TajoCli output message. +
    • +
  • [TAJO-735] - Remove multiple SLF4J bindings message. +
    • +
  • [TAJO-737] - Change version message when daemon starts up. +
    • +
  • [TAJO-743] - Change the default resource allocation policy of leaf tasks +
    • +
  • [TAJO-745] - APIs in TajoClient and JDBC should be case sensitive. +
    • +
  • [TAJO-755] - ALTER TABLESPACE LOCATION support +
    • +
  • [TAJO-758] - Supports parameter values in the SQL file. +
    • +
  • [TAJO-768] - Improve the log4j configuration +
    • +
  • [TAJO-769] - A minor improvements for HCatalogStore +
    • +
  • [TAJO-789] - Improve shuffle URI +
    • +
  • [TAJO-793] - CLI should be able to exit when single query is failed. +
    • +
  • [TAJO-797] - Implicit type conversion support +
    • +
  • [TAJO-801] - Multiple distinct should be supported. +
    • +
  • [TAJO-804] - Bump up Parquet version to 1.4.2 +
    • +
  • [TAJO-807] - Implement Round(numeric, int) function. +
    • +
  • [TAJO-811] - add simple fifo scheduler support +
    • +
  • [TAJO-824] - Improve SimpleParser to handle JSON statements +
    • +
  • [TAJO-840] - Improve query result print with counting empty table. +
    • +
  • [TAJO-842] - NULL handling in JDBC. +
    • +
  • [TAJO-843] - implements COALESCE for BOOLEAN, DATE, TIME, TIMESTAMP +
    • +
  • [TAJO-844] - JDBC should be support getTime, getDate, and getTimestamp. +
    • +
  • [TAJO-846] - Clean up the task history in woker +
    • +
  • [TAJO-853] - Refactoring FilterPushDown for OUTER JOIN +
    • +
  • [TAJO-895] - ConstEval should not be included in target list of projectable nodes +
    • +
  • [TAJO-900] - Reducing memory usage during query processing +
    • +
  • [TAJO-903] - Some left outer join cases are not optimized as the broadcast join. +
    • +
  • [TAJO-906] - Runtime code generation for evaluating expression trees +
    • +
  • [TAJO-910] - Simple query (non-forwarded query) should be supported against partition tables. +
    • +
  • [TAJO-911] - Refactoring Mysql/Maria Catalog Store +
    • +
  • [TAJO-928] - Session variables should override query configs in TajoConf. +
    • +
  • [TAJO-931] - Output file can be punctuated depending on the file size. +
    • +
  • [TAJO-932] - Upgrade Parquet to 1.5.0. +
    • +
  • [TAJO-933] - Fork some classes of Parquet as builtin third-party classes +
    • +
  • [TAJO-937] - Should use tajo.util.VersionInfo instead of TajoConstants.TAJO_VERSION +
    • +
  • [TAJO-953] - RawFile should release a DirectBuffer immediately +
    • +
  • [TAJO-956] - CONCAT should be support multiple params and null param. +
    • +
  • [TAJO-966] - Range partition should support split of multiple characters. +
    • +
  • [TAJO-983] - Worker should directly read Intermediate data stored in localhost rather than fetching +
    • +
  • [TAJO-984] - Improve the default data type handling in RowStoreUtil +
    • +
  • [TAJO-987] - Hash shuffle should be balanced according to intermediate volumes +
    • +
  • [TAJO-989] - Cleanup of child blocks after parent execution block is complete +
    • +
  • [TAJO-990] - Implement a tool to find tajo configurations. +
    • +
  • [TAJO-1010] - Improve multiple DISTINCT aggregation. +
    • +
  • [TAJO-1027] - Upgrade Hive to 0.13.0 and 0.13.1 +
    • +
  • [TAJO-1028] - JDBC should support SET command. +
    • +
  • [TAJO-1030] - Not supported JDBC APIs should return empty results instead of Exception +
    • +
  • [TAJO-1034] - Reduce Explicit Use of JVM Internal Class +
    • +
  • [TAJO-1040] - Misuse netty HashedWheelTimer +
    • +
  • [TAJO-1046] - Remove hadoop native dependency of pullserver +
    • +
  • [TAJO-1049] - Remove the parallel degree limit up to the maximum cluster capacity +
    • +
  • [TAJO-1052] - (Umbrella) Add and Update user documentation for 0.9.0 +
    • +
  • [TAJO-1071] - should be possible to get long query results with no prompt +
    • +
  • [TAJO-1093] - DateTimeFormat.to_char() is slower than SimpleDateFormat.format() +
    • +
+ +

New Feature +

+
    +
  • [TAJO-20] - INSERT INTO ... SELECT +
    • +
  • [TAJO-30] - Parquet Integration +
    • +
  • [TAJO-353] - Add Database support to Tajo +
    • +
  • [TAJO-377] - Implement concat function +
    • +
  • [TAJO-378] - Implement concat_ws function. +
    • +
  • [TAJO-480] - Umbrella Jira for adding ALTER TABLE statement +
    • +
  • [TAJO-704] - TajoMaster HA +
    • +
  • [TAJO-711] - Add Avro storage support +
    • +
  • [TAJO-847] - Supporting MariaDB-based Store, which is compatible with MySQL. +
    • +
  • [TAJO-849] - Add Parquet storage to HCatalogStore +
    • +
  • [TAJO-860] - Implements TRUNCATE table. +
    • +
  • [TAJO-1105] - Add thread which detects JVM pauses like HADOOP's +
    • +
+ +

Task +

+
    +
  • [TAJO-605] - Rename Options to KeyValueList +
    • +
  • [TAJO-621] - Add DOAP file for Tajo +
    • +
  • [TAJO-632] - add intellij idea projects files into git ignore +
    • +
  • [TAJO-642] - Change tajo documentation tool to sphinx +
    • +
  • [TAJO-657] - Missing table stat in RCFile +
    • +
  • [TAJO-681] - Embed sphinx rtd theme into tajo-docs +
    • +
  • [TAJO-694] - Bump up hadoop to 2.3.0 +
    • +
  • [TAJO-700] - Update site, wikis, pom.xml and other resources to point to the new repository location +
    • +
  • [TAJO-730] - Update Tajo site to reflect graduation +
    • +
  • [TAJO-752] - Escalate sub modules in tajo-core into the top-level modules +
    • +
  • [TAJO-753] - Clean up of maven dependencies +
    • +
  • [TAJO-788] - Update Tajo documentation and README, and BUILDING +
    • +
  • [TAJO-810] - Update Tajo site for 0.8.0 release +
    • +
  • [TAJO-814] - Set up Travis CI builds +
    • +
  • [TAJO-817] - tajo-core should not skip deploy. +
    • +
  • [TAJO-820] - Add missing license header to 0.8.0 release announcement. +
    • +
  • [TAJO-834] - Add Travis notification to issues@tajo.a.o and IRC. +
    • +
  • [TAJO-859] - Update site for new committer Alvin Henrick +
    • +
  • [TAJO-886] - Add IRC page to community section in site. +
    • +
  • [TAJO-887] - Eliminate HiveQL support feature +
    • +
  • [TAJO-890] - Redirect stdout of maven test to /dev/null in Travis CI script +
    • +
  • [TAJO-1001] - Add missed postgresql license to NOTICE.txt and LICENSE.txt +
    • +
  • [TAJO-1007] - Update site for new committer and new contributors +
    • +
  • [TAJO-1054] - Wrong comment in ByteUtils.splitWorker() +
    • +
  • [TAJO-1070] - BSTIndexScanExec should not seek a negative offset +
    • +
  • [TAJO-1078] - Update contributor list +
    • +
+ +

Test +

+
    +
  • [TAJO-686] - Integration test aborted +
    • +
+ + + diff --git a/tajo-project/src/site/site.xml b/tajo-project/src/site/site.xml index 3ffd57e9a9..d63f55b653 100644 --- a/tajo-project/src/site/site.xml +++ b/tajo-project/src/site/site.xml @@ -90,7 +90,7 @@ - + @@ -104,9 +104,10 @@ - - - + + + + From b636e8b8be623a87d291d7be7aa1dd645502469f Mon Sep 17 00:00:00 2001 From: HyoungJun Kim Date: Wed, 22 Oct 2014 23:46:59 +0900 Subject: [PATCH 07/15] TAJO-1121: Remove the 'v2' storage package. Closes #203 --- CHANGES | 1 + .../java/org/apache/tajo/conf/TajoConf.java | 7 - .../engine/planner/PhysicalPlannerImpl.java | 6 +- .../planner/physical/BSTIndexScanExec.java | 4 +- .../physical/ColPartitionStoreExec.java | 4 +- .../planner/physical/ExternalSortExec.java | 6 +- .../physical/HashShuffleFileWriteExec.java | 4 +- .../physical/PartitionMergeScanExec.java | 6 +- .../physical/RangeShuffleFileWriteExec.java | 4 +- .../engine/planner/physical/SeqScanExec.java | 4 +- .../planner/physical/StoreTableExec.java | 6 +- .../org/apache/tajo/master/GlobalEngine.java | 2 +- .../master/NonForwardQueryResultScanner.java | 4 +- .../org/apache/tajo/master/TajoMaster.java | 11 +- .../apache/tajo/master/querymaster/Query.java | 4 +- .../tajo/master/querymaster/QueryMaster.java | 9 +- .../master/querymaster/QueryMasterTask.java | 4 +- .../master/querymaster/Repartitioner.java | 6 +- .../tajo/master/querymaster/SubQuery.java | 9 +- .../apache/tajo/worker/TajoQueryEngine.java | 7 +- .../tajo/worker/WorkerHeartbeatService.java | 6 +- .../org/apache/tajo/BackendTestingUtil.java | 2 +- .../planner/global/TestBroadcastJoinPlan.java | 4 +- .../planner/physical/TestBNLJoinExec.java | 8 +- .../planner/physical/TestBSTIndexExec.java | 8 +- .../physical/TestExternalSortExec.java | 6 +- .../physical/TestFullOuterHashJoinExec.java | 12 +- .../physical/TestFullOuterMergeJoinExec.java | 14 +- .../physical/TestHashAntiJoinExec.java | 8 +- .../planner/physical/TestHashJoinExec.java | 8 +- .../physical/TestHashSemiJoinExec.java | 8 +- .../physical/TestLeftOuterHashJoinExec.java | 12 +- .../physical/TestLeftOuterNLJoinExec.java | 12 +- .../planner/physical/TestMergeJoinExec.java | 8 +- .../planner/physical/TestNLJoinExec.java | 8 +- .../planner/physical/TestPhysicalPlanner.java | 18 +- .../TestProgressExternalSortExec.java | 6 +- .../physical/TestRightOuterHashJoinExec.java | 10 +- .../physical/TestRightOuterMergeJoinExec.java | 14 +- .../engine/planner/physical/TestSortExec.java | 6 +- .../tajo/engine/query/TestJoinBroadcast.java | 2 +- .../org/apache/tajo/jdbc/TestResultSet.java | 6 +- .../tajo/master/TestExecutionBlockCursor.java | 5 +- .../org/apache/tajo/storage/TestRowFile.java | 6 +- .../worker/TestRangeRetrieverHandler.java | 8 +- .../tajo/storage/AbstractStorageManager.java | 729 ------- .../tajo/storage/{v2 => }/DiskDeviceInfo.java | 2 +- .../tajo/storage/{v2 => }/DiskInfo.java | 2 +- .../tajo/storage/{v2 => }/DiskMountInfo.java | 2 +- .../tajo/storage/{v2 => }/DiskUtil.java | 2 +- .../storage/HashShuffleAppenderManager.java | 3 +- .../org/apache/tajo/storage/MergeScanner.java | 2 +- .../apache/tajo/storage/StorageManager.java | 743 ++++++- .../tajo/storage/StorageManagerFactory.java | 98 - .../tajo/storage/v2/CSVFileScanner.java | 386 ---- .../storage/v2/DiskFileScanScheduler.java | 205 -- .../tajo/storage/v2/FileScanRunner.java | 70 - .../apache/tajo/storage/v2/FileScannerV2.java | 234 --- .../org/apache/tajo/storage/v2/RCFile.java | 1818 ----------------- .../apache/tajo/storage/v2/RCFileScanner.java | 300 --- .../apache/tajo/storage/v2/ScanScheduler.java | 192 -- .../tajo/storage/v2/ScheduledInputStream.java | 513 ----- .../tajo/storage/v2/StorageManagerV2.java | 152 -- .../tajo/storage/TestCompressionStorages.java | 10 +- .../apache/tajo/storage/TestFileSystems.java | 4 +- .../apache/tajo/storage/TestMergeScanner.java | 8 +- .../tajo/storage/TestStorageManager.java | 12 +- .../org/apache/tajo/storage/TestStorages.java | 48 +- .../tajo/storage/index/TestBSTIndex.java | 52 +- .../index/TestSingleCSVFileBSTIndex.java | 4 +- .../tajo/storage/v2/TestCSVCompression.java | 213 -- .../tajo/storage/v2/TestCSVScanner.java | 168 -- .../apache/tajo/storage/v2/TestStorages.java | 292 --- .../src/test/resources/storage-default.xml | 57 - 74 files changed, 966 insertions(+), 5668 deletions(-) delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java rename tajo-storage/src/main/java/org/apache/tajo/storage/{v2 => }/DiskDeviceInfo.java (97%) rename tajo-storage/src/main/java/org/apache/tajo/storage/{v2 => }/DiskInfo.java (97%) rename tajo-storage/src/main/java/org/apache/tajo/storage/{v2 => }/DiskMountInfo.java (98%) rename tajo-storage/src/main/java/org/apache/tajo/storage/{v2 => }/DiskUtil.java (99%) delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/v2/FileScanRunner.java delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/v2/ScheduledInputStream.java delete mode 100644 tajo-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java delete mode 100644 tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java delete mode 100644 tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java delete mode 100644 tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java diff --git a/CHANGES b/CHANGES index 3d44a7cb2d..e498d9e3e3 100644 --- a/CHANGES +++ b/CHANGES @@ -14,6 +14,7 @@ Release 0.9.1 - unreleased TAJO-1092: Improve the function system to allow other function implementation types. (hyunsik) + TAJO-1121: Remove the 'v2' storage package. (Hyoungjun Kim) BUG FIXES diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 66d3030f43..181ef2e536 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -233,13 +233,6 @@ public static enum ConfVars implements ConfigKey { // for RCFile HIVEUSEEXPLICITRCFILEHEADER("tajo.exec.rcfile.use.explicit.header", true), - // for Storage Manager v2 - STORAGE_MANAGER_VERSION_2("tajo.storage-manager.v2", false), - STORAGE_MANAGER_DISK_SCHEDULER_MAX_READ_BYTES_PER_SLOT("tajo.storage-manager.max-read-bytes", 8 * 1024 * 1024), - STORAGE_MANAGER_DISK_SCHEDULER_REPORT_INTERVAL("tajo.storage-manager.disk-scheduler.report-interval", 60 * 1000), - STORAGE_MANAGER_CONCURRENCY_PER_DISK("tajo.storage-manager.disk-scheduler.per-disk-concurrency", 2), - - // RPC -------------------------------------------------------------------- RPC_POOL_MAX_IDLE("tajo.rpc.pool.idle.max", 10), diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 6b1c65cd16..485677a41d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -45,8 +45,8 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray; -import org.apache.tajo.storage.AbstractStorageManager; import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -75,9 +75,9 @@ public class PhysicalPlannerImpl implements PhysicalPlanner { private static final int UNGENERATED_PID = -1; protected final TajoConf conf; - protected final AbstractStorageManager sm; + protected final StorageManager sm; - public PhysicalPlannerImpl(final TajoConf conf, final AbstractStorageManager sm) { + public PhysicalPlannerImpl(final TajoConf conf, final StorageManager sm) { this.conf = conf; this.sm = sm; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index bdd1e07394..51f70a0b7e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -48,7 +48,7 @@ public class BSTIndexScanExec extends PhysicalExec { private float progress; public BSTIndexScanExec(TaskAttemptContext context, - AbstractStorageManager sm , ScanNode scanNode , + StorageManager sm , ScanNode scanNode , FileFragment fragment, Path fileName , Schema keySchema, TupleComparator comparator , Datum[] datum) throws IOException { super(context, scanNode.getInSchema(), scanNode.getOutSchema()); @@ -56,7 +56,7 @@ public BSTIndexScanExec(TaskAttemptContext context, this.qual = scanNode.getQual(); this.datum = datum; - this.fileScanner = StorageManagerFactory.getSeekableScanner(context.getConf(), + this.fileScanner = StorageManager.getSeekableScanner(context.getConf(), scanNode.getTableDesc().getMeta(), scanNode.getInSchema(), fragment, outSchema); this.fileScanner.init(); this.projector = new Projector(context, inSchema, outSchema, scanNode.getTargets()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index 92738e5e59..a07290401c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -35,7 +35,7 @@ import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.engine.planner.logical.StoreTableNode; import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.StorageManagerFactory; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.worker.TaskAttemptContext; @@ -160,7 +160,7 @@ public void openAppender(int suffixId) throws IOException { actualFilePath = new Path(lastFileName + "_" + suffixId); } - appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, actualFilePath); + appender = StorageManager.getStorageManager(context.getConf()).getAppender(meta, outSchema, actualFilePath); appender.enableStats(); appender.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java index 00945906af..cfd7fb784d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java @@ -106,7 +106,7 @@ public class ExternalSortExec extends SortExec { /** total bytes of input data */ private long sortAndStoredBytes; - private ExternalSortExec(final TaskAttemptContext context, final AbstractStorageManager sm, final SortNode plan) + private ExternalSortExec(final TaskAttemptContext context, final StorageManager sm, final SortNode plan) throws PhysicalPlanningException { super(context, plan.getInSchema(), plan.getOutSchema(), null, plan.getSortKeys()); @@ -129,7 +129,7 @@ private ExternalSortExec(final TaskAttemptContext context, final AbstractStorage } public ExternalSortExec(final TaskAttemptContext context, - final AbstractStorageManager sm, final SortNode plan, + final StorageManager sm, final SortNode plan, final CatalogProtos.FragmentProto[] fragments) throws PhysicalPlanningException { this(context, sm, plan); @@ -141,7 +141,7 @@ public ExternalSortExec(final TaskAttemptContext context, } public ExternalSortExec(final TaskAttemptContext context, - final AbstractStorageManager sm, final SortNode plan, final PhysicalExec child) + final StorageManager sm, final SortNode plan, final PhysicalExec child) throws IOException { this(context, sm, plan); setChild(child); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java index cee2b77ced..47fcb8d71f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java @@ -27,7 +27,7 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.logical.ShuffleFileWriteNode; -import org.apache.tajo.storage.AbstractStorageManager; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.HashShuffleAppender; import org.apache.tajo.storage.HashShuffleAppenderManager; import org.apache.tajo.storage.Tuple; @@ -55,7 +55,7 @@ public final class HashShuffleFileWriteExec extends UnaryPhysicalExec { private HashShuffleAppenderManager hashShuffleAppenderManager; private int numHashShuffleBufferTuples; - public HashShuffleFileWriteExec(TaskAttemptContext context, final AbstractStorageManager sm, + public HashShuffleFileWriteExec(TaskAttemptContext context, final StorageManager sm, final ShuffleFileWriteNode plan, final PhysicalExec child) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema(), child); Preconditions.checkArgument(plan.hasShuffleKeys()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java index d87a30debc..4c72075c79 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java @@ -23,7 +23,7 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.logical.ScanNode; -import org.apache.tajo.storage.AbstractStorageManager; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.worker.TaskAttemptContext; @@ -44,12 +44,12 @@ public class PartitionMergeScanExec extends PhysicalExec { private List scanners = Lists.newArrayList(); private Iterator iterator; - private AbstractStorageManager sm; + private StorageManager sm; private float progress; protected TableStats inputStats; - public PartitionMergeScanExec(TaskAttemptContext context, AbstractStorageManager sm, + public PartitionMergeScanExec(TaskAttemptContext context, StorageManager sm, ScanNode plan, CatalogProtos.FragmentProto[] fragments) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema()); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java index 68379d161c..4e5d1cfac2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java @@ -51,7 +51,7 @@ public class RangeShuffleFileWriteExec extends UnaryPhysicalExec { private FileAppender appender; private TableMeta meta; - public RangeShuffleFileWriteExec(final TaskAttemptContext context, final AbstractStorageManager sm, + public RangeShuffleFileWriteExec(final TaskAttemptContext context, final StorageManager sm, final PhysicalExec child, final Schema inSchema, final Schema outSchema, final SortSpec[] sortSpecs) throws IOException { super(context, inSchema, outSchema, child); @@ -78,7 +78,7 @@ public void init() throws IOException { context.getDataChannel().getStoreType() : CatalogProtos.StoreType.RAW); FileSystem fs = new RawLocalFileSystem(); fs.mkdirs(storeTablePath); - this.appender = (FileAppender) StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, + this.appender = (FileAppender) StorageManager.getStorageManager(context.getConf()).getAppender(meta, outSchema, new Path(storeTablePath, "output")); this.appender.enableStats(); this.appender.init(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index 2f0c12fc7f..c7f8e2dd9d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -67,7 +67,7 @@ public class SeqScanExec extends PhysicalExec { private boolean cacheRead = false; - public SeqScanExec(TaskAttemptContext context, AbstractStorageManager sm, ScanNode plan, + public SeqScanExec(TaskAttemptContext context, StorageManager sm, ScanNode plan, CatalogProtos.FragmentProto [] fragments) throws IOException { super(context, plan.getInSchema(), plan.getOutSchema()); @@ -218,7 +218,7 @@ private void initScanner(Schema projected) throws IOException { fragments), projected ); } else { - this.scanner = StorageManagerFactory.getStorageManager( + this.scanner = StorageManager.getStorageManager( context.getConf()).getScanner(plan.getTableDesc().getMeta(), plan.getPhysicalSchema(), fragments[0], projected); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index fd0c04fceb..b496e42c9d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -29,7 +29,7 @@ import org.apache.tajo.engine.planner.logical.InsertNode; import org.apache.tajo.engine.planner.logical.PersistentStoreNode; import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.StorageManagerFactory; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.worker.TaskAttemptContext; @@ -89,10 +89,10 @@ public void openNewFile(int suffixId) throws IOException { if (plan instanceof InsertNode) { InsertNode createTableNode = (InsertNode) plan; - appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, + appender = StorageManager.getStorageManager(context.getConf()).getAppender(meta, createTableNode.getTableSchema(), context.getOutputPath()); } else { - appender = StorageManagerFactory.getStorageManager(context.getConf()).getAppender(meta, outSchema, lastFileName); + appender = StorageManager.getStorageManager(context.getConf()).getAppender(meta, outSchema, lastFileName); } appender.enableStats(); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 342f8d7a2f..382b789207 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -79,7 +79,7 @@ public class GlobalEngine extends AbstractService { private final static Log LOG = LogFactory.getLog(GlobalEngine.class); private final MasterContext context; - private final AbstractStorageManager sm; + private final StorageManager sm; private SQLAnalyzer analyzer; private CatalogService catalog; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java index f8c51fdf7f..c0bd8420ba 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java @@ -33,7 +33,7 @@ import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; -import org.apache.tajo.storage.StorageManagerFactory; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.worker.TaskAttemptContext; @@ -88,7 +88,7 @@ private void initSeqScanExec() throws IOException { try { // scanNode must be clone cause SeqScanExec change target in the case of a partitioned table. scanExec = new SeqScanExec(taskContext, - StorageManagerFactory.getStorageManager(tajoConf), (ScanNode)scanNode.clone(), fragments); + StorageManager.getStorageManager(tajoConf), (ScanNode)scanNode.clone(), fragments); } catch (CloneNotSupportedException e) { throw new IOException(e.getMessage(), e); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index f2679212a2..8d4a41f8b7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -49,8 +49,7 @@ import org.apache.tajo.master.rm.WorkerResourceManager; import org.apache.tajo.master.session.SessionManager; import org.apache.tajo.rpc.RpcChannelFactory; -import org.apache.tajo.storage.AbstractStorageManager; -import org.apache.tajo.storage.StorageManagerFactory; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.util.*; import org.apache.tajo.util.metrics.TajoSystemMetrics; import org.apache.tajo.webapp.QueryExecutorServlet; @@ -105,7 +104,7 @@ public class TajoMaster extends CompositeService { private CatalogServer catalogServer; private CatalogService catalog; - private AbstractStorageManager storeManager; + private StorageManager storeManager; private GlobalEngine globalEngine; private AsyncDispatcher dispatcher; private TajoMasterClientService tajoMasterClientService; @@ -160,7 +159,7 @@ public void serviceInit(Configuration _conf) throws Exception { // check the system directory and create if they are not created. checkAndInitializeSystemDirectories(); - this.storeManager = StorageManagerFactory.getStorageManager(systemConf); + this.storeManager = StorageManager.getStorageManager(systemConf); catalogServer = new CatalogServer(FunctionLoader.load()); addIfService(catalogServer); @@ -395,7 +394,7 @@ public CatalogServer getCatalogServer() { return this.catalogServer; } - public AbstractStorageManager getStorageManager() { + public StorageManager getStorageManager() { return this.storeManager; } @@ -438,7 +437,7 @@ public GlobalEngine getGlobalEngine() { return globalEngine; } - public AbstractStorageManager getStorageManager() { + public StorageManager getStorageManager() { return storeManager; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index d3f5b1dc1f..b8240b832c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -48,7 +48,7 @@ import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.event.*; -import org.apache.tajo.storage.AbstractStorageManager; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.TUtil; @@ -70,7 +70,7 @@ public class Query implements EventHandler { private Map subqueries; private final EventHandler eventHandler; private final MasterPlan plan; - private final AbstractStorageManager sm; + private final StorageManager sm; QueryMasterTask.QueryMasterTaskContext context; private ExecutionBlockCursor cursor; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java index b8c39e02e3..7c3d799a9a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMaster.java @@ -41,8 +41,7 @@ import org.apache.tajo.rpc.NullCallback; import org.apache.tajo.rpc.RpcConnectionPool; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; -import org.apache.tajo.storage.AbstractStorageManager; -import org.apache.tajo.storage.StorageManagerFactory; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.util.HAServiceUtil; import org.apache.tajo.util.NetUtils; import org.apache.tajo.worker.TajoWorker; @@ -71,7 +70,7 @@ public class QueryMaster extends CompositeService implements EventHandler { private GlobalPlanner globalPlanner; - private AbstractStorageManager storageManager; + private StorageManager storageManager; private TajoConf systemConf; @@ -116,7 +115,7 @@ public void init(Configuration conf) { this.dispatcher = new TajoAsyncDispatcher("querymaster_" + System.currentTimeMillis()); addIfService(dispatcher); - this.storageManager = StorageManagerFactory.getStorageManager(systemConf); + this.storageManager = StorageManager.getStorageManager(systemConf); globalPlanner = new GlobalPlanner(systemConf, workerContext); @@ -373,7 +372,7 @@ public Clock getClock() { return clock; } - public AbstractStorageManager getStorageManager() { + public StorageManager getStorageManager() { return storageManager; } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index 1ffaf5643f..cb06df940b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -57,7 +57,7 @@ import org.apache.tajo.rpc.CallFuture; import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.RpcConnectionPool; -import org.apache.tajo.storage.AbstractStorageManager; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.util.HAServiceUtil; import org.apache.tajo.util.metrics.TajoMetrics; import org.apache.tajo.util.metrics.reporter.MetricsConsoleReporter; @@ -556,7 +556,7 @@ public QueryId getQueryId() { return queryId; } - public AbstractStorageManager getStorageManager() { + public StorageManager getStorageManager() { return queryMasterContext.getStorageManager(); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index a0616117cc..615ebcfcb5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -47,7 +47,7 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty; import org.apache.tajo.master.TaskSchedulerContext; import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry; -import org.apache.tajo.storage.AbstractStorageManager; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.fragment.FileFragment; @@ -81,7 +81,7 @@ public static void scheduleFragmentsForJoinQuery(TaskSchedulerContext schedulerC MasterPlan masterPlan = subQuery.getMasterPlan(); ExecutionBlock execBlock = subQuery.getBlock(); QueryMasterTask.QueryMasterTaskContext masterContext = subQuery.getContext(); - AbstractStorageManager storageManager = subQuery.getStorageManager(); + StorageManager storageManager = subQuery.getStorageManager(); ScanNode[] scans = execBlock.getScanNodes(); @@ -478,7 +478,7 @@ public static Map>> merge /** * It creates a number of fragments for all partitions. */ - public static List getFragmentsFromPartitionedTable(AbstractStorageManager sm, + public static List getFragmentsFromPartitionedTable(StorageManager sm, ScanNode scan, TableDesc table) throws IOException { List fragments = Lists.newArrayList(); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index 907fc658d2..a126144c50 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -61,7 +61,7 @@ import org.apache.tajo.master.event.*; import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext; import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry; -import org.apache.tajo.storage.AbstractStorageManager; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; @@ -94,7 +94,7 @@ public class SubQuery implements EventHandler { private TableStats resultStatistics; private TableStats inputStatistics; private EventHandler eventHandler; - private final AbstractStorageManager sm; + private final StorageManager sm; private AbstractTaskScheduler taskScheduler; private QueryMasterTask.QueryMasterTaskContext context; private final List diagnostics = new ArrayList(); @@ -282,7 +282,8 @@ SubQueryEventType.SQ_KILL, new KillTasksTransition()) private List hashShuffleIntermediateEntries = new ArrayList(); private AtomicInteger completeReportReceived = new AtomicInteger(0); - public SubQuery(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPlan, ExecutionBlock block, AbstractStorageManager sm) { + public SubQuery(QueryMasterTask.QueryMasterTaskContext context, MasterPlan masterPlan, + ExecutionBlock block, StorageManager sm) { this.context = context; this.masterPlan = masterPlan; this.block = block; @@ -434,7 +435,7 @@ public int getPriority() { return this.priority; } - public AbstractStorageManager getStorageManager() { + public StorageManager getStorageManager() { return sm; } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java index 1731854a54..5cf6c4619c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java @@ -24,18 +24,17 @@ import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.exception.InternalException; -import org.apache.tajo.storage.AbstractStorageManager; -import org.apache.tajo.storage.StorageManagerFactory; +import org.apache.tajo.storage.StorageManager; import java.io.IOException; public class TajoQueryEngine { - private final AbstractStorageManager storageManager; + private final StorageManager storageManager; private final PhysicalPlanner phyPlanner; public TajoQueryEngine(TajoConf conf) throws IOException { - this.storageManager = StorageManagerFactory.getStorageManager(conf); + this.storageManager = StorageManager.getStorageManager(conf); this.phyPlanner = new PhysicalPlannerImpl(conf, storageManager); } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java b/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java index 6a90f7498a..f12e83c161 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/WorkerHeartbeatService.java @@ -32,9 +32,9 @@ import org.apache.tajo.rpc.NettyClientBase; import org.apache.tajo.rpc.RpcConnectionPool; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; -import org.apache.tajo.storage.v2.DiskDeviceInfo; -import org.apache.tajo.storage.v2.DiskMountInfo; -import org.apache.tajo.storage.v2.DiskUtil; +import org.apache.tajo.storage.DiskDeviceInfo; +import org.apache.tajo.storage.DiskMountInfo; +import org.apache.tajo.storage.DiskUtil; import org.apache.tajo.util.HAServiceUtil; import java.io.File; diff --git a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java index 0064e41a8c..45d3c5109e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/BackendTestingUtil.java @@ -47,7 +47,7 @@ public class BackendTestingUtil { public static void writeTmpTable(TajoConf conf, Path tablePath) throws IOException { - AbstractStorageManager sm = StorageManagerFactory.getStorageManager(conf, tablePath); + StorageManager sm = StorageManager.getStorageManager(conf, tablePath); FileSystem fs = sm.getFileSystem(); Appender appender; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index f1122ce670..8810692be1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -44,7 +44,7 @@ import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.TajoMaster; import org.apache.tajo.storage.Appender; -import org.apache.tajo.storage.StorageManagerFactory; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.CommonTestingUtil; @@ -141,7 +141,7 @@ private TableDesc makeTestData(String tableName, Schema schema, int dataSize) th contentsData += j; } } - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(tableMeta, schema, + Appender appender = StorageManager.getStorageManager(conf).getAppender(tableMeta, schema, dataPath); appender.init(); Tuple tuple = new VTuple(schema.size()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index 3fecabd5a6..197ff653e1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -59,7 +59,7 @@ public class TestBNLJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private static int OUTER_TUPLE_NUM = 1000; @@ -76,7 +76,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -86,7 +86,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < OUTER_TUPLE_NUM; i++) { @@ -107,7 +107,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < INNER_TUPLE_NUM; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index f817776b99..00a44165cb 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -66,7 +66,7 @@ public class TestBSTIndexExec { private SQLAnalyzer analyzer; private LogicalPlanner planner; private LogicalOptimizer optimizer; - private AbstractStorageManager sm; + private StorageManager sm; private Schema idxSchema; private TupleComparator comp; private BSTIndex.BSTIndexWriter writer; @@ -91,7 +91,7 @@ public void setup() throws Exception { Path workDir = CommonTestingUtil.getTestDir(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, workDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = StorageManagerFactory.getStorageManager(conf, workDir); + sm = StorageManager.getStorageManager(conf, workDir); idxPath = new Path(workDir, "test.idx"); @@ -117,7 +117,7 @@ public void setup() throws Exception { fs = tablePath.getFileSystem(conf); fs.mkdirs(tablePath.getParent()); - FileAppender appender = (FileAppender)StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, + FileAppender appender = (FileAppender)StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); @@ -189,7 +189,7 @@ public void testEqual() throws Exception { } private class TmpPlanner extends PhysicalPlannerImpl { - public TmpPlanner(TajoConf conf, AbstractStorageManager sm) { + public TmpPlanner(TajoConf conf, StorageManager sm) { super(conf, sm); } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index e7aac3c79c..6572506b25 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -56,7 +56,7 @@ public class TestExternalSortExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private final int numTuple = 100000; @@ -73,7 +73,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -82,7 +82,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.size()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 548d43a899..aba5169585 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -58,7 +58,7 @@ public class TestFullOuterHashJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -81,7 +81,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -104,7 +104,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -133,7 +133,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -172,7 +172,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -224,7 +224,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 1b9f7aaa0a..7998c61bbf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -59,7 +59,7 @@ public class TestFullOuterMergeJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -85,7 +85,7 @@ public void setUp() throws Exception { catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -108,7 +108,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -146,7 +146,7 @@ public void setUp() throws Exception { TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = StorageManagerFactory.getStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); + Appender appender4 = StorageManager.getStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { @@ -177,7 +177,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -216,7 +216,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -268,7 +268,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); appender5.flush(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 7055e44c17..77f524d30d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -57,7 +57,7 @@ public class TestHashAntiJoinExec { private SQLAnalyzer analyzer; private LogicalPlanner planner; private LogicalOptimizer optimizer; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private TableDesc employee; @@ -72,7 +72,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); @@ -82,7 +82,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -108,7 +108,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index a8828abf42..eff647d9f1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -59,7 +59,7 @@ public class TestHashJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -75,7 +75,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); @@ -85,7 +85,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -108,7 +108,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 6373f232a4..148592dea2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -57,7 +57,7 @@ public class TestHashSemiJoinExec { private SQLAnalyzer analyzer; private LogicalPlanner planner; private LogicalOptimizer optimizer; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private TableDesc employee; @@ -72,7 +72,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); @@ -82,7 +82,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -108,7 +108,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); // make 27 tuples diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index ef740e37dc..3e404859db 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -59,7 +59,7 @@ public class TestLeftOuterHashJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -82,7 +82,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -105,7 +105,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -134,7 +134,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -173,7 +173,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -225,7 +225,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index 2a8d9f0108..9ee51c3d5b 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -60,7 +60,7 @@ public class TestLeftOuterNLJoinExec { private SQLAnalyzer analyzer; private LogicalPlanner planner; private QueryContext defaultContext; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private TableDesc dep3; @@ -81,7 +81,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -104,7 +104,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -133,7 +133,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -172,7 +172,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -224,7 +224,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index ed19e4f5f7..ddad0a21bf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -60,7 +60,7 @@ public class TestMergeJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private TableDesc employee; private TableDesc people; @@ -75,7 +75,7 @@ public void setUp() throws Exception { catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); FileSystem fs = testDir.getFileSystem(conf); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); Schema employeeSchema = new Schema(); employeeSchema.addColumn("managerid", Type.INT4); @@ -85,7 +85,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -114,7 +114,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 10; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 1120bbe1f7..20053cbf30 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -60,7 +60,7 @@ public class TestNLJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private TableDesc employee; @@ -76,7 +76,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -86,7 +86,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 50; i++) { @@ -109,7 +109,7 @@ public void setUp() throws Exception { peopleSchema.addColumn("age", Type.INT4); TableMeta peopleMeta = CatalogUtil.newTableMeta(StoreType.CSV); Path peoplePath = new Path(testDir, "people.csv"); - appender = StorageManagerFactory.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); + appender = StorageManager.getStorageManager(conf).getAppender(peopleMeta, peopleSchema, peoplePath); appender.init(); tuple = new VTuple(peopleSchema.size()); for (int i = 1; i < 50; i += 2) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 8262d8ff6d..30c2cd5e38 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -84,7 +84,7 @@ public class TestPhysicalPlanner { private static SQLAnalyzer analyzer; private static LogicalPlanner planner; private static LogicalOptimizer optimizer; - private static AbstractStorageManager sm; + private static StorageManager sm; private static Path testDir; private static Session session = LocalTajoTestingUtility.createDummySession(); private static QueryContext defaultContext; @@ -102,7 +102,7 @@ public static void setUp() throws Exception { util.startCatalogCluster(); conf = util.getConfiguration(); testDir = CommonTestingUtil.getTestDir("target/test-data/TestPhysicalPlanner"); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); @@ -125,7 +125,7 @@ public static void setUp() throws Exception { Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, employeeSchema, employeePath); appender.init(); Tuple tuple = new VTuple(employeeSchema.size()); @@ -144,7 +144,7 @@ public static void setUp() throws Exception { Path scorePath = new Path(testDir, "score"); TableMeta scoreMeta = CatalogUtil.newTableMeta(StoreType.CSV, new KeyValueSet()); - appender = StorageManagerFactory.getStorageManager(conf).getAppender(scoreMeta, scoreSchema, scorePath); + appender = StorageManager.getStorageManager(conf).getAppender(scoreMeta, scoreSchema, scorePath); appender.init(); score = new TableDesc( CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"), scoreSchema, scoreMeta, @@ -185,7 +185,7 @@ public static void createLargeScoreTable() throws IOException { Schema scoreSchmea = score.getSchema(); TableMeta scoreLargeMeta = CatalogUtil.newTableMeta(StoreType.RAW, new KeyValueSet()); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(scoreLargeMeta, scoreSchmea, + Appender appender = StorageManager.getStorageManager(conf).getAppender(scoreLargeMeta, scoreSchmea, scoreLargePath); appender.enableStats(); appender.init(); @@ -448,7 +448,7 @@ public final void testStorePlan() throws IOException, PlanningException { exec.next(); exec.close(); - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), + Scanner scanner = StorageManager.getStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; @@ -508,7 +508,7 @@ public final void testStorePlanWithMaxOutputFileSize() throws IOException, Plann // checking the file contents long totalNum = 0; for (FileStatus status : fs.listStatus(ctx.getOutputPath().getParent())) { - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getFileScanner( + Scanner scanner = StorageManager.getStorageManager(conf).getFileScanner( CatalogUtil.newTableMeta(StoreType.CSV), rootNode.getOutSchema(), status.getPath()); @@ -545,7 +545,7 @@ public final void testStorePlanWithRCFile() throws IOException, PlanningExceptio exec.next(); exec.close(); - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), + Scanner scanner = StorageManager.getStorageManager(conf).getFileScanner(outputMeta, rootNode.getOutSchema(), ctx.getOutputPath()); scanner.init(); Tuple tuple; @@ -1080,7 +1080,7 @@ public final void testIndexedStoreExec() throws IOException, PlanningException { Path outputPath = StorageUtil.concatPath(workDir, "output", "output"); TableMeta meta = CatalogUtil.newTableMeta(channel.getStoreType(), new KeyValueSet()); SeekableScanner scanner = - StorageManagerFactory.getSeekableScanner(conf, meta, exec.getSchema(), outputPath); + StorageManager.getSeekableScanner(conf, meta, exec.getSchema(), outputPath); scanner.init(); int cnt = 0; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index f649dacb28..c8b9e4ab04 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -62,7 +62,7 @@ public class TestProgressExternalSortExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private final int numTuple = 100000; @@ -80,7 +80,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf.setVar(TajoConf.ConfVars.WORKER_TEMPORAL_DIR, testDir.toString()); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); Schema schema = new Schema(); schema.addColumn("managerid", TajoDataTypes.Type.INT4); @@ -89,7 +89,7 @@ public void setUp() throws Exception { TableMeta employeeMeta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.RAW); Path employeePath = new Path(testDir, "employee.csv"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, employeePath); appender.enableStats(); appender.init(); Tuple tuple = new VTuple(schema.size()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 506555ed51..09f5ab8dec 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -59,7 +59,7 @@ public class TestRightOuterHashJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -80,7 +80,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -103,7 +103,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -132,7 +132,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -171,7 +171,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 0ce74eb600..f8e2f41b03 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -58,7 +58,7 @@ public class TestRightOuterMergeJoinExec { private CatalogService catalog; private SQLAnalyzer analyzer; private LogicalPlanner planner; - private AbstractStorageManager sm; + private StorageManager sm; private Path testDir; private QueryContext defaultContext; @@ -84,7 +84,7 @@ public void setUp() throws Exception { catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); //----------------- dep3 ------------------------------ // dep_id | dep_name | loc_id @@ -107,7 +107,7 @@ public void setUp() throws Exception { TableMeta dep3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep3Path = new Path(testDir, "dep3.csv"); - Appender appender1 = StorageManagerFactory.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); + Appender appender1 = StorageManager.getStorageManager(conf).getAppender(dep3Meta, dep3Schema, dep3Path); appender1.init(); Tuple tuple = new VTuple(dep3Schema.size()); for (int i = 0; i < 10; i++) { @@ -145,7 +145,7 @@ public void setUp() throws Exception { TableMeta dep4Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path dep4Path = new Path(testDir, "dep4.csv"); - Appender appender4 = StorageManagerFactory.getStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); + Appender appender4 = StorageManager.getStorageManager(conf).getAppender(dep4Meta, dep4Schema, dep4Path); appender4.init(); Tuple tuple4 = new VTuple(dep4Schema.size()); for (int i = 0; i < 11; i++) { @@ -176,7 +176,7 @@ public void setUp() throws Exception { TableMeta job3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path job3Path = new Path(testDir, "job3.csv"); - Appender appender2 = StorageManagerFactory.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); + Appender appender2 = StorageManager.getStorageManager(conf).getAppender(job3Meta, job3Schema, job3Path); appender2.init(); Tuple tuple2 = new VTuple(job3Schema.size()); for (int i = 1; i < 4; i++) { @@ -215,7 +215,7 @@ public void setUp() throws Exception { TableMeta emp3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path emp3Path = new Path(testDir, "emp3.csv"); - Appender appender3 = StorageManagerFactory.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); + Appender appender3 = StorageManager.getStorageManager(conf).getAppender(emp3Meta, emp3Schema, emp3Path); appender3.init(); Tuple tuple3 = new VTuple(emp3Schema.size()); @@ -267,7 +267,7 @@ public void setUp() throws Exception { TableMeta phone3Meta = CatalogUtil.newTableMeta(StoreType.CSV); Path phone3Path = new Path(testDir, "phone3.csv"); - Appender appender5 = StorageManagerFactory.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, + Appender appender5 = StorageManager.getStorageManager(conf).getAppender(phone3Meta, phone3Schema, phone3Path); appender5.init(); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 4d4cc3d794..63e189987a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -55,7 +55,7 @@ public class TestSortExec { private static SQLAnalyzer analyzer; private static LogicalPlanner planner; private static LogicalOptimizer optimizer; - private static AbstractStorageManager sm; + private static StorageManager sm; private static Path workDir; private static Path tablePath; private static TableMeta employeeMeta; @@ -68,7 +68,7 @@ public static void setUp() throws Exception { util = TpchTestBase.getInstance().getTestingCluster(); catalog = util.getMaster().getCatalog(); workDir = CommonTestingUtil.getTestDir(TEST_PATH); - sm = StorageManagerFactory.getStorageManager(conf, workDir); + sm = StorageManager.getStorageManager(conf, workDir); Schema schema = new Schema(); schema.addColumn("managerid", Type.INT4); @@ -80,7 +80,7 @@ public static void setUp() throws Exception { tablePath = StorageUtil.concatPath(workDir, "employee", "table1"); sm.getFileSystem().mkdirs(tablePath.getParent()); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(employeeMeta, schema, tablePath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(employeeMeta, schema, tablePath); appender.init(); Tuple tuple = new VTuple(schema.size()); for (int i = 0; i < 100; i++) { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java index 755c378d5b..b95ae41ee4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java @@ -565,7 +565,7 @@ private void createMultiFile(String tableName, int numRowsEachFile, TupleCreator } Path dataPath = new Path(table.getPath(), fileIndex + ".csv"); fileIndex++; - appender = StorageManagerFactory.getStorageManager(conf).getAppender(tableMeta, schema, + appender = StorageManager.getStorageManager(conf).getAppender(tableMeta, schema, dataPath); appender.init(); } diff --git a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index 2c06b8835e..1edf239d12 100644 --- a/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-core/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -55,7 +55,7 @@ public class TestResultSet { private static TajoTestingCluster util; private static TajoConf conf; private static TableDesc desc; - private static AbstractStorageManager sm; + private static StorageManager sm; private static TableMeta scoreMeta; private static Schema scoreSchema; @@ -63,7 +63,7 @@ public class TestResultSet { public static void setup() throws Exception { util = TpchTestBase.getInstance().getTestingCluster(); conf = util.getConfiguration(); - sm = StorageManagerFactory.getStorageManager(conf); + sm = StorageManager.getStorageManager(conf); scoreSchema = new Schema(); scoreSchema.addColumn("deptname", Type.TEXT); @@ -73,7 +73,7 @@ public static void setup() throws Exception { Path p = sm.getTablePath("score"); sm.getFileSystem().mkdirs(p); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(scoreMeta, scoreSchema, + Appender appender = StorageManager.getStorageManager(conf).getAppender(scoreMeta, scoreSchema, new Path(p, "score")); appender.init(); int deptSize = 100; diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 6bbf63bca5..234c58e572 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -34,8 +34,7 @@ import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.storage.AbstractStorageManager; -import org.apache.tajo.storage.StorageManagerFactory; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.util.CommonTestingUtil; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -83,7 +82,7 @@ public static void setUp() throws Exception { logicalPlanner = new LogicalPlanner(catalog); optimizer = new LogicalOptimizer(conf); - AbstractStorageManager sm = StorageManagerFactory.getStorageManager(conf); + StorageManager sm = StorageManager.getStorageManager(conf); dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); diff --git a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java index f2784e50e5..742b07f499 100644 --- a/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java +++ b/tajo-core/src/test/java/org/apache/tajo/storage/TestRowFile.java @@ -69,7 +69,7 @@ public void test() throws IOException { TableMeta meta = CatalogUtil.newTableMeta(StoreType.ROWFILE); - AbstractStorageManager sm = StorageManagerFactory.getStorageManager(conf, new Path(conf.getVar(ConfVars.ROOT_DIR))); + StorageManager sm = StorageManager.getStorageManager(conf, new Path(conf.getVar(ConfVars.ROOT_DIR))); Path tablePath = new Path("/test"); Path metaPath = new Path(tablePath, ".meta"); @@ -79,7 +79,7 @@ public void test() throws IOException { FileUtil.writeProto(fs, metaPath, meta.getProto()); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, dataPath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, dataPath); appender.enableStats(); appender.init(); @@ -110,7 +110,7 @@ public void test() throws IOException { int tupleCnt = 0; start = System.currentTimeMillis(); - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment); + Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, fragment); scanner.init(); while ((tuple=scanner.next()) != null) { tupleCnt++; diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index 5f8efe7116..aff5d73fad 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -70,7 +70,7 @@ public class TestRangeRetrieverHandler { private SQLAnalyzer analyzer; private LogicalPlanner planner; private LogicalOptimizer optimizer; - private AbstractStorageManager sm; + private StorageManager sm; private Schema schema; private static int TEST_TUPLE = 10000; private FileSystem fs; @@ -86,7 +86,7 @@ public void setUp() throws Exception { catalog = util.getMiniCatalogCluster().getCatalog(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, testDir.toUri().toString()); catalog.createDatabase(TajoConstants.DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); - sm = StorageManagerFactory.getStorageManager(conf, testDir); + sm = StorageManager.getStorageManager(conf, testDir); analyzer = new SQLAnalyzer(); planner = new LogicalPlanner(catalog); @@ -181,7 +181,7 @@ public void testGet() throws Exception { reader.open(); TableMeta meta = CatalogUtil.newTableMeta(StoreType.RAW, new KeyValueSet()); - SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, + SeekableScanner scanner = StorageManager.getSeekableScanner(conf, meta, schema, StorageUtil.concatPath(testDir, "output", "output")); scanner.init(); @@ -304,7 +304,7 @@ public void testGetFromDescendingOrder() throws Exception { new Path(testDir, "output/index"), keySchema, comp); reader.open(); TableMeta outputMeta = CatalogUtil.newTableMeta(StoreType.RAW, new KeyValueSet()); - SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, outputMeta, schema, + SeekableScanner scanner = StorageManager.getSeekableScanner(conf, outputMeta, schema, StorageUtil.concatPath(testDir, "output", "output")); scanner.init(); int cnt = 0; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java deleted file mode 100644 index 4bf0057350..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java +++ /dev/null @@ -1,729 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - - -import com.google.common.collect.Lists; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.storage.fragment.FragmentConvertor; -import org.apache.tajo.util.Bytes; -import org.apache.tajo.util.FileUtil; - -import java.io.FileNotFoundException; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; - -import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; - -public abstract class AbstractStorageManager { - private final Log LOG = LogFactory.getLog(AbstractStorageManager.class); - - protected final TajoConf conf; - protected final FileSystem fs; - protected final Path tableBaseDir; - protected final boolean blocksMetadataEnabled; - private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0)); - - /** - * Cache of scanner handlers for each storage type. - */ - protected static final Map> SCANNER_HANDLER_CACHE - = new ConcurrentHashMap>(); - - /** - * Cache of appender handlers for each storage type. - */ - protected static final Map> APPENDER_HANDLER_CACHE - = new ConcurrentHashMap>(); - - /** - * Cache of constructors for each class. Pins the classes so they - * can't be garbage collected until ReflectionUtils can be collected. - */ - private static final Map, Constructor> CONSTRUCTOR_CACHE = - new ConcurrentHashMap, Constructor>(); - - public abstract Class getScannerClass(CatalogProtos.StoreType storeType) throws IOException; - - public abstract Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException; - - protected AbstractStorageManager(TajoConf conf) throws IOException { - this.conf = conf; - this.tableBaseDir = TajoConf.getWarehouseDir(conf); - this.fs = tableBaseDir.getFileSystem(conf); - this.blocksMetadataEnabled = conf.getBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, - DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT); - if (!this.blocksMetadataEnabled) - LOG.warn("does not support block metadata. ('dfs.datanode.hdfs-blocks-metadata.enabled')"); - } - - public Scanner getFileScanner(TableMeta meta, Schema schema, Path path) - throws IOException { - FileSystem fs = path.getFileSystem(conf); - FileStatus status = fs.getFileStatus(path); - return getFileScanner(meta, schema, path, status); - } - - public Scanner getFileScanner(TableMeta meta, Schema schema, Path path, FileStatus status) - throws IOException { - FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); - return getScanner(meta, schema, fragment); - } - - public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment) throws IOException { - return getScanner(meta, schema, FragmentConvertor.convert(conf, meta.getStoreType(), fragment), schema); - } - - public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, Schema target) throws IOException { - return getScanner(meta, schema, FragmentConvertor.convert(conf, meta.getStoreType(), fragment), target); - } - - public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) throws IOException { - return getScanner(meta, schema, fragment, schema); - } - - public FileSystem getFileSystem() { - return this.fs; - } - - public Path getWarehouseDir() { - return this.tableBaseDir; - } - - public void delete(Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - fs.delete(tablePath, true); - } - - public boolean exists(Path path) throws IOException { - FileSystem fileSystem = path.getFileSystem(conf); - return fileSystem.exists(path); - } - - /** - * This method deletes only data contained in the given path. - * - * @param path The path in which data are deleted. - * @throws IOException - */ - public void deleteData(Path path) throws IOException { - FileSystem fileSystem = path.getFileSystem(conf); - FileStatus[] fileLists = fileSystem.listStatus(path); - for (FileStatus status : fileLists) { - fileSystem.delete(status.getPath(), true); - } - } - - public Path getTablePath(String tableName) { - return new Path(tableBaseDir, tableName); - } - - public Appender getAppender(TableMeta meta, Schema schema, Path path) - throws IOException { - Appender appender; - - Class appenderClass; - - String handlerName = meta.getStoreType().name().toLowerCase(); - appenderClass = APPENDER_HANDLER_CACHE.get(handlerName); - if (appenderClass == null) { - appenderClass = conf.getClass( - String.format("tajo.storage.appender-handler.%s.class", - meta.getStoreType().name().toLowerCase()), null, - FileAppender.class); - APPENDER_HANDLER_CACHE.put(handlerName, appenderClass); - } - - if (appenderClass == null) { - throw new IOException("Unknown Storage Type: " + meta.getStoreType()); - } - - appender = newAppenderInstance(appenderClass, conf, meta, schema, path); - - return appender; - } - - public TableMeta getTableMeta(Path tablePath) throws IOException { - TableMeta meta; - - FileSystem fs = tablePath.getFileSystem(conf); - Path tableMetaPath = new Path(tablePath, ".meta"); - if (!fs.exists(tableMetaPath)) { - throw new FileNotFoundException(".meta file not found in " + tablePath.toString()); - } - - FSDataInputStream tableMetaIn = fs.open(tableMetaPath); - - CatalogProtos.TableProto tableProto = (CatalogProtos.TableProto) FileUtil.loadProto(tableMetaIn, - CatalogProtos.TableProto.getDefaultInstance()); - meta = new TableMeta(tableProto); - - return meta; - } - - public FileFragment[] split(String tableName) throws IOException { - Path tablePath = new Path(tableBaseDir, tableName); - return split(tableName, tablePath, fs.getDefaultBlockSize()); - } - - public FileFragment[] split(String tableName, long fragmentSize) throws IOException { - Path tablePath = new Path(tableBaseDir, tableName); - return split(tableName, tablePath, fragmentSize); - } - - public FileFragment[] splitBroadcastTable(Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - List listTablets = new ArrayList(); - FileFragment tablet; - - FileStatus[] fileLists = fs.listStatus(tablePath); - for (FileStatus file : fileLists) { - tablet = new FileFragment(tablePath.getName(), file.getPath(), 0, file.getLen()); - listTablets.add(tablet); - } - - FileFragment[] tablets = new FileFragment[listTablets.size()]; - listTablets.toArray(tablets); - - return tablets; - } - - public FileFragment[] split(Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - return split(tablePath.getName(), tablePath, fs.getDefaultBlockSize()); - } - - public FileFragment[] split(String tableName, Path tablePath) throws IOException { - return split(tableName, tablePath, fs.getDefaultBlockSize()); - } - - private FileFragment[] split(String tableName, Path tablePath, long size) - throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - - long defaultBlockSize = size; - List listTablets = new ArrayList(); - FileFragment tablet; - - FileStatus[] fileLists = fs.listStatus(tablePath); - for (FileStatus file : fileLists) { - long remainFileSize = file.getLen(); - long start = 0; - if (remainFileSize > defaultBlockSize) { - while (remainFileSize > defaultBlockSize) { - tablet = new FileFragment(tableName, file.getPath(), start, defaultBlockSize); - listTablets.add(tablet); - start += defaultBlockSize; - remainFileSize -= defaultBlockSize; - } - listTablets.add(new FileFragment(tableName, file.getPath(), start, remainFileSize)); - } else { - listTablets.add(new FileFragment(tableName, file.getPath(), 0, remainFileSize)); - } - } - - FileFragment[] tablets = new FileFragment[listTablets.size()]; - listTablets.toArray(tablets); - - return tablets; - } - - public static FileFragment[] splitNG(Configuration conf, String tableName, TableMeta meta, - Path tablePath, long size) - throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - - long defaultBlockSize = size; - List listTablets = new ArrayList(); - FileFragment tablet; - - FileStatus[] fileLists = fs.listStatus(tablePath); - for (FileStatus file : fileLists) { - long remainFileSize = file.getLen(); - long start = 0; - if (remainFileSize > defaultBlockSize) { - while (remainFileSize > defaultBlockSize) { - tablet = new FileFragment(tableName, file.getPath(), start, defaultBlockSize); - listTablets.add(tablet); - start += defaultBlockSize; - remainFileSize -= defaultBlockSize; - } - listTablets.add(new FileFragment(tableName, file.getPath(), start, remainFileSize)); - } else { - listTablets.add(new FileFragment(tableName, file.getPath(), 0, remainFileSize)); - } - } - - FileFragment[] tablets = new FileFragment[listTablets.size()]; - listTablets.toArray(tablets); - - return tablets; - } - - public long calculateSize(Path tablePath) throws IOException { - FileSystem fs = tablePath.getFileSystem(conf); - long totalSize = 0; - - if (fs.exists(tablePath)) { - totalSize = fs.getContentSummary(tablePath).getLength(); - } - - return totalSize; - } - - ///////////////////////////////////////////////////////////////////////////// - // FileInputFormat Area - ///////////////////////////////////////////////////////////////////////////// - - public static final PathFilter hiddenFileFilter = new PathFilter() { - public boolean accept(Path p) { - String name = p.getName(); - return !name.startsWith("_") && !name.startsWith("."); - } - }; - - /** - * Proxy PathFilter that accepts a path only if all filters given in the - * constructor do. Used by the listPaths() to apply the built-in - * hiddenFileFilter together with a user provided one (if any). - */ - private static class MultiPathFilter implements PathFilter { - private List filters; - - public MultiPathFilter(List filters) { - this.filters = filters; - } - - public boolean accept(Path path) { - for (PathFilter filter : filters) { - if (!filter.accept(path)) { - return false; - } - } - return true; - } - } - - /** - * List input directories. - * Subclasses may override to, e.g., select only files matching a regular - * expression. - * - * @return array of FileStatus objects - * @throws IOException if zero items. - */ - protected List listStatus(Path... dirs) throws IOException { - List result = new ArrayList(); - if (dirs.length == 0) { - throw new IOException("No input paths specified in job"); - } - - List errors = new ArrayList(); - - // creates a MultiPathFilter with the hiddenFileFilter and the - // user provided one (if any). - List filters = new ArrayList(); - filters.add(hiddenFileFilter); - - PathFilter inputFilter = new MultiPathFilter(filters); - - for (int i = 0; i < dirs.length; ++i) { - Path p = dirs[i]; - - FileSystem fs = p.getFileSystem(conf); - FileStatus[] matches = fs.globStatus(p, inputFilter); - if (matches == null) { - errors.add(new IOException("Input path does not exist: " + p)); - } else if (matches.length == 0) { - errors.add(new IOException("Input Pattern " + p + " matches 0 files")); - } else { - for (FileStatus globStat : matches) { - if (globStat.isDirectory()) { - for (FileStatus stat : fs.listStatus(globStat.getPath(), - inputFilter)) { - result.add(stat); - } - } else { - result.add(globStat); - } - } - } - } - - if (!errors.isEmpty()) { - throw new InvalidInputException(errors); - } - LOG.info("Total input paths to process : " + result.size()); - return result; - } - - /** - * Is the given filename splitable? Usually, true, but if the file is - * stream compressed, it will not be. - *

- * FileInputFormat implementations can override this and return - * false to ensure that individual input files are never split-up - * so that Mappers process entire files. - * - * - * @param path the file name to check - * @param status get the file length - * @return is this file isSplittable? - */ - protected boolean isSplittable(TableMeta meta, Schema schema, Path path, FileStatus status) throws IOException { - Scanner scanner = getFileScanner(meta, schema, path, status); - boolean split = scanner.isSplittable(); - scanner.close(); - return split; - } - - private static final double SPLIT_SLOP = 1.1; // 10% slop - - protected int getBlockIndex(BlockLocation[] blkLocations, - long offset) { - for (int i = 0; i < blkLocations.length; i++) { - // is the offset inside this block? - if ((blkLocations[i].getOffset() <= offset) && - (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())) { - return i; - } - } - BlockLocation last = blkLocations[blkLocations.length - 1]; - long fileLength = last.getOffset() + last.getLength() - 1; - throw new IllegalArgumentException("Offset " + offset + - " is outside of file (0.." + - fileLength + ")"); - } - - /** - * A factory that makes the split for this class. It can be overridden - * by sub-classes to make sub-types - */ - protected FileFragment makeSplit(String fragmentId, Path file, long start, long length) { - return new FileFragment(fragmentId, file, start, length); - } - - protected FileFragment makeSplit(String fragmentId, Path file, long start, long length, - String[] hosts) { - return new FileFragment(fragmentId, file, start, length, hosts); - } - - protected FileFragment makeSplit(String fragmentId, Path file, BlockLocation blockLocation) - throws IOException { - return new FileFragment(fragmentId, file, blockLocation); - } - - // for Non Splittable. eg, compressed gzip TextFile - protected FileFragment makeNonSplit(String fragmentId, Path file, long start, long length, - BlockLocation[] blkLocations) throws IOException { - - Map hostsBlockMap = new HashMap(); - for (BlockLocation blockLocation : blkLocations) { - for (String host : blockLocation.getHosts()) { - if (hostsBlockMap.containsKey(host)) { - hostsBlockMap.put(host, hostsBlockMap.get(host) + 1); - } else { - hostsBlockMap.put(host, 1); - } - } - } - - List> entries = new ArrayList>(hostsBlockMap.entrySet()); - Collections.sort(entries, new Comparator>() { - - @Override - public int compare(Map.Entry v1, Map.Entry v2) { - return v1.getValue().compareTo(v2.getValue()); - } - }); - - String[] hosts = new String[blkLocations[0].getHosts().length]; - - for (int i = 0; i < hosts.length; i++) { - Map.Entry entry = entries.get((entries.size() - 1) - i); - hosts[i] = entry.getKey(); - } - return new FileFragment(fragmentId, file, start, length, hosts); - } - - /** - * Get the minimum split size - * - * @return the minimum number of bytes that can be in a split - */ - public long getMinSplitSize() { - return conf.getLongVar(TajoConf.ConfVars.MINIMUM_SPLIT_SIZE); - } - - /** - * Get Disk Ids by Volume Bytes - */ - private int[] getDiskIds(VolumeId[] volumeIds) { - int[] diskIds = new int[volumeIds.length]; - for (int i = 0; i < volumeIds.length; i++) { - int diskId = -1; - if (volumeIds[i] != null && volumeIds[i].hashCode() > 0) { - diskId = volumeIds[i].hashCode() - zeroVolumeId.hashCode(); - } - diskIds[i] = diskId; - } - return diskIds; - } - - /** - * Generate the map of host and make them into Volume Ids. - * - */ - private Map> getVolumeMap(List frags) { - Map> volumeMap = new HashMap>(); - for (FileFragment frag : frags) { - String[] hosts = frag.getHosts(); - int[] diskIds = frag.getDiskIds(); - for (int i = 0; i < hosts.length; i++) { - Set volumeList = volumeMap.get(hosts[i]); - if (volumeList == null) { - volumeList = new HashSet(); - volumeMap.put(hosts[i], volumeList); - } - - if (diskIds.length > 0 && diskIds[i] > -1) { - volumeList.add(diskIds[i]); - } - } - } - - return volumeMap; - } - /** - * Generate the list of files and make them into FileSplits. - * - * @throws IOException - */ - public List getSplits(String tableName, TableMeta meta, Schema schema, Path... inputs) - throws IOException { - // generate splits' - - List splits = Lists.newArrayList(); - List volumeSplits = Lists.newArrayList(); - List blockLocations = Lists.newArrayList(); - - for (Path p : inputs) { - FileSystem fs = p.getFileSystem(conf); - ArrayList files = Lists.newArrayList(); - if (fs.isFile(p)) { - files.addAll(Lists.newArrayList(fs.getFileStatus(p))); - } else { - files.addAll(listStatus(p)); - } - - int previousSplitSize = splits.size(); - for (FileStatus file : files) { - Path path = file.getPath(); - long length = file.getLen(); - if (length > 0) { - // Get locations of blocks of file - BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length); - boolean splittable = isSplittable(meta, schema, path, file); - if (blocksMetadataEnabled && fs instanceof DistributedFileSystem) { - - if (splittable) { - for (BlockLocation blockLocation : blkLocations) { - volumeSplits.add(makeSplit(tableName, path, blockLocation)); - } - blockLocations.addAll(Arrays.asList(blkLocations)); - - } else { // Non splittable - long blockSize = blkLocations[0].getLength(); - if (blockSize >= length) { - blockLocations.addAll(Arrays.asList(blkLocations)); - for (BlockLocation blockLocation : blkLocations) { - volumeSplits.add(makeSplit(tableName, path, blockLocation)); - } - } else { - splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); - } - } - - } else { - if (splittable) { - - long minSize = Math.max(getMinSplitSize(), 1); - - long blockSize = file.getBlockSize(); // s3n rest api contained block size but blockLocations is one - long splitSize = Math.max(minSize, blockSize); - long bytesRemaining = length; - - // for s3 - while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) { - int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); - splits.add(makeSplit(tableName, path, length - bytesRemaining, splitSize, - blkLocations[blkIndex].getHosts())); - bytesRemaining -= splitSize; - } - if (bytesRemaining > 0) { - int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); - splits.add(makeSplit(tableName, path, length - bytesRemaining, bytesRemaining, - blkLocations[blkIndex].getHosts())); - } - } else { // Non splittable - splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); - } - } - } else { - //for zero length files - splits.add(makeSplit(tableName, path, 0, length)); - } - } - if(LOG.isDebugEnabled()){ - LOG.debug("# of splits per partition: " + (splits.size() - previousSplitSize)); - } - } - - // Combine original fileFragments with new VolumeId information - setVolumeMeta(volumeSplits, blockLocations); - splits.addAll(volumeSplits); - LOG.info("Total # of splits: " + splits.size()); - return splits; - } - - private void setVolumeMeta(List splits, final List blockLocations) - throws IOException { - - int locationSize = blockLocations.size(); - int splitSize = splits.size(); - if (locationSize == 0 || splitSize == 0) return; - - if (locationSize != splitSize) { - // splits and locations don't match up - LOG.warn("Number of block locations not equal to number of splits: " - + "#locations=" + locationSize - + " #splits=" + splitSize); - return; - } - - DistributedFileSystem fs = (DistributedFileSystem)DistributedFileSystem.get(conf); - int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); - int blockLocationIdx = 0; - - Iterator iter = splits.iterator(); - while (locationSize > blockLocationIdx) { - - int subSize = Math.min(locationSize - blockLocationIdx, lsLimit); - List locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize); - //BlockStorageLocation containing additional volume location information for each replica of each block. - BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations); - - for (BlockStorageLocation blockStorageLocation : blockStorageLocations) { - iter.next().setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds())); - blockLocationIdx++; - } - } - LOG.info("# of splits with volumeId " + splitSize); - } - - private static class InvalidInputException extends IOException { - List errors; - public InvalidInputException(List errors) { - this.errors = errors; - } - - @Override - public String getMessage(){ - StringBuffer sb = new StringBuffer(); - int messageLimit = Math.min(errors.size(), 10); - for (int i = 0; i < messageLimit ; i ++) { - sb.append(errors.get(i).getMessage()).append("\n"); - } - - if(messageLimit < errors.size()) - sb.append("skipped .....").append("\n"); - - return sb.toString(); - } - } - - private static final Class[] DEFAULT_SCANNER_PARAMS = { - Configuration.class, - Schema.class, - TableMeta.class, - FileFragment.class - }; - - private static final Class[] DEFAULT_APPENDER_PARAMS = { - Configuration.class, - Schema.class, - TableMeta.class, - Path.class - }; - - /** - * create a scanner instance. - */ - public static T newScannerInstance(Class theClass, Configuration conf, Schema schema, TableMeta meta, - Fragment fragment) { - T result; - try { - Constructor meth = (Constructor) CONSTRUCTOR_CACHE.get(theClass); - if (meth == null) { - meth = theClass.getDeclaredConstructor(DEFAULT_SCANNER_PARAMS); - meth.setAccessible(true); - CONSTRUCTOR_CACHE.put(theClass, meth); - } - result = meth.newInstance(new Object[]{conf, schema, meta, fragment}); - } catch (Exception e) { - throw new RuntimeException(e); - } - - return result; - } - - /** - * create a scanner instance. - */ - public static T newAppenderInstance(Class theClass, Configuration conf, TableMeta meta, Schema schema, - Path path) { - T result; - try { - Constructor meth = (Constructor) CONSTRUCTOR_CACHE.get(theClass); - if (meth == null) { - meth = theClass.getDeclaredConstructor(DEFAULT_APPENDER_PARAMS); - meth.setAccessible(true); - CONSTRUCTOR_CACHE.put(theClass, meth); - } - result = meth.newInstance(new Object[]{conf, schema, meta, path}); - } catch (Exception e) { - throw new RuntimeException(e); - } - - return result; - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskDeviceInfo.java b/tajo-storage/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java similarity index 97% rename from tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskDeviceInfo.java rename to tajo-storage/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java index 7802c91742..23963497c8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskDeviceInfo.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/DiskDeviceInfo.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.storage.v2; +package org.apache.tajo.storage; import java.util.ArrayList; import java.util.List; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskInfo.java b/tajo-storage/src/main/java/org/apache/tajo/storage/DiskInfo.java similarity index 97% rename from tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskInfo.java rename to tajo-storage/src/main/java/org/apache/tajo/storage/DiskInfo.java index d71154ca09..22f18ba1f8 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskInfo.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/DiskInfo.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.storage.v2; +package org.apache.tajo.storage; public class DiskInfo { private int id; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskMountInfo.java b/tajo-storage/src/main/java/org/apache/tajo/storage/DiskMountInfo.java similarity index 98% rename from tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskMountInfo.java rename to tajo-storage/src/main/java/org/apache/tajo/storage/DiskMountInfo.java index 56100f25be..aadb0e750c 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskMountInfo.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/DiskMountInfo.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.storage.v2; +package org.apache.tajo.storage; import com.google.common.base.Objects; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/DiskUtil.java similarity index 99% rename from tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskUtil.java rename to tajo-storage/src/main/java/org/apache/tajo/storage/DiskUtil.java index 66827c2649..2d68870008 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskUtil.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/DiskUtil.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.storage.v2; +package org.apache.tajo.storage; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.HdfsConfiguration; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index f0699b7b45..84d81d5659 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -31,6 +31,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.util.Pair; +import org.apache.tajo.storage.StorageManager; import java.io.IOException; import java.util.ArrayList; @@ -84,7 +85,7 @@ public HashShuffleAppender getAppender(TajoConf tajoConf, ExecutionBlockId ebId, if (!fs.exists(dataFile.getParent())) { fs.mkdirs(dataFile.getParent()); } - FileAppender appender = (FileAppender) StorageManagerFactory.getStorageManager( + FileAppender appender = (FileAppender) StorageManager.getStorageManager( tajoConf).getAppender(meta, outSchema, dataFile); appender.enableStats(); appender.init(); diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java index 637df2ce74..4122c76d66 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java @@ -128,7 +128,7 @@ public void reset() throws IOException { private Scanner getNextScanner() throws IOException { if (iterator.hasNext()) { currentFragment = iterator.next(); - currentScanner = StorageManagerFactory.getStorageManager((TajoConf)conf).getScanner(meta, schema, + currentScanner = StorageManager.getStorageManager((TajoConf)conf).getScanner(meta, schema, currentFragment, target); currentScanner.init(); return currentScanner; diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java index 4b23f4d328..87b41973a1 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManager.java @@ -18,25 +18,744 @@ package org.apache.tajo.storage; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.*; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; +import org.apache.tajo.storage.fragment.FragmentConvertor; +import org.apache.tajo.util.Bytes; +import org.apache.tajo.util.FileUtil; +import java.io.FileNotFoundException; import java.io.IOException; +import java.lang.reflect.Constructor; +import java.net.URI; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; /** * StorageManager */ -public class StorageManager extends AbstractStorageManager { +public class StorageManager { + private final Log LOG = LogFactory.getLog(StorageManager.class); - protected StorageManager(TajoConf conf) throws IOException { - super(conf); + protected final TajoConf conf; + protected final FileSystem fs; + protected final Path tableBaseDir; + protected final boolean blocksMetadataEnabled; + private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0)); + + private static final Map storageManagers = Maps.newHashMap(); + + /** + * Cache of scanner handlers for each storage type. + */ + protected static final Map> SCANNER_HANDLER_CACHE + = new ConcurrentHashMap>(); + + /** + * Cache of appender handlers for each storage type. + */ + protected static final Map> APPENDER_HANDLER_CACHE + = new ConcurrentHashMap>(); + + /** + * Cache of constructors for each class. Pins the classes so they + * can't be garbage collected until ReflectionUtils can be collected. + */ + private static final Map, Constructor> CONSTRUCTOR_CACHE = + new ConcurrentHashMap, Constructor>(); + + private StorageManager(TajoConf conf) throws IOException { + this.conf = conf; + this.tableBaseDir = TajoConf.getWarehouseDir(conf); + this.fs = tableBaseDir.getFileSystem(conf); + this.blocksMetadataEnabled = conf.getBoolean(DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED, + DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT); + if (!this.blocksMetadataEnabled) + LOG.warn("does not support block metadata. ('dfs.datanode.hdfs-blocks-metadata.enabled')"); + } + + public static StorageManager getStorageManager(TajoConf conf) throws IOException { + return getStorageManager(conf, null); + } + + public static synchronized StorageManager getStorageManager ( + TajoConf conf, Path warehouseDir) throws IOException { + + URI uri; + TajoConf localConf = new TajoConf(conf); + if (warehouseDir != null) { + localConf.setVar(ConfVars.WAREHOUSE_DIR, warehouseDir.toUri().toString()); + } + + uri = TajoConf.getWarehouseDir(localConf).toUri(); + + String key = "file".equals(uri.getScheme()) ? "file" : uri.toString(); + + if(storageManagers.containsKey(key)) { + StorageManager sm = storageManagers.get(key); + return sm; + } else { + StorageManager storageManager = new StorageManager(localConf); + storageManagers.put(key, storageManager); + return storageManager; + } + } + + public Scanner getFileScanner(TableMeta meta, Schema schema, Path path) + throws IOException { + FileSystem fs = path.getFileSystem(conf); + FileStatus status = fs.getFileStatus(path); + return getFileScanner(meta, schema, path, status); + } + + public Scanner getFileScanner(TableMeta meta, Schema schema, Path path, FileStatus status) + throws IOException { + FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); + return getScanner(meta, schema, fragment); + } + + public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment) throws IOException { + return getScanner(meta, schema, FragmentConvertor.convert(conf, meta.getStoreType(), fragment), schema); + } + + public Scanner getScanner(TableMeta meta, Schema schema, FragmentProto fragment, Schema target) throws IOException { + return getScanner(meta, schema, FragmentConvertor.convert(conf, meta.getStoreType(), fragment), target); + } + + public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment) throws IOException { + return getScanner(meta, schema, fragment, schema); + } + + public FileSystem getFileSystem() { + return this.fs; + } + + public Path getWarehouseDir() { + return this.tableBaseDir; + } + + public void delete(Path tablePath) throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + fs.delete(tablePath, true); + } + + public boolean exists(Path path) throws IOException { + FileSystem fileSystem = path.getFileSystem(conf); + return fileSystem.exists(path); + } + + /** + * This method deletes only data contained in the given path. + * + * @param path The path in which data are deleted. + * @throws IOException + */ + public void deleteData(Path path) throws IOException { + FileSystem fileSystem = path.getFileSystem(conf); + FileStatus[] fileLists = fileSystem.listStatus(path); + for (FileStatus status : fileLists) { + fileSystem.delete(status.getPath(), true); + } + } + + public Path getTablePath(String tableName) { + return new Path(tableBaseDir, tableName); + } + + public Appender getAppender(TableMeta meta, Schema schema, Path path) + throws IOException { + Appender appender; + + Class appenderClass; + + String handlerName = meta.getStoreType().name().toLowerCase(); + appenderClass = APPENDER_HANDLER_CACHE.get(handlerName); + if (appenderClass == null) { + appenderClass = conf.getClass( + String.format("tajo.storage.appender-handler.%s.class", + meta.getStoreType().name().toLowerCase()), null, + FileAppender.class); + APPENDER_HANDLER_CACHE.put(handlerName, appenderClass); + } + + if (appenderClass == null) { + throw new IOException("Unknown Storage Type: " + meta.getStoreType()); + } + + appender = newAppenderInstance(appenderClass, conf, meta, schema, path); + + return appender; + } + + public TableMeta getTableMeta(Path tablePath) throws IOException { + TableMeta meta; + + FileSystem fs = tablePath.getFileSystem(conf); + Path tableMetaPath = new Path(tablePath, ".meta"); + if (!fs.exists(tableMetaPath)) { + throw new FileNotFoundException(".meta file not found in " + tablePath.toString()); + } + + FSDataInputStream tableMetaIn = fs.open(tableMetaPath); + + CatalogProtos.TableProto tableProto = (CatalogProtos.TableProto) FileUtil.loadProto(tableMetaIn, + CatalogProtos.TableProto.getDefaultInstance()); + meta = new TableMeta(tableProto); + + return meta; + } + + public FileFragment[] split(String tableName) throws IOException { + Path tablePath = new Path(tableBaseDir, tableName); + return split(tableName, tablePath, fs.getDefaultBlockSize()); + } + + public FileFragment[] split(String tableName, long fragmentSize) throws IOException { + Path tablePath = new Path(tableBaseDir, tableName); + return split(tableName, tablePath, fragmentSize); + } + + public FileFragment[] splitBroadcastTable(Path tablePath) throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + List listTablets = new ArrayList(); + FileFragment tablet; + + FileStatus[] fileLists = fs.listStatus(tablePath); + for (FileStatus file : fileLists) { + tablet = new FileFragment(tablePath.getName(), file.getPath(), 0, file.getLen()); + listTablets.add(tablet); + } + + FileFragment[] tablets = new FileFragment[listTablets.size()]; + listTablets.toArray(tablets); + + return tablets; + } + + public FileFragment[] split(Path tablePath) throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + return split(tablePath.getName(), tablePath, fs.getDefaultBlockSize()); + } + + public FileFragment[] split(String tableName, Path tablePath) throws IOException { + return split(tableName, tablePath, fs.getDefaultBlockSize()); + } + + private FileFragment[] split(String tableName, Path tablePath, long size) + throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + + long defaultBlockSize = size; + List listTablets = new ArrayList(); + FileFragment tablet; + + FileStatus[] fileLists = fs.listStatus(tablePath); + for (FileStatus file : fileLists) { + long remainFileSize = file.getLen(); + long start = 0; + if (remainFileSize > defaultBlockSize) { + while (remainFileSize > defaultBlockSize) { + tablet = new FileFragment(tableName, file.getPath(), start, defaultBlockSize); + listTablets.add(tablet); + start += defaultBlockSize; + remainFileSize -= defaultBlockSize; + } + listTablets.add(new FileFragment(tableName, file.getPath(), start, remainFileSize)); + } else { + listTablets.add(new FileFragment(tableName, file.getPath(), 0, remainFileSize)); + } + } + + FileFragment[] tablets = new FileFragment[listTablets.size()]; + listTablets.toArray(tablets); + + return tablets; + } + + public static FileFragment[] splitNG(Configuration conf, String tableName, TableMeta meta, + Path tablePath, long size) + throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + + long defaultBlockSize = size; + List listTablets = new ArrayList(); + FileFragment tablet; + + FileStatus[] fileLists = fs.listStatus(tablePath); + for (FileStatus file : fileLists) { + long remainFileSize = file.getLen(); + long start = 0; + if (remainFileSize > defaultBlockSize) { + while (remainFileSize > defaultBlockSize) { + tablet = new FileFragment(tableName, file.getPath(), start, defaultBlockSize); + listTablets.add(tablet); + start += defaultBlockSize; + remainFileSize -= defaultBlockSize; + } + listTablets.add(new FileFragment(tableName, file.getPath(), start, remainFileSize)); + } else { + listTablets.add(new FileFragment(tableName, file.getPath(), 0, remainFileSize)); + } + } + + FileFragment[] tablets = new FileFragment[listTablets.size()]; + listTablets.toArray(tablets); + + return tablets; + } + + public long calculateSize(Path tablePath) throws IOException { + FileSystem fs = tablePath.getFileSystem(conf); + long totalSize = 0; + + if (fs.exists(tablePath)) { + totalSize = fs.getContentSummary(tablePath).getLength(); + } + + return totalSize; + } + + ///////////////////////////////////////////////////////////////////////////// + // FileInputFormat Area + ///////////////////////////////////////////////////////////////////////////// + + public static final PathFilter hiddenFileFilter = new PathFilter() { + public boolean accept(Path p) { + String name = p.getName(); + return !name.startsWith("_") && !name.startsWith("."); + } + }; + + /** + * Proxy PathFilter that accepts a path only if all filters given in the + * constructor do. Used by the listPaths() to apply the built-in + * hiddenFileFilter together with a user provided one (if any). + */ + private static class MultiPathFilter implements PathFilter { + private List filters; + + public MultiPathFilter(List filters) { + this.filters = filters; + } + + public boolean accept(Path path) { + for (PathFilter filter : filters) { + if (!filter.accept(path)) { + return false; + } + } + return true; + } + } + + /** + * List input directories. + * Subclasses may override to, e.g., select only files matching a regular + * expression. + * + * @return array of FileStatus objects + * @throws IOException if zero items. + */ + protected List listStatus(Path... dirs) throws IOException { + List result = new ArrayList(); + if (dirs.length == 0) { + throw new IOException("No input paths specified in job"); + } + + List errors = new ArrayList(); + + // creates a MultiPathFilter with the hiddenFileFilter and the + // user provided one (if any). + List filters = new ArrayList(); + filters.add(hiddenFileFilter); + + PathFilter inputFilter = new MultiPathFilter(filters); + + for (int i = 0; i < dirs.length; ++i) { + Path p = dirs[i]; + + FileSystem fs = p.getFileSystem(conf); + FileStatus[] matches = fs.globStatus(p, inputFilter); + if (matches == null) { + errors.add(new IOException("Input path does not exist: " + p)); + } else if (matches.length == 0) { + errors.add(new IOException("Input Pattern " + p + " matches 0 files")); + } else { + for (FileStatus globStat : matches) { + if (globStat.isDirectory()) { + for (FileStatus stat : fs.listStatus(globStat.getPath(), + inputFilter)) { + result.add(stat); + } + } else { + result.add(globStat); + } + } + } + } + + if (!errors.isEmpty()) { + throw new InvalidInputException(errors); + } + LOG.info("Total input paths to process : " + result.size()); + return result; + } + + /** + * Is the given filename splitable? Usually, true, but if the file is + * stream compressed, it will not be. + *

+ * FileInputFormat implementations can override this and return + * false to ensure that individual input files are never split-up + * so that Mappers process entire files. + * + * + * @param path the file name to check + * @param status get the file length + * @return is this file isSplittable? + */ + protected boolean isSplittable(TableMeta meta, Schema schema, Path path, FileStatus status) throws IOException { + Scanner scanner = getFileScanner(meta, schema, path, status); + boolean split = scanner.isSplittable(); + scanner.close(); + return split; + } + + private static final double SPLIT_SLOP = 1.1; // 10% slop + + protected int getBlockIndex(BlockLocation[] blkLocations, + long offset) { + for (int i = 0; i < blkLocations.length; i++) { + // is the offset inside this block? + if ((blkLocations[i].getOffset() <= offset) && + (offset < blkLocations[i].getOffset() + blkLocations[i].getLength())) { + return i; + } + } + BlockLocation last = blkLocations[blkLocations.length - 1]; + long fileLength = last.getOffset() + last.getLength() - 1; + throw new IllegalArgumentException("Offset " + offset + + " is outside of file (0.." + + fileLength + ")"); + } + + /** + * A factory that makes the split for this class. It can be overridden + * by sub-classes to make sub-types + */ + protected FileFragment makeSplit(String fragmentId, Path file, long start, long length) { + return new FileFragment(fragmentId, file, start, length); + } + + protected FileFragment makeSplit(String fragmentId, Path file, long start, long length, + String[] hosts) { + return new FileFragment(fragmentId, file, start, length, hosts); + } + + protected FileFragment makeSplit(String fragmentId, Path file, BlockLocation blockLocation) + throws IOException { + return new FileFragment(fragmentId, file, blockLocation); + } + + // for Non Splittable. eg, compressed gzip TextFile + protected FileFragment makeNonSplit(String fragmentId, Path file, long start, long length, + BlockLocation[] blkLocations) throws IOException { + + Map hostsBlockMap = new HashMap(); + for (BlockLocation blockLocation : blkLocations) { + for (String host : blockLocation.getHosts()) { + if (hostsBlockMap.containsKey(host)) { + hostsBlockMap.put(host, hostsBlockMap.get(host) + 1); + } else { + hostsBlockMap.put(host, 1); + } + } + } + + List> entries = new ArrayList>(hostsBlockMap.entrySet()); + Collections.sort(entries, new Comparator>() { + + @Override + public int compare(Map.Entry v1, Map.Entry v2) { + return v1.getValue().compareTo(v2.getValue()); + } + }); + + String[] hosts = new String[blkLocations[0].getHosts().length]; + + for (int i = 0; i < hosts.length; i++) { + Map.Entry entry = entries.get((entries.size() - 1) - i); + hosts[i] = entry.getKey(); + } + return new FileFragment(fragmentId, file, start, length, hosts); + } + + /** + * Get the minimum split size + * + * @return the minimum number of bytes that can be in a split + */ + public long getMinSplitSize() { + return conf.getLongVar(TajoConf.ConfVars.MINIMUM_SPLIT_SIZE); + } + + /** + * Get Disk Ids by Volume Bytes + */ + private int[] getDiskIds(VolumeId[] volumeIds) { + int[] diskIds = new int[volumeIds.length]; + for (int i = 0; i < volumeIds.length; i++) { + int diskId = -1; + if (volumeIds[i] != null && volumeIds[i].hashCode() > 0) { + diskId = volumeIds[i].hashCode() - zeroVolumeId.hashCode(); + } + diskIds[i] = diskId; + } + return diskIds; + } + + /** + * Generate the map of host and make them into Volume Ids. + * + */ + private Map> getVolumeMap(List frags) { + Map> volumeMap = new HashMap>(); + for (FileFragment frag : frags) { + String[] hosts = frag.getHosts(); + int[] diskIds = frag.getDiskIds(); + for (int i = 0; i < hosts.length; i++) { + Set volumeList = volumeMap.get(hosts[i]); + if (volumeList == null) { + volumeList = new HashSet(); + volumeMap.put(hosts[i], volumeList); + } + + if (diskIds.length > 0 && diskIds[i] > -1) { + volumeList.add(diskIds[i]); + } + } + } + + return volumeMap; + } + /** + * Generate the list of files and make them into FileSplits. + * + * @throws IOException + */ + public List getSplits(String tableName, TableMeta meta, Schema schema, Path... inputs) + throws IOException { + // generate splits' + + List splits = Lists.newArrayList(); + List volumeSplits = Lists.newArrayList(); + List blockLocations = Lists.newArrayList(); + + for (Path p : inputs) { + FileSystem fs = p.getFileSystem(conf); + ArrayList files = Lists.newArrayList(); + if (fs.isFile(p)) { + files.addAll(Lists.newArrayList(fs.getFileStatus(p))); + } else { + files.addAll(listStatus(p)); + } + + int previousSplitSize = splits.size(); + for (FileStatus file : files) { + Path path = file.getPath(); + long length = file.getLen(); + if (length > 0) { + // Get locations of blocks of file + BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length); + boolean splittable = isSplittable(meta, schema, path, file); + if (blocksMetadataEnabled && fs instanceof DistributedFileSystem) { + + if (splittable) { + for (BlockLocation blockLocation : blkLocations) { + volumeSplits.add(makeSplit(tableName, path, blockLocation)); + } + blockLocations.addAll(Arrays.asList(blkLocations)); + + } else { // Non splittable + long blockSize = blkLocations[0].getLength(); + if (blockSize >= length) { + blockLocations.addAll(Arrays.asList(blkLocations)); + for (BlockLocation blockLocation : blkLocations) { + volumeSplits.add(makeSplit(tableName, path, blockLocation)); + } + } else { + splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); + } + } + + } else { + if (splittable) { + + long minSize = Math.max(getMinSplitSize(), 1); + + long blockSize = file.getBlockSize(); // s3n rest api contained block size but blockLocations is one + long splitSize = Math.max(minSize, blockSize); + long bytesRemaining = length; + + // for s3 + while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) { + int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); + splits.add(makeSplit(tableName, path, length - bytesRemaining, splitSize, + blkLocations[blkIndex].getHosts())); + bytesRemaining -= splitSize; + } + if (bytesRemaining > 0) { + int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining); + splits.add(makeSplit(tableName, path, length - bytesRemaining, bytesRemaining, + blkLocations[blkIndex].getHosts())); + } + } else { // Non splittable + splits.add(makeNonSplit(tableName, path, 0, length, blkLocations)); + } + } + } else { + //for zero length files + splits.add(makeSplit(tableName, path, 0, length)); + } + } + if(LOG.isDebugEnabled()){ + LOG.debug("# of splits per partition: " + (splits.size() - previousSplitSize)); + } + } + + // Combine original fileFragments with new VolumeId information + setVolumeMeta(volumeSplits, blockLocations); + splits.addAll(volumeSplits); + LOG.info("Total # of splits: " + splits.size()); + return splits; + } + + private void setVolumeMeta(List splits, final List blockLocations) + throws IOException { + + int locationSize = blockLocations.size(); + int splitSize = splits.size(); + if (locationSize == 0 || splitSize == 0) return; + + if (locationSize != splitSize) { + // splits and locations don't match up + LOG.warn("Number of block locations not equal to number of splits: " + + "#locations=" + locationSize + + " #splits=" + splitSize); + return; + } + + DistributedFileSystem fs = (DistributedFileSystem)DistributedFileSystem.get(conf); + int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); + int blockLocationIdx = 0; + + Iterator iter = splits.iterator(); + while (locationSize > blockLocationIdx) { + + int subSize = Math.min(locationSize - blockLocationIdx, lsLimit); + List locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize); + //BlockStorageLocation containing additional volume location information for each replica of each block. + BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations); + + for (BlockStorageLocation blockStorageLocation : blockStorageLocations) { + iter.next().setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds())); + blockLocationIdx++; + } + } + LOG.info("# of splits with volumeId " + splitSize); + } + + private static class InvalidInputException extends IOException { + List errors; + public InvalidInputException(List errors) { + this.errors = errors; + } + + @Override + public String getMessage(){ + StringBuffer sb = new StringBuffer(); + int messageLimit = Math.min(errors.size(), 10); + for (int i = 0; i < messageLimit ; i ++) { + sb.append(errors.get(i).getMessage()).append("\n"); + } + + if(messageLimit < errors.size()) + sb.append("skipped .....").append("\n"); + + return sb.toString(); + } + } + + private static final Class[] DEFAULT_SCANNER_PARAMS = { + Configuration.class, + Schema.class, + TableMeta.class, + FileFragment.class + }; + + private static final Class[] DEFAULT_APPENDER_PARAMS = { + Configuration.class, + Schema.class, + TableMeta.class, + Path.class + }; + + /** + * create a scanner instance. + */ + public static T newScannerInstance(Class theClass, Configuration conf, Schema schema, TableMeta meta, + Fragment fragment) { + T result; + try { + Constructor meth = (Constructor) CONSTRUCTOR_CACHE.get(theClass); + if (meth == null) { + meth = theClass.getDeclaredConstructor(DEFAULT_SCANNER_PARAMS); + meth.setAccessible(true); + CONSTRUCTOR_CACHE.put(theClass, meth); + } + result = meth.newInstance(new Object[]{conf, schema, meta, fragment}); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return result; + } + + /** + * create a scanner instance. + */ + public static T newAppenderInstance(Class theClass, Configuration conf, TableMeta meta, Schema schema, + Path path) { + T result; + try { + Constructor meth = (Constructor) CONSTRUCTOR_CACHE.get(theClass); + if (meth == null) { + meth = theClass.getDeclaredConstructor(DEFAULT_APPENDER_PARAMS); + meth.setAccessible(true); + CONSTRUCTOR_CACHE.put(theClass, meth); + } + result = meth.newInstance(new Object[]{conf, schema, meta, path}); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return result; } - @Override public Class getScannerClass(CatalogProtos.StoreType storeType) throws IOException { String handlerName = storeType.name().toLowerCase(); Class scannerClass = SCANNER_HANDLER_CACHE.get(handlerName); @@ -53,7 +772,6 @@ public Class getScannerClass(CatalogProtos.StoreType storeTyp return scannerClass; } - @Override public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException { if (fragment instanceof FileFragment) { FileFragment fileFragment = (FileFragment)fragment; @@ -75,4 +793,19 @@ public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Sche return scanner; } + + public static synchronized SeekableScanner getSeekableScanner( + TajoConf conf, TableMeta meta, Schema schema, FileFragment fragment, Schema target) throws IOException { + return (SeekableScanner)getStorageManager(conf, null).getScanner(meta, schema, fragment, target); + } + + public static synchronized SeekableScanner getSeekableScanner( + TajoConf conf, TableMeta meta, Schema schema, Path path) throws IOException { + + FileSystem fs = path.getFileSystem(conf); + FileStatus status = fs.getFileStatus(path); + FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); + + return getSeekableScanner(conf, meta, schema, fragment, schema); + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java deleted file mode 100644 index 85bb8619a4..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageManagerFactory.java +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import com.google.common.collect.Maps; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.storage.v2.StorageManagerV2; - -import java.io.IOException; -import java.net.URI; -import java.util.Map; - -import static org.apache.tajo.conf.TajoConf.ConfVars; - -public class StorageManagerFactory { - private static final Map storageManagers = Maps.newHashMap(); - - public static AbstractStorageManager getStorageManager(TajoConf conf) throws IOException { - return getStorageManager(conf, null); - } - - public static synchronized AbstractStorageManager getStorageManager ( - TajoConf conf, Path warehouseDir) throws IOException { - return getStorageManager(conf, warehouseDir, conf.getBoolVar(ConfVars.STORAGE_MANAGER_VERSION_2)); - } - - private static synchronized AbstractStorageManager getStorageManager ( - TajoConf conf, Path warehouseDir, boolean v2) throws IOException { - - URI uri; - TajoConf localConf = new TajoConf(conf); - if (warehouseDir != null) { - localConf.setVar(ConfVars.WAREHOUSE_DIR, warehouseDir.toUri().toString()); - } - - uri = TajoConf.getWarehouseDir(localConf).toUri(); - - String key = "file".equals(uri.getScheme()) ? "file" : uri.toString(); - - if(v2) { - key += "_v2"; - } - - if(storageManagers.containsKey(key)) { - AbstractStorageManager sm = storageManagers.get(key); - return sm; - } else { - AbstractStorageManager storageManager; - - if(v2) { - storageManager = new StorageManagerV2(localConf); - } else { - storageManager = new StorageManager(localConf); - } - - storageManagers.put(key, storageManager); - - return storageManager; - } - } - - public static synchronized SeekableScanner getSeekableScanner( - TajoConf conf, TableMeta meta, Schema schema, FileFragment fragment, Schema target) throws IOException { - return (SeekableScanner)getStorageManager(conf, null, false).getScanner(meta, schema, fragment, target); - } - - public static synchronized SeekableScanner getSeekableScanner( - TajoConf conf, TableMeta meta, Schema schema, Path path) throws IOException { - - FileSystem fs = path.getFileSystem(conf); - FileStatus status = fs.getFileStatus(path); - FileFragment fragment = new FileFragment(path.getName(), path, 0, status.getLen()); - - return getSeekableScanner(conf, meta, schema, fragment, schema); - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java deleted file mode 100644 index e15ca6e3c3..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/CSVFileScanner.java +++ /dev/null @@ -1,386 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.Seekable; -import org.apache.hadoop.io.compress.*; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.storage.LazyTuple; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.compress.CodecPool; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.util.BytesUtils; - -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; - -public class CSVFileScanner extends FileScannerV2 { - public static final String DELIMITER = "csvfile.delimiter"; - public static final String DELIMITER_DEFAULT = "|"; - public static final byte LF = '\n'; - private static final Log LOG = LogFactory.getLog(CSVFileScanner.class); - - private final static int DEFAULT_BUFFER_SIZE = 256 * 1024; - private int bufSize; - private char delimiter; - private ScheduledInputStream sin; - private InputStream is; // decompressd stream - private CompressionCodecFactory factory; - private CompressionCodec codec; - private Decompressor decompressor; - private Seekable filePosition; - private boolean splittable = true; - private long startOffset, length; - private byte[] buf = null; - private byte[][] tuples = null; - private long[] tupleOffsets = null; - private int currentIdx = 0, validIdx = 0; - private byte[] tail = null; - private long pageStart = -1; - private long prevTailLen = -1; - private int[] targetColumnIndexes; - private boolean eof = false; - private boolean first = true; - - private long totalReadBytesForFetch; - private long totalReadBytesFromDisk; - - public CSVFileScanner(Configuration conf, final Schema schema, final TableMeta meta, final FileFragment fragment) - throws IOException { - super(conf, meta, schema, fragment); - factory = new CompressionCodecFactory(conf); - codec = factory.getCodec(this.fragment.getPath()); - if (isCompress() && !(codec instanceof SplittableCompressionCodec)) { - splittable = false; - } - } - - @Override - public void init() throws IOException { - // Buffer size, Delimiter - this.bufSize = DEFAULT_BUFFER_SIZE; - String delim = meta.getOption(DELIMITER, DELIMITER_DEFAULT); - this.delimiter = delim.charAt(0); - - super.init(); - } - - @Override - protected boolean initFirstScan(int maxBytesPerSchedule) throws IOException { - synchronized(this) { - eof = false; - first = true; - if(sin == null) { - FSDataInputStream fin = fs.open(fragment.getPath(), 128 * 1024); - sin = new ScheduledInputStream(fragment.getPath(), fin, - fragment.getStartKey(), fragment.getEndKey(), fs.getLength(fragment.getPath())); - startOffset = fragment.getStartKey(); - length = fragment.getEndKey(); - - if (startOffset > 0) { - startOffset--; // prev line feed - } - } - } - return true; - } - - private boolean scanFirst() throws IOException { - if (codec != null) { - decompressor = CodecPool.getDecompressor(codec); - if (codec instanceof SplittableCompressionCodec) { - SplitCompressionInputStream cIn = ((SplittableCompressionCodec) codec).createInputStream( - sin, decompressor, startOffset, startOffset + length, - SplittableCompressionCodec.READ_MODE.BYBLOCK); - - startOffset = cIn.getAdjustedStart(); - length = cIn.getAdjustedEnd() - startOffset; - filePosition = cIn; - is = cIn; - } else { - is = new DataInputStream(codec.createInputStream(sin, decompressor)); - } - } else { - sin.seek(startOffset); - filePosition = sin; - is = sin; - } - - tuples = new byte[0][]; - if (targets == null) { - targets = schema.toArray(); - } - - targetColumnIndexes = new int[targets.length]; - for (int i = 0; i < targets.length; i++) { - targetColumnIndexes[i] = schema.getColumnId(targets[i].getQualifiedName()); - } - - if (LOG.isDebugEnabled()) { - LOG.debug("CSVScanner open:" + fragment.getPath() + "," + startOffset + "," + length + - "," + fs.getFileStatus(fragment.getPath()).getLen()); - } - - if (startOffset != 0) { - int rbyte; - while ((rbyte = is.read()) != LF) { - if(rbyte == -1) break; - } - } - - if (fragmentable() < 1) { - close(); - return false; - } - return true; - } - - @Override - public boolean isStopScanScheduling() { - if(sin != null && sin.isEndOfStream()) { - return true; - } else { - return false; - } - } - - private long fragmentable() throws IOException { - return startOffset + length - getFilePosition(); - } - - @Override - protected long getFilePosition() throws IOException { - long retVal; - if (filePosition != null) { - retVal = filePosition.getPos(); - } else { - retVal = sin.getPos(); - } - return retVal; - } - - @Override - public boolean isFetchProcessing() { - if(sin != null && - (sin.getAvaliableSize() >= 64 * 1024 * 1024)) { - return true; - } else { - return false; - } - } - - private void page() throws IOException { - // Index initialization - currentIdx = 0; - - // Buffer size set - if (isSplittable() && fragmentable() < DEFAULT_BUFFER_SIZE) { - bufSize = (int) fragmentable(); - } - - if (this.tail == null || this.tail.length == 0) { - this.pageStart = getFilePosition(); - this.prevTailLen = 0; - } else { - this.pageStart = getFilePosition() - this.tail.length; - this.prevTailLen = this.tail.length; - } - - // Read - int rbyte; - buf = new byte[bufSize]; - rbyte = is.read(buf); - - if (rbyte < 0) { - eof = true; // EOF - return; - } - - if (prevTailLen == 0) { - tail = new byte[0]; - tuples = BytesUtils.splitPreserveAllTokens(buf, rbyte, (char) LF); - } else { - byte[] lastRow = ArrayUtils.addAll(tail, buf); - tuples = BytesUtils.splitPreserveAllTokens(lastRow, rbyte + tail.length, (char) LF); - tail = null; - } - - // Check tail - if ((char) buf[rbyte - 1] != LF) { - if ((fragmentable() < 1 || rbyte != bufSize)) { - int lineFeedPos = 0; - byte[] temp = new byte[DEFAULT_BUFFER_SIZE]; - - // find line feed - while ((temp[lineFeedPos] = (byte)is.read()) != (byte)LF) { - if(temp[lineFeedPos] < 0) { - break; - } - lineFeedPos++; - } - - tuples[tuples.length - 1] = ArrayUtils.addAll(tuples[tuples.length - 1], - ArrayUtils.subarray(temp, 0, lineFeedPos)); - validIdx = tuples.length; - } else { - tail = tuples[tuples.length - 1]; - validIdx = tuples.length - 1; - } - } else { - tail = new byte[0]; - validIdx = tuples.length - 1; - } - - if(!isCompress()) makeTupleOffset(); - } - - private void makeTupleOffset() { - long curTupleOffset = 0; - this.tupleOffsets = new long[this.validIdx]; - for (int i = 0; i < this.validIdx; i++) { - this.tupleOffsets[i] = curTupleOffset + this.pageStart; - curTupleOffset += this.tuples[i].length + 1;//tuple byte + 1byte line feed - } - } - - protected Tuple nextTuple() throws IOException { - if(first) { - boolean more = scanFirst(); - first = false; - if(!more) { - return null; - } - } - try { - if (currentIdx == validIdx) { - if (isSplittable() && fragmentable() < 1) { - close(); - return null; - } else { - page(); - } - - if(eof){ - close(); - return null; - } - } - - long offset = -1; - if(!isCompress()){ - offset = this.tupleOffsets[currentIdx]; - } - - byte[][] cells = BytesUtils.splitPreserveAllTokens(tuples[currentIdx++], delimiter, targetColumnIndexes); - return new LazyTuple(schema, cells, offset); - } catch (Throwable t) { - LOG.error(t.getMessage(), t); - } - return null; - } - - private boolean isCompress() { - return codec != null; - } - - @Override - public void scannerReset() { - if(sin != null) { - try { - filePosition.seek(0); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - } - if(sin != null) { - try { - sin.seek(0); - sin.reset(); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - } - } - - @Override - public void close() throws IOException { - if(closed.get()) { - return; - } - if(sin != null) { - totalReadBytesForFetch = sin.getTotalReadBytesForFetch(); - totalReadBytesFromDisk = sin.getTotalReadBytesFromDisk(); - } - try { - if(is != null) { - is.close(); - } - is = null; - sin = null; - } finally { - if (decompressor != null) { - CodecPool.returnDecompressor(decompressor); - decompressor = null; - } - tuples = null; - super.close(); - } - } - - @Override - protected boolean scanNext(int length) throws IOException { - synchronized(this) { - if(isClosed()) { - return false; - } - return sin.readNext(length); - } - } - - @Override - public boolean isProjectable() { - return true; - } - - @Override - public boolean isSelectable() { - return false; - } - - @Override - public void setSearchCondition(Object expr) { - } - - @Override - public boolean isSplittable(){ - return splittable; - } - - @Override - protected long[] reportReadBytes() { - return new long[]{totalReadBytesForFetch, totalReadBytesFromDisk}; - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java deleted file mode 100644 index 1babf990ad..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/DiskFileScanScheduler.java +++ /dev/null @@ -1,205 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Queue; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -import static org.apache.tajo.conf.TajoConf.ConfVars; - -public final class DiskFileScanScheduler extends Thread { - private static final Log LOG = LogFactory.getLog(DiskFileScanScheduler.class); - - private Queue requestQueue = new LinkedList(); - - List fetchingScanners = new ArrayList(); - - private int scanConcurrency; - - private AtomicInteger numOfRunningScanners = new AtomicInteger(0); - - private Object requestQueueMonitor = new Object(); // c++ code style - - private StorageManagerV2.StorgaeManagerContext smContext; - - private DiskDeviceInfo diskDeviceInfo; - - private AtomicBoolean stopped = new AtomicBoolean(false); - - private long totalScanCount = 0; - - private FetchWaitingThread fetchWaitingThread; - - private AtomicLong totalReadBytesForFetch = new AtomicLong(0); - - private AtomicLong totalReadBytesFromDisk = new AtomicLong(0); - - private long[] lastReportReadBytes; - - private long lastReportTime = 0; - - public DiskFileScanScheduler( - StorageManagerV2.StorgaeManagerContext smContext, - DiskDeviceInfo diskDeviceInfo) { - super("DiskFileScanner:" + diskDeviceInfo); - this.smContext = smContext; - this.diskDeviceInfo = diskDeviceInfo; - initScannerPool(); - this.fetchWaitingThread = new FetchWaitingThread(); - this.fetchWaitingThread.start(); - } - - public void incrementReadBytes(long[] readBytes) { - totalReadBytesForFetch.addAndGet(readBytes[0]); - totalReadBytesFromDisk.addAndGet(readBytes[1]); - } - - public int getDiskId() { - return diskDeviceInfo.getId(); - } - - public void run() { - synchronized (requestQueueMonitor) { - while(!stopped.get()) { - if(isAllScannerRunning()) { - try { - requestQueueMonitor.wait(2000); - continue; - } catch (InterruptedException e) { - break; - } - } else { - FileScannerV2 fileScanner = requestQueue.poll(); - if(fileScanner == null) { - try { - requestQueueMonitor.wait(2000); - continue; - } catch (InterruptedException e) { - break; - } - } - if(fileScanner.isStopScanScheduling()) { - LOG.info("Exit from Disk Queue:" + fileScanner.getId()); - continue; - } - if(fileScanner.isFetchProcessing()) { - synchronized(fetchingScanners) { - fetchingScanners.add(fileScanner); - //fetchingScanners.notifyAll(); - } - } else { - numOfRunningScanners.incrementAndGet(); - FileScanRunner fileScanRunner = new FileScanRunner( - DiskFileScanScheduler.this, smContext, - fileScanner, requestQueueMonitor, - numOfRunningScanners); - totalScanCount++; - fileScanRunner.start(); - } - } - } - } - } - - protected void requestScanFile(FileScannerV2 fileScannerV2) { - synchronized (requestQueueMonitor) { - requestQueue.offer(fileScannerV2); - requestQueueMonitor.notifyAll(); - } - } - - public class FetchWaitingThread extends Thread { - List workList = new ArrayList(20); - public void run() { - while(!stopped.get()) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - break; - } - workList.clear(); - synchronized(fetchingScanners) { - workList.addAll(fetchingScanners); - fetchingScanners.clear(); - } - synchronized(requestQueueMonitor) { - for(FileScannerV2 eachScanner: workList) { - requestQueue.offer(eachScanner); - } - requestQueueMonitor.notifyAll(); - } - } - } - } - - private void initScannerPool() { - // TODO finally implements heuristic, currently set with property - scanConcurrency = smContext.getConf().getIntVar(ConfVars.STORAGE_MANAGER_CONCURRENCY_PER_DISK); - } - - public int getTotalQueueSize() { - return requestQueue.size(); - } - - boolean isAllScannerRunning() { - return numOfRunningScanners.get() >= scanConcurrency; - } - - public long getTotalScanCount() { - return totalScanCount; - } - - public void stopScan() { - stopped.set(true); - if (fetchWaitingThread != null) { - fetchWaitingThread.interrupt(); - } - - this.interrupt(); - } - - public void printDiskSchedulerInfo() { - long currentReadBytes[] = new long[]{totalReadBytesForFetch.get(), totalReadBytesFromDisk.get()}; - int[] throughput = new int[2]; - if(lastReportTime != 0 && lastReportReadBytes != null) { - int sec = (int)((System.currentTimeMillis() - lastReportTime)/1000); - throughput[0] = (int)((currentReadBytes[0] - lastReportReadBytes[0])/sec); - throughput[1] = (int)((currentReadBytes[1] - lastReportReadBytes[1])/sec); - } - lastReportTime = System.currentTimeMillis(); - - LOG.info("===>" + DiskFileScanScheduler.this.diskDeviceInfo - + ", request=" + requestQueue.size() - + ", fetching=" + fetchingScanners.size() - + ", running=" + numOfRunningScanners.get() - + ", totalScan=" + totalScanCount - + ", FetchThroughput=" + throughput[0]/1024 + "KB" - + ", DiskScanThroughput=" + throughput[1]/1024 + "KB"); - - lastReportReadBytes = currentReadBytes; - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/FileScanRunner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/FileScanRunner.java deleted file mode 100644 index 07fbe6ce46..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/FileScanRunner.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import java.util.concurrent.atomic.AtomicInteger; - -public class FileScanRunner extends Thread { - private static final Log LOG = LogFactory.getLog(FileScanRunner.class); - - StorageManagerV2.StorgaeManagerContext smContext; - FileScannerV2 fileScanner; - Object requestQueueMonitor; - AtomicInteger numOfRunningScanners; - DiskFileScanScheduler diskFileScanScheduler; - - int maxReadBytes; - - public FileScanRunner(DiskFileScanScheduler diskFileScanScheduler, - StorageManagerV2.StorgaeManagerContext smContext, - FileScannerV2 fileScanner, Object requestQueueMonitor, - AtomicInteger numOfRunningScanners) { - super("FileScanRunner:" + fileScanner.getId()); - this.diskFileScanScheduler = diskFileScanScheduler; - this.fileScanner = fileScanner; - this.smContext = smContext; - this.requestQueueMonitor = requestQueueMonitor; - this.numOfRunningScanners = numOfRunningScanners; - - this.maxReadBytes = smContext.getMaxReadBytesPerScheduleSlot(); - } - - public void run() { - try { -// long startTime = System.currentTimeMillis(); -// boolean fetching = fileScanner.isFetchProcessing(); - fileScanner.scan(maxReadBytes); -// if(diskFileScanScheduler.getDiskId() == 1) { -// LOG.info("========>" + diskFileScanScheduler.getDiskId() + "," + fileScanner.getId() + -// ",fetching=" + fetching + -// ", scanTime:" + (System.currentTimeMillis() - startTime) + " ms"); -// } - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } finally { - synchronized(requestQueueMonitor) { - numOfRunningScanners.decrementAndGet(); - requestQueueMonitor.notifyAll(); - } - } - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java deleted file mode 100644 index da7084c721..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/FileScannerV2.java +++ /dev/null @@ -1,234 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.statistics.ColumnStats; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.storage.Scanner; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.fragment.FileFragment; - -import java.io.IOException; -import java.util.concurrent.atomic.AtomicBoolean; - -public abstract class FileScannerV2 implements Scanner { - private static final Log LOG = LogFactory.getLog(FileScannerV2.class); - - protected AtomicBoolean closed = new AtomicBoolean(false); - - protected FileSystem fs; - - protected boolean inited = false; - protected final Configuration conf; - protected final TableMeta meta; - protected final Schema schema; - protected final FileFragment fragment; - protected final int columnNum; - protected Column[] targets; - protected long totalScanTime = 0; - protected int allocatedDiskId; - - protected StorageManagerV2.StorgaeManagerContext smContext; - - protected AtomicBoolean firstSchdeuled = new AtomicBoolean(true); - - protected float progress; - - protected TableStats tableStats; - - protected abstract boolean scanNext(int length) throws IOException; - - protected abstract boolean initFirstScan(int maxBytesPerSchedule) throws IOException; - - protected abstract long getFilePosition() throws IOException; - - protected abstract Tuple nextTuple() throws IOException; - - public abstract boolean isFetchProcessing(); - - public abstract boolean isStopScanScheduling(); - - public abstract void scannerReset(); - - protected abstract long[] reportReadBytes(); - - public FileScannerV2(final Configuration conf, - final TableMeta meta, - final Schema schema, - final FileFragment fragment) throws IOException { - this.conf = conf; - this.meta = meta; - this.schema = schema; - this.fragment = fragment; - this.columnNum = this.schema.size(); - - this.fs = fragment.getPath().getFileSystem(conf); - } - - public void init() throws IOException { - closed.set(false); - firstSchdeuled.set(true); - - if(!inited) { - smContext.requestFileScan(this); - } - inited = true; - progress = 0.0f; - - tableStats = new TableStats(); - if (fragment != null) { - tableStats.setNumBytes(fragment.getEndKey()); - tableStats.setNumBlocks(1); - } - - if (schema != null) { - for(Column eachColumn: schema.getColumns()) { - ColumnStats columnStats = new ColumnStats(eachColumn); - tableStats.addColumnStat(columnStats); - } - } - } - - @Override - public void reset() throws IOException { - scannerReset(); - close(); - inited = false; - init(); - } - - public void setAllocatedDiskId(int allocatedDiskId) { - this.allocatedDiskId = allocatedDiskId; - } - - public String getId() { - return fragment.getPath().getName() + ":" + fragment.getStartKey() + ":" + - fragment.getEndKey() + "_" + System.currentTimeMillis(); - } - - @Override - public Schema getSchema() { - return schema; - } - - @Override - public void setTarget(Column[] targets) { - if (inited) { - throw new IllegalStateException("Should be called before init()"); - } - this.targets = targets; - } - - public Path getPath() { - return fragment.getPath(); - } - - public int getDiskId() { - if(fragment.getDiskIds().length <= 0) { - //LOG.warn("===> No DiskId:" + fragment.getPath() + ":" + fragment.getStartKey()); - return -1; - } else { - return fragment.getDiskIds()[0]; - } - } - - public void setSearchCondition(Object expr) { - if (inited) { - throw new IllegalStateException("Should be called before init()"); - } - } - - public void setStorageManagerContext(StorageManagerV2.StorgaeManagerContext context) { - this.smContext = context; - } - - public String toString() { - return fragment.getPath() + ":" + fragment.getStartKey(); - } - - public void scan(int maxBytesPerSchedule) throws IOException { - long startTime = System.currentTimeMillis(); - try { - synchronized(firstSchdeuled) { - if(firstSchdeuled.get()) { - boolean moreData = initFirstScan(maxBytesPerSchedule); - firstSchdeuled.set(false); - firstSchdeuled.notifyAll(); - if(moreData) { - smContext.requestFileScan(this); - } - return; - } - } - boolean moreData = scanNext(maxBytesPerSchedule); - - if(moreData) { - smContext.requestFileScan(this); - } - } finally { - totalScanTime += System.currentTimeMillis() - startTime; - } - } - - @Override - public void close() throws IOException { - if(closed.get()) { - return; - } - long[] readBytes = reportReadBytes(); - smContext.incrementReadBytes(allocatedDiskId, readBytes); - closed.set(true); - progress = 1.0f; - LOG.info(toString() + " closed, totalScanTime=" + totalScanTime); - } - - public boolean isClosed() { - return closed.get(); - } - - public Tuple next() throws IOException { - synchronized(firstSchdeuled) { - if(firstSchdeuled.get()) { - try { - firstSchdeuled.wait(); - } catch (InterruptedException e) { - } - } - } - return nextTuple(); - } - - @Override - public float getProgress() { - return progress; - } - - @Override - public TableStats getInputStats() { - return tableStats; - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java deleted file mode 100644 index ac585983a3..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFile.java +++ /dev/null @@ -1,1818 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.io.*; -import org.apache.hadoop.io.SequenceFile.Metadata; -import org.apache.hadoop.io.compress.*; -import org.apache.hadoop.util.Progressable; -import org.apache.hadoop.util.ReflectionUtils; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.StorageUtil; -import org.apache.tajo.storage.rcfile.*; - -import java.io.*; -import java.rmi.server.UID; -import java.security.MessageDigest; -import java.util.Arrays; - -/** - * RCFiles, short of Record Columnar File, are flat files - * consisting of binary key/value pairs, which shares much similarity with - * SequenceFile. - * - * RCFile stores columns of a table in a record columnar way. It first - * partitions rows horizontally into row splits. and then it vertically - * partitions each row split in a columnar way. RCFile first stores the meta - * data of a row split, as the key part of a record, and all the data of a row - * split as the value part. When writing, RCFile.Writer first holds records' - * value bytes in memory, and determines a row split if the raw bytes size of - * buffered records overflow a given parameterWriter.columnsBufferSize, - * which can be set like: conf.setInt(COLUMNS_BUFFER_SIZE_CONF_STR, - 4 * 1024 * 1024) . - *

- * RCFile provides {@link org.apache.tajo.storage.v2.RCFile.Writer}, {@link org.apache.tajo.storage.v2.RCFile.Reader} and classes for - * writing, reading respectively. - *

- * - *

- * RCFile stores columns of a table in a record columnar way. It first - * partitions rows horizontally into row splits. and then it vertically - * partitions each row split in a columnar way. RCFile first stores the meta - * data of a row split, as the key part of a record, and all the data of a row - * split as the value part. - *

- * - *

- * RCFile compresses values in a more fine-grained manner then record level - * compression. However, It currently does not support compress the key part - * yet. The actual compression algorithm used to compress key and/or values can - * be specified by using the appropriate {@link org.apache.hadoop.io.compress.CompressionCodec}. - *

- * - *

- * The {@link org.apache.tajo.storage.v2.RCFile.Reader} is used to read and explain the bytes of RCFile. - *

- * - *

RCFile Formats

- * - * - * - *
    - *
  • version - 3 bytes of magic header RCF, followed by 1 byte of - * actual version number (e.g. RCF1)
    • - *
  • compression - A boolean which specifies if compression is turned on for - * keys/values in this file.
    • - *
  • compression codec - CompressionCodec class which is used - * for compression of keys and/or values (if compression is enabled).
    • - *
  • metadata - {@link org.apache.hadoop.io.SequenceFile.Metadata} for this file.
    • - *
  • sync - A sync marker to denote end of the header.
    • - *
- * - *
RCFile Format
- *
    - *
  • Header
    • - *
  • Record - *
  • Key part - *
      - *
    • Record length in bytes
      • - *
    • Key length in bytes
      • - *
    • Number_of_rows_in_this_record(vint)
      • - *
    • Column_1_ondisk_length(vint)
      • - *
    • Column_1_row_1_value_plain_length
      • - *
    • Column_1_row_2_value_plain_length
      • - *
    • ...
      • - *
    • Column_2_ondisk_length(vint)
      • - *
    • Column_2_row_1_value_plain_length
      • - *
    • Column_2_row_2_value_plain_length
      • - *
    • ...
      • - *
    - *
    • - * - *
  • Value part - *
      - *
    • Compressed or plain data of [column_1_row_1_value, - * column_1_row_2_value,....]
      • - *
    • Compressed or plain data of [column_2_row_1_value, - * column_2_row_2_value,....]
      • - *
    - *
    • - *
- * - */ -public class RCFile { - - private static final Log LOG = LogFactory.getLog(RCFile.class); - - public static final String RECORD_INTERVAL_CONF_STR = "hive.io.rcfile.record.interval"; - - public static final String COLUMN_NUMBER_METADATA_STR = "hive.io.rcfile.column.number"; - - public static final String COLUMN_NUMBER_CONF_STR = "hive.io.rcfile.column.number.conf"; - - public static final String TOLERATE_CORRUPTIONS_CONF_STR = - "hive.io.rcfile.tolerate.corruptions"; - - // HACK: We actually need BlockMissingException, but that is not available - // in all hadoop versions. - public static final String BLOCK_MISSING_MESSAGE = - "Could not obtain block"; - - // All of the versions should be place in this list. - private static final int ORIGINAL_VERSION = 0; // version with SEQ - private static final int NEW_MAGIC_VERSION = 1; // version with RCF - - private static final int CURRENT_VERSION = NEW_MAGIC_VERSION; - - // The first version of RCFile used the sequence file header. - private static final byte[] ORIGINAL_MAGIC = new byte[] { - (byte) 'S', (byte) 'E', (byte) 'Q'}; - // the version that was included with the original magic, which is mapped - // into ORIGINAL_VERSION - private static final byte ORIGINAL_MAGIC_VERSION_WITH_METADATA = 6; - - private static final byte[] ORIGINAL_MAGIC_VERSION = new byte[] { - (byte) 'S', (byte) 'E', (byte) 'Q', ORIGINAL_MAGIC_VERSION_WITH_METADATA - }; - - // The 'magic' bytes at the beginning of the RCFile - private static final byte[] MAGIC = new byte[] { - (byte) 'R', (byte) 'C', (byte) 'F'}; - - private static final int SYNC_ESCAPE = -1; // "length" of sync entries - private static final int SYNC_HASH_SIZE = 16; // number of bytes in hash - private static final int SYNC_SIZE = 4 + SYNC_HASH_SIZE; // escape + hash - - /** The number of bytes between sync points. */ - public static final int SYNC_INTERVAL = 100 * SYNC_SIZE; - - /** - * KeyBuffer is the key of each record in RCFile. Its on-disk layout is as - * below: - * - *
    - *
  • record length in bytes,it is the sum of bytes used to store the key - * part and the value part.
    • - *
  • Key length in bytes, it is how many bytes used by the key part.
    • - *
  • number_of_rows_in_this_record(vint),
    • - *
  • column_1_ondisk_length(vint),
    • - *
  • column_1_row_1_value_plain_length,
    • - *
  • column_1_row_2_value_plain_length,
    • - *
  • ....
    • - *
  • column_2_ondisk_length(vint),
    • - *
  • column_2_row_1_value_plain_length,
    • - *
  • column_2_row_2_value_plain_length,
    • - *
  • .... .
    • - *
  • {the end of the key part}
    • - *
- */ - public static class KeyBuffer implements WritableComparable { - // each column's length in the value - private int[] eachColumnValueLen = null; - private int[] eachColumnUncompressedValueLen = null; - // stores each cell's length of a column in one DataOutputBuffer element - private NonSyncDataOutputBuffer[] allCellValLenBuffer = null; - // how many rows in this split - private int numberRows = 0; - // how many columns - private int columnNumber = 0; - - // return the number of columns recorded in this file's header - public int getColumnNumber() { - return columnNumber; - } - - @SuppressWarnings("unused") - @Deprecated - public KeyBuffer(){ - } - - KeyBuffer(int columnNum) { - columnNumber = columnNum; - eachColumnValueLen = new int[columnNumber]; - eachColumnUncompressedValueLen = new int[columnNumber]; - allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; - } - - @SuppressWarnings("unused") - @Deprecated - KeyBuffer(int numberRows, int columnNum) { - this(columnNum); - this.numberRows = numberRows; - } - - /** - * add in a new column's meta data. - * - * @param columnValueLen - * this total bytes number of this column's values in this split - * @param colValLenBuffer - * each cell's length of this column's in this split - */ - void setColumnLenInfo(int columnValueLen, - NonSyncDataOutputBuffer colValLenBuffer, - int columnUncompressedValueLen, int columnIndex) { - eachColumnValueLen[columnIndex] = columnValueLen; - eachColumnUncompressedValueLen[columnIndex] = columnUncompressedValueLen; - allCellValLenBuffer[columnIndex] = colValLenBuffer; - } - - @Override - public void readFields(DataInput in) throws IOException { - eachColumnValueLen = new int[columnNumber]; - eachColumnUncompressedValueLen = new int[columnNumber]; - allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; - - numberRows = WritableUtils.readVInt(in); - - for (int i = 0; i < columnNumber; i++) { - eachColumnValueLen[i] = WritableUtils.readVInt(in); - eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); - int bufLen = WritableUtils.readVInt(in); - - if (allCellValLenBuffer[i] == null) { - allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); - } else { - allCellValLenBuffer[i].reset(); - } - allCellValLenBuffer[i].write(in, bufLen); - } - } - - @Override - public void write(DataOutput out) throws IOException { - // out.writeInt(numberRows); - WritableUtils.writeVLong(out, numberRows); - for (int i = 0; i < eachColumnValueLen.length; i++) { - WritableUtils.writeVLong(out, eachColumnValueLen[i]); - WritableUtils.writeVLong(out, eachColumnUncompressedValueLen[i]); - NonSyncDataOutputBuffer colRowsLenBuf = allCellValLenBuffer[i]; - int bufLen = colRowsLenBuf.getLength(); - WritableUtils.writeVLong(out, bufLen); - out.write(colRowsLenBuf.getData(), 0, bufLen); - } - } - - /** - * get number of bytes to store the keyBuffer. - * - * @return number of bytes used to store this KeyBuffer on disk - * @throws java.io.IOException - */ - public int getSize() throws IOException { - int ret = 0; - ret += WritableUtils.getVIntSize(numberRows); - for (int i = 0; i < eachColumnValueLen.length; i++) { - ret += WritableUtils.getVIntSize(eachColumnValueLen[i]); - ret += WritableUtils.getVIntSize(eachColumnUncompressedValueLen[i]); - ret += WritableUtils.getVIntSize(allCellValLenBuffer[i].getLength()); - ret += allCellValLenBuffer[i].getLength(); - } - - return ret; - } - - @Override - public int compareTo(Object arg0) { - throw new RuntimeException("compareTo not supported in class " - + this.getClass().getName()); - } - - @Override - public boolean equals(Object obj){ - return super.equals(obj); - } - - @Override - public int hashCode(){ - return super.hashCode(); - } - } - - /** - * ValueBuffer is the value of each record in RCFile. Its on-disk layout is as - * below: - *
    - *
  • Compressed or plain data of [column_1_row_1_value, - * column_1_row_2_value,....]
    • - *
  • Compressed or plain data of [column_2_row_1_value, - * column_2_row_2_value,....]
    • - *
- */ - public static class ValueBuffer implements WritableComparable { - - class LazyDecompressionCallbackImpl implements LazyDecompressionCallback { - - int index = -1; - int colIndex = -1; - - public LazyDecompressionCallbackImpl(int index, int colIndex) { - super(); - this.index = index; - this.colIndex = colIndex; - } - - @Override - public byte[] decompress() throws IOException { - - if (decompressedFlag[index] || codec == null) { - return loadedColumnsValueBuffer[index].getData(); - } - - NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; - decompressBuffer.reset(); - DataInputStream valueIn = new DataInputStream(deflatFilter); - deflatFilter.resetState(); - if (deflatFilter instanceof SchemaAwareCompressionInputStream) { - ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); - } - decompressBuffer.reset(compressedData.getData(), - keyBuffer.eachColumnValueLen[colIndex]); - - NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; - decompressedColBuf.reset(); - decompressedColBuf.write(valueIn, - keyBuffer.eachColumnUncompressedValueLen[colIndex]); - decompressedFlag[index] = true; - numCompressed--; - return decompressedColBuf.getData(); - } - } - - // used to load columns' value into memory - private NonSyncDataOutputBuffer[] loadedColumnsValueBuffer = null; - private NonSyncDataOutputBuffer[] compressedColumnsValueBuffer = null; - private boolean[] decompressedFlag = null; - private int numCompressed; - private LazyDecompressionCallbackImpl[] lazyDecompressCallbackObjs = null; - private boolean lazyDecompress = true; - - boolean inited = false; - - // used for readFields - KeyBuffer keyBuffer; - private int columnNumber = 0; - - // set true for columns that needed to skip loading into memory. - boolean[] skippedColIDs = null; - - CompressionCodec codec; - - Decompressor valDecompressor = null; - NonSyncDataInputBuffer decompressBuffer = new NonSyncDataInputBuffer(); - CompressionInputStream deflatFilter = null; - - @SuppressWarnings("unused") - @Deprecated - public ValueBuffer() throws IOException { - } - - @SuppressWarnings("unused") - @Deprecated - public ValueBuffer(KeyBuffer keyBuffer) throws IOException { - this(keyBuffer, keyBuffer.columnNumber, null, null, true); - } - - @SuppressWarnings("unused") - @Deprecated - public ValueBuffer(KeyBuffer keyBuffer, boolean[] skippedColIDs) - throws IOException { - this(keyBuffer, keyBuffer.columnNumber, skippedColIDs, null, true); - } - - @SuppressWarnings("unused") - @Deprecated - public ValueBuffer(KeyBuffer currentKey, int columnNumber, - boolean[] skippedCols, CompressionCodec codec) throws IOException { - this(currentKey, columnNumber, skippedCols, codec, true); - } - - public ValueBuffer(KeyBuffer currentKey, int columnNumber, - boolean[] skippedCols, CompressionCodec codec, boolean lazyDecompress) - throws IOException { - this.lazyDecompress = lazyDecompress; - keyBuffer = currentKey; - this.columnNumber = columnNumber; - - if (skippedCols != null && skippedCols.length > 0) { - skippedColIDs = skippedCols; - } else { - skippedColIDs = new boolean[columnNumber]; - for (int i = 0; i < skippedColIDs.length; i++) { - skippedColIDs[i] = false; - } - } - - int skipped = 0; - for (boolean currentSkip : skippedColIDs) { - if (currentSkip) { - skipped++; - } - } - loadedColumnsValueBuffer = new NonSyncDataOutputBuffer[columnNumber - - skipped]; - decompressedFlag = new boolean[columnNumber - skipped]; - lazyDecompressCallbackObjs = new LazyDecompressionCallbackImpl[columnNumber - - skipped]; - compressedColumnsValueBuffer = new NonSyncDataOutputBuffer[columnNumber - - skipped]; - this.codec = codec; - if (codec != null) { - valDecompressor = CodecPool.getDecompressor(codec); - deflatFilter = codec.createInputStream(decompressBuffer, - valDecompressor); - } - if (codec != null) { - numCompressed = decompressedFlag.length; - } else { - numCompressed = 0; - } - for (int k = 0, readIndex = 0; k < columnNumber; k++) { - if (skippedColIDs[k]) { - continue; - } - loadedColumnsValueBuffer[readIndex] = new NonSyncDataOutputBuffer(); - if (codec != null) { - decompressedFlag[readIndex] = false; - lazyDecompressCallbackObjs[readIndex] = new LazyDecompressionCallbackImpl( - readIndex, k); - compressedColumnsValueBuffer[readIndex] = new NonSyncDataOutputBuffer(); - } else { - decompressedFlag[readIndex] = true; - } - readIndex++; - } - } - - @SuppressWarnings("unused") - @Deprecated - public void setColumnValueBuffer(NonSyncDataOutputBuffer valBuffer, - int addIndex) { - loadedColumnsValueBuffer[addIndex] = valBuffer; - } - - @Override - public void readFields(DataInput in) throws IOException { - int addIndex = 0; - int skipTotal = 0; - for (int i = 0; i < columnNumber; i++) { - int vaRowsLen = keyBuffer.eachColumnValueLen[i]; - // skip this column - if (skippedColIDs[i]) { - skipTotal += vaRowsLen; - continue; - } - - if (skipTotal != 0) { - StorageUtil.skipFully(in, skipTotal); - skipTotal = 0; - } - - NonSyncDataOutputBuffer valBuf; - if (codec != null){ - // load into compressed buf first - valBuf = compressedColumnsValueBuffer[addIndex]; - } else { - valBuf = loadedColumnsValueBuffer[addIndex]; - } - valBuf.reset(); - valBuf.write(in, vaRowsLen); - if (codec != null) { - decompressedFlag[addIndex] = false; - if (!lazyDecompress) { - lazyDecompressCallbackObjs[addIndex].decompress(); - decompressedFlag[addIndex] = true; - } - } - addIndex++; - } - if (codec != null) { - numCompressed = decompressedFlag.length; - } - - if (skipTotal != 0) { - StorageUtil.skipFully(in, skipTotal); - } - } - - @Override - public void write(DataOutput out) throws IOException { - if (codec != null) { - for (NonSyncDataOutputBuffer currentBuf : compressedColumnsValueBuffer) { - out.write(currentBuf.getData(), 0, currentBuf.getLength()); - } - } else { - for (NonSyncDataOutputBuffer currentBuf : loadedColumnsValueBuffer) { - out.write(currentBuf.getData(), 0, currentBuf.getLength()); - } - } - } - - public void clearColumnBuffer() throws IOException { - decompressBuffer.reset(); - } - - public void close() { - for (NonSyncDataOutputBuffer element : loadedColumnsValueBuffer) { - IOUtils.closeStream(element); - } - if (codec != null) { - IOUtils.closeStream(decompressBuffer); - CodecPool.returnDecompressor(valDecompressor); - } - } - - @Override - public int compareTo(Object arg0) { - throw new RuntimeException("compareTo not supported in class " - + this.getClass().getName()); - } - - @Override - public boolean equals(Object obj){ - return super.equals(obj); - } - - @Override - public int hashCode(){ - return super.hashCode(); - } - } - - /** - * Create a metadata object with alternating key-value pairs. - * Eg. metadata(key1, value1, key2, value2) - */ - public static Metadata createMetadata(Text... values) { - if (values.length % 2 != 0) { - throw new IllegalArgumentException("Must have a matched set of " + - "key-value pairs. " + values.length+ - " strings supplied."); - } - Metadata result = new Metadata(); - for(int i=0; i < values.length; i += 2) { - result.set(values[i], values[i+1]); - } - return result; - } - - /** - * Write KeyBuffer/ValueBuffer pairs to a RCFile. RCFile's format is - * compatible with SequenceFile's. - * - */ - public static class Writer { - - Configuration conf; - FSDataOutputStream out; - - CompressionCodec codec = null; - Metadata metadata = null; - - // Insert a globally unique 16-byte value every few entries, so that one - // can seek into the middle of a file and then synchronize with record - // starts and ends by scanning for this value. - long lastSyncPos; // position of last sync - byte[] sync; // 16 random bytes - { - try { - MessageDigest digester = MessageDigest.getInstance("MD5"); - long time = System.currentTimeMillis(); - digester.update((new UID() + "@" + time).getBytes()); - sync = digester.digest(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - // how many records the writer buffers before it writes to disk - private int RECORD_INTERVAL = Integer.MAX_VALUE; - // the max size of memory for buffering records before writes them out - private int columnsBufferSize = 4 * 1024 * 1024; // 4M - // the conf string for COLUMNS_BUFFER_SIZE - public static final String COLUMNS_BUFFER_SIZE_CONF_STR = "hive.io.rcfile.record.buffer.size"; - - // how many records already buffered - private int bufferedRecords = 0; - - private final ColumnBuffer[] columnBuffers; - - private int columnNumber = 0; - - private final int[] columnValuePlainLength; - - KeyBuffer key = null; - private final int[] plainTotalColumnLength; - private final int[] comprTotalColumnLength; - - boolean useNewMagic = true; - - /* - * used for buffering appends before flush them out - */ - static class ColumnBuffer { - // used for buffer a column's values - NonSyncDataOutputBuffer columnValBuffer; - // used to store each value's length - NonSyncDataOutputBuffer valLenBuffer; - - /* - * use a run-length encoding. We only record run length if a same - * 'prevValueLen' occurs more than one time. And we negative the run - * length to distinguish a runLength and a normal value length. For - * example, if the values' lengths are 1,1,1,2, we record 1, ~2,2. And for - * value lengths 1,2,3 we record 1,2,3. - */ - int runLength = 0; - int prevValueLength = -1; - - ColumnBuffer() throws IOException { - columnValBuffer = new NonSyncDataOutputBuffer(); - valLenBuffer = new NonSyncDataOutputBuffer(); - } - - public void append(BytesRefWritable data) throws IOException { - data.writeDataTo(columnValBuffer); - int currentLen = data.getLength(); - - if (prevValueLength < 0) { - startNewGroup(currentLen); - return; - } - - if (currentLen != prevValueLength) { - flushGroup(); - startNewGroup(currentLen); - } else { - runLength++; - } - } - - private void startNewGroup(int currentLen) { - prevValueLength = currentLen; - runLength = 0; - } - - public void clear() throws IOException { - valLenBuffer.reset(); - columnValBuffer.reset(); - prevValueLength = -1; - runLength = 0; - } - - public void flushGroup() throws IOException { - if (prevValueLength >= 0) { - WritableUtils.writeVLong(valLenBuffer, prevValueLength); - if (runLength > 0) { - WritableUtils.writeVLong(valLenBuffer, ~runLength); - } - runLength = -1; - prevValueLength = -1; - } - } - } - - public long getLength() throws IOException { - return out.getPos(); - } - - /** Constructs a RCFile Writer. */ - public Writer(FileSystem fs, Configuration conf, Path name) throws IOException { - this(fs, conf, name, null, new Metadata(), null); - } - - /** - * Constructs a RCFile Writer. - * - * @param fs - * the file system used - * @param conf - * the configuration file - * @param name - * the file name - * @throws java.io.IOException - */ - public Writer(FileSystem fs, Configuration conf, Path name, - Progressable progress, CompressionCodec codec) throws IOException { - this(fs, conf, name, progress, new Metadata(), codec); - } - - /** - * Constructs a RCFile Writer. - * - * @param fs - * the file system used - * @param conf - * the configuration file - * @param name - * the file name - * @param progress a progress meter to update as the file is written - * @param metadata a string to string map in the file header - * @throws java.io.IOException - */ - public Writer(FileSystem fs, Configuration conf, Path name, - Progressable progress, Metadata metadata, CompressionCodec codec) throws IOException { - this(fs, conf, name, fs.getConf().getInt("io.file.buffer.size", 4096), - fs.getDefaultReplication(), fs.getDefaultBlockSize(), progress, - metadata, codec); - } - - /** - * - * Constructs a RCFile Writer. - * - * @param fs - * the file system used - * @param conf - * the configuration file - * @param name - * the file name - * @param bufferSize the size of the file buffer - * @param replication the number of replicas for the file - * @param blockSize the block size of the file - * @param progress the progress meter for writing the file - * @param metadata a string to string map in the file header - * @throws java.io.IOException - */ - public Writer(FileSystem fs, Configuration conf, Path name, int bufferSize, - short replication, long blockSize, Progressable progress, - Metadata metadata, CompressionCodec codec) throws IOException { - RECORD_INTERVAL = conf.getInt(RECORD_INTERVAL_CONF_STR, RECORD_INTERVAL); - columnNumber = conf.getInt(COLUMN_NUMBER_CONF_STR, 0); - - if (metadata == null) { - metadata = new Metadata(); - } - metadata.set(new Text(COLUMN_NUMBER_METADATA_STR), new Text("" - + columnNumber)); - - columnsBufferSize = conf.getInt(COLUMNS_BUFFER_SIZE_CONF_STR, - 4 * 1024 * 1024); - - columnValuePlainLength = new int[columnNumber]; - - columnBuffers = new ColumnBuffer[columnNumber]; - for (int i = 0; i < columnNumber; i++) { - columnBuffers[i] = new ColumnBuffer(); - } - - init(conf, fs.create(name, true, bufferSize, replication, - blockSize, progress), codec, metadata); - initializeFileHeader(); - writeFileHeader(); - finalizeFileHeader(); - key = new KeyBuffer(columnNumber); - - plainTotalColumnLength = new int[columnNumber]; - comprTotalColumnLength = new int[columnNumber]; - } - - /** Write the initial part of file header. */ - void initializeFileHeader() throws IOException { - if (useNewMagic) { - out.write(MAGIC); - out.write(CURRENT_VERSION); - } else { - out.write(ORIGINAL_MAGIC_VERSION); - } - } - - /** Write the final part of file header. */ - void finalizeFileHeader() throws IOException { - out.write(sync); // write the sync bytes - out.flush(); // flush header - } - - boolean isCompressed() { - return codec != null; - } - - /** Write and flush the file header. */ - void writeFileHeader() throws IOException { - if (useNewMagic) { - out.writeBoolean(isCompressed()); - } else { - Text.writeString(out, KeyBuffer.class.getName()); - Text.writeString(out, ValueBuffer.class.getName()); - out.writeBoolean(isCompressed()); - out.writeBoolean(false); - } - - if (isCompressed()) { - Text.writeString(out, (codec.getClass()).getName()); - } - metadata.write(out); - } - - void init(Configuration conf, FSDataOutputStream out, - CompressionCodec codec, Metadata metadata) throws IOException { - this.conf = conf; - this.out = out; - this.codec = codec; - this.metadata = metadata; - this.useNewMagic = - conf.getBoolean(TajoConf.ConfVars.HIVEUSEEXPLICITRCFILEHEADER.varname, true); - } - - /** Returns the compression codec of data in this file. */ - @SuppressWarnings("unused") - @Deprecated - public CompressionCodec getCompressionCodec() { - return codec; - } - - /** create a sync point. */ - public void sync() throws IOException { - if (sync != null && lastSyncPos != out.getPos()) { - out.writeInt(SYNC_ESCAPE); // mark the start of the sync - out.write(sync); // write sync - lastSyncPos = out.getPos(); // update lastSyncPos - } - } - - /** Returns the configuration of this file. */ - @SuppressWarnings("unused") - @Deprecated - Configuration getConf() { - return conf; - } - - private void checkAndWriteSync() throws IOException { - if (sync != null && out.getPos() >= lastSyncPos + SYNC_INTERVAL) { - sync(); - } - } - - private int columnBufferSize = 0; - - /** - * Append a row of values. Currently it only can accept < - * {@link BytesRefArrayWritable}. If its size() is less than the - * column number in the file, zero bytes are appended for the empty columns. - * If its size() is greater then the column number in the file, the exceeded - * columns' bytes are ignored. - * - * @param val a BytesRefArrayWritable with the list of serialized columns - * @throws java.io.IOException - */ - public void append(Writable val) throws IOException { - - if (!(val instanceof BytesRefArrayWritable)) { - throw new UnsupportedOperationException( - "Currently the writer can only accept BytesRefArrayWritable"); - } - - BytesRefArrayWritable columns = (BytesRefArrayWritable) val; - int size = columns.size(); - for (int i = 0; i < size; i++) { - BytesRefWritable cu = columns.get(i); - int plainLen = cu.getLength(); - columnBufferSize += plainLen; - columnValuePlainLength[i] += plainLen; - columnBuffers[i].append(cu); - } - - if (size < columnNumber) { - for (int i = columns.size(); i < columnNumber; i++) { - columnBuffers[i].append(BytesRefWritable.ZeroBytesRefWritable); - } - } - - bufferedRecords++; - if ((columnBufferSize > columnsBufferSize) - || (bufferedRecords >= RECORD_INTERVAL)) { - flushRecords(); - } - } - - private void flushRecords() throws IOException { - - key.numberRows = bufferedRecords; - - Compressor compressor = null; - NonSyncDataOutputBuffer valueBuffer = null; - CompressionOutputStream deflateFilter = null; - DataOutputStream deflateOut = null; - boolean isCompressed = isCompressed(); - int valueLength = 0; - if (isCompressed) { - ReflectionUtils.setConf(codec, this.conf); - compressor = CodecPool.getCompressor(codec); - valueBuffer = new NonSyncDataOutputBuffer(); - deflateFilter = codec.createOutputStream(valueBuffer, compressor); - deflateOut = new DataOutputStream(deflateFilter); - } - - try { - for (int columnIndex = 0; columnIndex < columnNumber; columnIndex++) { - ColumnBuffer currentBuf = columnBuffers[columnIndex]; - currentBuf.flushGroup(); - - NonSyncDataOutputBuffer columnValue = currentBuf.columnValBuffer; - int colLen; - int plainLen = columnValuePlainLength[columnIndex]; - - if (isCompressed) { - if (deflateFilter instanceof SchemaAwareCompressionOutputStream) { - ((SchemaAwareCompressionOutputStream) deflateFilter). - setColumnIndex(columnIndex); - } - deflateFilter.resetState(); - deflateOut.write(columnValue.getData(), 0, columnValue.getLength()); - deflateOut.flush(); - deflateFilter.finish(); - // find how much compressed data was added for this column - colLen = valueBuffer.getLength() - valueLength; - } else { - colLen = columnValuePlainLength[columnIndex]; - } - valueLength += colLen; - key.setColumnLenInfo(colLen, currentBuf.valLenBuffer, plainLen, - columnIndex); - plainTotalColumnLength[columnIndex] += plainLen; - comprTotalColumnLength[columnIndex] += colLen; - columnValuePlainLength[columnIndex] = 0; - } - } catch (IOException e) { - IOUtils.cleanup(LOG, deflateOut); - throw e; - } - - int keyLength = key.getSize(); - if (keyLength < 0) { - throw new IOException("negative length keys not allowed: " + key); - } - if (compressor != null) { - CodecPool.returnCompressor(compressor); - } - - // Write the key out - writeKey(key, keyLength + valueLength, keyLength); - // write the value out - if (isCompressed) { - out.write(valueBuffer.getData(), 0, valueBuffer.getLength()); - } else { - for(int columnIndex=0; columnIndex < columnNumber; ++columnIndex) { - NonSyncDataOutputBuffer buf = - columnBuffers[columnIndex].columnValBuffer; - out.write(buf.getData(), 0, buf.getLength()); - } - } - - // clear the columnBuffers - clearColumnBuffers(); - - bufferedRecords = 0; - columnBufferSize = 0; - } - - /** - * flush a block out without doing anything except compressing the key part. - */ - public void flushBlock(KeyBuffer keyBuffer, ValueBuffer valueBuffer, - int recordLen, int keyLength, - @SuppressWarnings("unused") int compressedKeyLen) throws IOException { - writeKey(keyBuffer, recordLen, keyLength); - valueBuffer.write(out); - } - - private void writeKey(KeyBuffer keyBuffer, int recordLen, - int keyLength) throws IOException { - checkAndWriteSync(); // sync - out.writeInt(recordLen); // total record length - out.writeInt(keyLength); // key portion length - - if(this.isCompressed()) { - Compressor compressor = CodecPool.getCompressor(codec); - NonSyncDataOutputBuffer compressionBuffer = - new NonSyncDataOutputBuffer(); - CompressionOutputStream deflateFilter = - codec.createOutputStream(compressionBuffer, compressor); - DataOutputStream deflateOut = new DataOutputStream(deflateFilter); - //compress key and write key out - compressionBuffer.reset(); - deflateFilter.resetState(); - keyBuffer.write(deflateOut); - deflateOut.flush(); - deflateFilter.finish(); - int compressedKeyLen = compressionBuffer.getLength(); - out.writeInt(compressedKeyLen); - out.write(compressionBuffer.getData(), 0, compressedKeyLen); - CodecPool.returnCompressor(compressor); - } else { - out.writeInt(keyLength); - keyBuffer.write(out); - } - } - - private void clearColumnBuffers() throws IOException { - for (int i = 0; i < columnNumber; i++) { - columnBuffers[i].clear(); - } - } - - public void close() throws IOException { - if (bufferedRecords > 0) { - flushRecords(); - } - clearColumnBuffers(); - - if (out != null) { - - // Close the underlying stream if we own it... - out.flush(); - out.close(); - out = null; - } - for (int i = 0; i < columnNumber; i++) { - LOG.info("Column#" + i + " : Plain Total Column Value Length: " - + plainTotalColumnLength[i] - + ", Compr Total Column Value Length: " + comprTotalColumnLength[i]); - } - } - } - - /** - * Read KeyBuffer/ValueBuffer pairs from a RCFile. - * - */ - public static class Reader { - private static class SelectedColumn { - public int colIndex; - public int rowReadIndex; - public int runLength; - public int prvLength; - } - private final Path file; -// private final FSDataInputStream in; - - private byte version; - - private CompressionCodec codec = null; - private Metadata metadata = null; - - private final byte[] sync = new byte[SYNC_HASH_SIZE]; - private final byte[] syncCheck = new byte[SYNC_HASH_SIZE]; - private boolean syncSeen; - private long lastSeenSyncPos = 0; - - private long headerEnd; - private final long end; - private int currentKeyLength; - private int currentRecordLength; - - private final Configuration conf; - - private final ValueBuffer currentValue; - - private int readRowsIndexInBuffer = 0; - - private int recordsNumInValBuffer = 0; - - private int columnNumber = 0; - - private int loadColumnNum; - - private int passedRowsNum = 0; - - // Should we try to tolerate corruption? Default is No. - private boolean tolerateCorruptions = false; - - private boolean decompress = false; - - private Decompressor keyDecompressor; - NonSyncDataOutputBuffer keyDecompressedData = new NonSyncDataOutputBuffer(); - - //Current state of each selected column - e.g. current run length, etc. - // The size of the array is equal to the number of selected columns - private final SelectedColumn[] selectedColumns; - - // map of original column id -> index among selected columns - private final int[] revPrjColIDs; - - // column value lengths for each of the selected columns - private final NonSyncDataInputBuffer[] colValLenBufferReadIn; - - private ScheduledInputStream sin; - - public Reader(Path file, ScheduledInputStream sin, FileSystem fs, Configuration conf) throws IOException { - this(file, sin, conf.getInt("io.file.buffer.size", 4096), conf, 0, fs.getFileStatus(file).getLen()); - } - /** Create a new RCFile reader. */ - public Reader(Path file, ScheduledInputStream sin, int bufferSize, Configuration conf, - long start, long length) throws IOException { - tolerateCorruptions = conf.getBoolean( - TOLERATE_CORRUPTIONS_CONF_STR, false); - conf.setInt("io.file.buffer.size", bufferSize); -// in = openFile(fs, file, bufferSize, length); - this.file = file; - this.sin = sin; - this.conf = conf; - end = start + length; - boolean succeed = false; - try { - if (start > 0) { - seek(0); - init(); - seek(start); - } else { - seek(0); - init(); - } - succeed = true; - } finally { - if (!succeed) { - if (sin != null) { - try { - sin.close(); - } catch(IOException e) { - if (LOG != null && LOG.isDebugEnabled()) { - LOG.debug("Exception in closing " + sin, e); - } - } - } - } - } - - columnNumber = Integer.parseInt(metadata.get( - new Text(COLUMN_NUMBER_METADATA_STR)).toString()); - - java.util.ArrayList notSkipIDs = ColumnProjectionUtils - .getReadColumnIDs(conf); - boolean[] skippedColIDs = new boolean[columnNumber]; - if (notSkipIDs.size() > 0) { - for (int i = 0; i < skippedColIDs.length; i++) { - skippedColIDs[i] = true; - } - for (int read : notSkipIDs) { - if (read < columnNumber) { - skippedColIDs[read] = false; - } - } - } else { - // TODO: if no column name is specified e.g, in select count(1) from tt; - // skip all columns, this should be distinguished from the case: - // select * from tt; - for (int i = 0; i < skippedColIDs.length; i++) { - skippedColIDs[i] = false; - } - } - - loadColumnNum = columnNumber; - if (skippedColIDs.length > 0) { - for (boolean skippedColID : skippedColIDs) { - if (skippedColID) { - loadColumnNum -= 1; - } - } - } - - - revPrjColIDs = new int[columnNumber]; - // get list of selected column IDs - selectedColumns = new SelectedColumn[loadColumnNum]; - colValLenBufferReadIn = new NonSyncDataInputBuffer[loadColumnNum]; - for (int i = 0, j = 0; i < columnNumber; ++i) { - if (!skippedColIDs[i]) { - SelectedColumn col = new SelectedColumn(); - col.colIndex = i; - col.runLength = 0; - col.prvLength = -1; - col.rowReadIndex = 0; - selectedColumns[j] = col; - colValLenBufferReadIn[j] = new NonSyncDataInputBuffer(); - revPrjColIDs[i] = j; - j++; - } else { - revPrjColIDs[i] = -1; - } - } - - currentKey = createKeyBuffer(); - boolean lazyDecompress = !tolerateCorruptions; - currentValue = new ValueBuffer( - null, columnNumber, skippedColIDs, codec, lazyDecompress); - } - - /** - * Return the metadata (Text to Text map) that was written into the - * file. - */ - public Metadata getMetadata() { - return metadata; - } - - /** - * Return the metadata value associated with the given key. - * @param key the metadata key to retrieve - */ - public Text getMetadataValueOf(Text key) { - return metadata.get(key); - } - - /** - * Override this method to specialize the type of - * {@link org.apache.hadoop.fs.FSDataInputStream} returned. - */ - protected FSDataInputStream openFile(FileSystem fs, Path file, - int bufferSize, long length) throws IOException { - return fs.open(file, bufferSize); - } - - private void init() throws IOException { - byte[] magic = new byte[MAGIC.length]; - sin.readFully(magic); - - if (Arrays.equals(magic, ORIGINAL_MAGIC)) { - byte vers = sin.readByte(); - if (vers != ORIGINAL_MAGIC_VERSION_WITH_METADATA) { - throw new IOException(file + " is a version " + vers + - " SequenceFile instead of an RCFile."); - } - version = ORIGINAL_VERSION; - } else { - if (!Arrays.equals(magic, MAGIC)) { - throw new IOException(file + " not a RCFile and has magic of " + - new String(magic)); - } - - // Set 'version' - version = sin.readByte(); - if (version > CURRENT_VERSION) { - throw new VersionMismatchException((byte) CURRENT_VERSION, version); - } - } - - if (version == ORIGINAL_VERSION) { - try { - Class keyCls = conf.getClassByName(Text.readString(sin)); - Class valCls = conf.getClassByName(Text.readString(sin)); - if (!keyCls.equals(KeyBuffer.class) - || !valCls.equals(ValueBuffer.class)) { - throw new IOException(file + " not a RCFile"); - } - } catch (ClassNotFoundException e) { - throw new IOException(file + " not a RCFile", e); - } - } - - decompress = sin.readBoolean(); // is compressed? - - if (version == ORIGINAL_VERSION) { - // is block-compressed? it should be always false. - boolean blkCompressed = sin.readBoolean(); - if (blkCompressed) { - throw new IOException(file + " not a RCFile."); - } - } - - // setup the compression codec - if (decompress) { - String codecClassname = Text.readString(sin); - try { - Class codecClass = conf.getClassByName( - codecClassname).asSubclass(CompressionCodec.class); - codec = ReflectionUtils.newInstance(codecClass, conf); - } catch (ClassNotFoundException cnfe) { - throw new IllegalArgumentException( - "Unknown codec: " + codecClassname, cnfe); - } - keyDecompressor = CodecPool.getDecompressor(codec); - } - - metadata = new Metadata(); - metadata.readFields(sin); - - sin.readFully(sync); // read sync bytes - headerEnd = sin.getPos(); - } - - /** Return the current byte position in the input file. */ - public long getPosition() throws IOException { - return sin.getPos(); - } - - /** - * Set the current byte position in the input file. - * - *

- * The position passed must be a position returned by - * {@link org.apache.tajo.storage.v2.RCFile.Writer#getLength()} when writing this file. To seek to an - * arbitrary position, use {@link org.apache.tajo.storage.v2.RCFile.Reader#sync(long)}. In another - * words, the current seek can only seek to the end of the file. For other - * positions, use {@link org.apache.tajo.storage.v2.RCFile.Reader#sync(long)}. - */ - public void seek(long position) throws IOException { - sin.seek(position); - sin.readNext(128 * 1024); - } - - /** - * Resets the values which determine if there are more rows in the buffer - * - * This can be used after one calls seek or sync, if one called next before that. - * Otherwise, the seek or sync will have no effect, it will continue to get rows from the - * buffer built up from the call to next. - */ - public void resetBuffer() { - readRowsIndexInBuffer = 0; - recordsNumInValBuffer = 0; - } - - /** Seek to the next sync mark past a given position. */ - public void sync(long position) throws IOException { - if (position + SYNC_SIZE >= end) { - seek(end); - return; - } - - //this is to handle syn(pos) where pos < headerEnd. - if (position < headerEnd) { - // seek directly to first record - sin.seek(headerEnd); - // note the sync marker "seen" in the header - syncSeen = true; - return; - } - - try { -// if(sin.getAvaliableSize() < 64 * 1024) { -// sin.readNext(256 * 1024); -// } - seek(position + 4); // skip escape - sin.readFully(syncCheck); - int syncLen = sync.length; - for (int i = 0; sin.getPos() < end; i++) { - int j = 0; - for (; j < syncLen; j++) { - if (sync[j] != syncCheck[(i + j) % syncLen]) { - break; - } - } - if (j == syncLen) { - sin.seek(sin.getPos() - SYNC_SIZE); // position before - // sync - return; - } -// if(sin.getAvaliableSize() < 64 * 1024) { -// sin.readNext(256 * 1024); -// } - syncCheck[i % syncLen] = sin.readByte(); - } - } catch (ChecksumException e) { // checksum failure - handleChecksumException(e); - } - } - - private void handleChecksumException(ChecksumException e) throws IOException { - if (conf.getBoolean("io.skip.checksum.errors", false)) { - LOG.warn("Bad checksum at " + getPosition() + ". Skipping entries."); - sync(getPosition() + conf.getInt("io.bytes.per.checksum", 512)); - } else { - throw e; - } - } - - private KeyBuffer createKeyBuffer() { - return new KeyBuffer(columnNumber); - } - - /** - * Read and return the next record length, potentially skipping over a sync - * block. - * - * @return the length of the next record or -1 if there is no next record - * @throws java.io.IOException - */ - private int readRecordLength() throws IOException { - if (sin.getPos() >= end) { - return -1; - } - int length = sin.readInt(); - if (sync != null && length == SYNC_ESCAPE) { // process - // a - // sync entry - lastSeenSyncPos = sin.getPos() - 4; // minus SYNC_ESCAPE's length - sin.readFully(syncCheck); // read syncCheck - if (!Arrays.equals(sync, syncCheck)) { - throw new IOException("File is corrupt!"); - } - syncSeen = true; - if (sin.getPos() >= end) { - return -1; - } - length = sin.readInt(); // re-read length - } else { - syncSeen = false; - } - return length; - } - - private void seekToNextKeyBuffer() throws IOException { - if (!keyInit) { - return; - } - if (!currentValue.inited) { - IOUtils.skipFully(sin, currentRecordLength - currentKeyLength); - } - } - - private int compressedKeyLen = 0; - NonSyncDataInputBuffer keyDataIn = new NonSyncDataInputBuffer(); - NonSyncDataInputBuffer keyDecompressBuffer = new NonSyncDataInputBuffer(); - NonSyncDataOutputBuffer keyTempBuffer = new NonSyncDataOutputBuffer(); - - KeyBuffer currentKey = null; - boolean keyInit = false; - - protected int nextKeyBuffer() throws IOException { - seekToNextKeyBuffer(); - currentRecordLength = readRecordLength(); - if (currentRecordLength == -1) { - keyInit = false; - return -1; - } - - currentKeyLength = sin.readInt(); - compressedKeyLen = sin.readInt(); - - if (decompress) { - keyTempBuffer.reset(); - keyTempBuffer.write(sin, compressedKeyLen); - keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); - CompressionInputStream deflatFilter = codec.createInputStream( - keyDecompressBuffer, keyDecompressor); - DataInputStream compressedIn = new DataInputStream(deflatFilter); - deflatFilter.resetState(); - keyDecompressedData.reset(); - keyDecompressedData.write(compressedIn, currentKeyLength); - keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); - currentKey.readFields(keyDataIn); - } else { - currentKey.readFields(sin); - } - - keyInit = true; - currentValue.inited = false; - - readRowsIndexInBuffer = 0; - recordsNumInValBuffer = currentKey.numberRows; - - for (int selIx = 0; selIx < selectedColumns.length; selIx++) { - SelectedColumn col = selectedColumns[selIx]; - int colIx = col.colIndex; - NonSyncDataOutputBuffer buf = currentKey.allCellValLenBuffer[colIx]; - colValLenBufferReadIn[selIx].reset(buf.getData(), buf.getLength()); - col.rowReadIndex = 0; - col.runLength = 0; - col.prvLength = -1; - } - - return currentKeyLength; - } - - protected void currentValueBuffer() throws IOException { - if (!keyInit) { - nextKeyBuffer(); - } - currentValue.keyBuffer = currentKey; - currentValue.clearColumnBuffer(); - currentValue.readFields(sin); - currentValue.inited = true; - } - - public boolean nextBlock() throws IOException { - int keyLength = nextKeyBuffer(); - if(keyLength > 0) { - currentValueBuffer(); - return true; - } - return false; - } - - private boolean rowFetched = false; - - // use this buffer to hold column's cells value length for usages in - // getColumn(), instead of using colValLenBufferReadIn directly. - private final NonSyncDataInputBuffer fetchColumnTempBuf = new NonSyncDataInputBuffer(); - - /** - * Fetch all data in the buffer for a given column. This is useful for - * columnar operators, which perform operations on an array data of one - * column. It should be used together with {@link #nextColumnsBatch()}. - * Calling getColumn() with not change the result of - * {@link #next(org.apache.hadoop.io.LongWritable)} and - * {@link #getCurrentRow(BytesRefArrayWritable)}. - * - * @param columnID the number of the column to get 0 to N-1 - * @throws java.io.IOException - */ - public BytesRefArrayWritable getColumn(int columnID, - BytesRefArrayWritable rest) throws IOException { - int selColIdx = revPrjColIDs[columnID]; - if (selColIdx == -1) { - return null; - } - - if (rest == null) { - rest = new BytesRefArrayWritable(); - } - - rest.resetValid(recordsNumInValBuffer); - - if (!currentValue.inited) { - currentValueBuffer(); - } - - int columnNextRowStart = 0; - fetchColumnTempBuf.reset(currentKey.allCellValLenBuffer[columnID] - .getData(), currentKey.allCellValLenBuffer[columnID].getLength()); - SelectedColumn selCol = selectedColumns[selColIdx]; - byte[] uncompData = null; - ValueBuffer.LazyDecompressionCallbackImpl decompCallBack = null; - boolean decompressed = currentValue.decompressedFlag[selColIdx]; - if (decompressed) { - uncompData = - currentValue.loadedColumnsValueBuffer[selColIdx].getData(); - } else { - decompCallBack = currentValue.lazyDecompressCallbackObjs[selColIdx]; - } - for (int i = 0; i < recordsNumInValBuffer; i++) { - colAdvanceRow(selColIdx, selCol); - int length = selCol.prvLength; - - BytesRefWritable currentCell = rest.get(i); - - if (decompressed) { - currentCell.set(uncompData, columnNextRowStart, length); - } else { - currentCell.set(decompCallBack, columnNextRowStart, length); - } - columnNextRowStart = columnNextRowStart + length; - } - return rest; - } - - /** - * Read in next key buffer and throw any data in current key buffer and - * current value buffer. It will influence the result of - * {@link #next(org.apache.hadoop.io.LongWritable)} and - * {@link #getCurrentRow(BytesRefArrayWritable)} - * - * @return whether there still has records or not - * @throws java.io.IOException - */ - @SuppressWarnings("unused") - @Deprecated - public boolean nextColumnsBatch() throws IOException { - passedRowsNum += (recordsNumInValBuffer - readRowsIndexInBuffer); - return nextKeyBuffer() > 0; - } - - /** - * Returns how many rows we fetched with next(). It only means how many rows - * are read by next(). The returned result may be smaller than actual number - * of rows passed by, because {@link #seek(long)}, - * {@link #nextColumnsBatch()} can change the underlying key buffer and - * value buffer. - * - * @return next row number - * @throws java.io.IOException - */ - public boolean next(LongWritable readRows) throws IOException { - if (hasRecordsInBuffer()) { - readRows.set(passedRowsNum); - readRowsIndexInBuffer++; - passedRowsNum++; - rowFetched = false; - return true; - } else { - keyInit = false; - } - - int ret = -1; - if (tolerateCorruptions) { - ret = nextKeyValueTolerateCorruptions(); - } else { - try { - ret = nextKeyBuffer(); - } catch (EOFException eof) { - eof.printStackTrace(); - } - } - return (ret > 0) && next(readRows); - } - - private int nextKeyValueTolerateCorruptions() throws IOException { - long currentOffset = sin.getPos(); - int ret; - try { - ret = nextKeyBuffer(); - this.currentValueBuffer(); - } catch (IOException ioe) { - // A BlockMissingException indicates a temporary error, - // not a corruption. Re-throw this exception. - String msg = ioe.getMessage(); - if (msg != null && msg.startsWith(BLOCK_MISSING_MESSAGE)) { - LOG.warn("Re-throwing block-missing exception" + ioe); - throw ioe; - } - // We have an IOException other than a BlockMissingException. - LOG.warn("Ignoring IOException in file " + file + - " after offset " + currentOffset, ioe); - ret = -1; - } catch (Throwable t) { - // We got an exception that is not IOException - // (typically OOM, IndexOutOfBounds, InternalError). - // This is most likely a corruption. - LOG.warn("Ignoring unknown error in " + file + - " after offset " + currentOffset, t); - ret = -1; - } - return ret; - } - - public boolean hasRecordsInBuffer() { - return readRowsIndexInBuffer < recordsNumInValBuffer; - } - - /** - * get the current row used,make sure called {@link #next(org.apache.hadoop.io.LongWritable)} - * first. - * - * @throws java.io.IOException - */ - public void getCurrentRow(BytesRefArrayWritable ret) throws IOException { - - if (!keyInit || rowFetched) { - return; - } - - if (tolerateCorruptions) { - if (!currentValue.inited) { - currentValueBuffer(); - } - ret.resetValid(columnNumber); - } else { - if (!currentValue.inited) { - currentValueBuffer(); - // do this only when not initialized, but we may need to find a way to - // tell the caller how to initialize the valid size - ret.resetValid(columnNumber); - } - } - - // we do not use BytesWritable here to avoid the byte-copy from - // DataOutputStream to BytesWritable - if (currentValue.numCompressed > 0) { - for (int j = 0; j < selectedColumns.length; ++j) { - SelectedColumn col = selectedColumns[j]; - int i = col.colIndex; - - BytesRefWritable ref = ret.unCheckedGet(i); - - colAdvanceRow(j, col); - - if (currentValue.decompressedFlag[j]) { - ref.set(currentValue.loadedColumnsValueBuffer[j].getData(), - col.rowReadIndex, col.prvLength); - } else { - ref.set(currentValue.lazyDecompressCallbackObjs[j], - col.rowReadIndex, col.prvLength); - } - col.rowReadIndex += col.prvLength; - } - } else { - // This version of the loop eliminates a condition check and branch - // and is measurably faster (20% or so) - for (int j = 0; j < selectedColumns.length; ++j) { - SelectedColumn col = selectedColumns[j]; - int i = col.colIndex; - - BytesRefWritable ref = ret.unCheckedGet(i); - - colAdvanceRow(j, col); - ref.set(currentValue.loadedColumnsValueBuffer[j].getData(), - col.rowReadIndex, col.prvLength); - col.rowReadIndex += col.prvLength; - } - } - rowFetched = true; - } - - /** - * Advance column state to the next now: update offsets, run lengths etc - * @param selCol - index among selectedColumns - * @param col - column object to update the state of. prvLength will be - * set to the new read position - * @throws java.io.IOException - */ - private void colAdvanceRow(int selCol, SelectedColumn col) throws IOException { - if (col.runLength > 0) { - --col.runLength; - } else { - int length = (int) WritableUtils.readVLong(colValLenBufferReadIn[selCol]); - if (length < 0) { - // we reach a runlength here, use the previous length and reset - // runlength - col.runLength = (~length) - 1; - } else { - col.prvLength = length; - col.runLength = 0; - } - } - } - - /** Returns true iff the previous call to next passed a sync mark. */ - @SuppressWarnings("unused") - public boolean syncSeen() { - return syncSeen; - } - - /** Returns the last seen sync position. */ - public long lastSeenSyncPos() { - return lastSeenSyncPos; - } - - /** Returns the name of the file. */ - @Override - public String toString() { - return file.toString(); - } - - @SuppressWarnings("unused") - public boolean isCompressedRCFile() { - return this.decompress; - } - - /** Close the reader. */ - public void close() { - IOUtils.closeStream(sin); - currentValue.close(); - if (decompress) { - IOUtils.closeStream(keyDecompressedData); - CodecPool.returnDecompressor(keyDecompressor); - } - } - - /** - * return the KeyBuffer object used in the reader. Internally in each - * reader, there is only one KeyBuffer object, which gets reused for every - * block. - */ - public KeyBuffer getCurrentKeyBufferObj() { - return this.currentKey; - } - - /** - * return the ValueBuffer object used in the reader. Internally in each - * reader, there is only one ValueBuffer object, which gets reused for every - * block. - */ - public ValueBuffer getCurrentValueBufferObj() { - return this.currentValue; - } - - //return the current block's length - public int getCurrentBlockLength() { - return this.currentRecordLength; - } - - //return the current block's key length - public int getCurrentKeyLength() { - return this.currentKeyLength; - } - - //return the current block's compressed key length - public int getCurrentCompressedKeyLen() { - return this.compressedKeyLen; - } - - //return the CompressionCodec used for this file - public CompressionCodec getCompressionCodec() { - return this.codec; - } - - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java deleted file mode 100644 index c8e19dda68..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/RCFileScanner.java +++ /dev/null @@ -1,300 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.commons.lang.StringEscapeUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.storage.BinarySerializerDeserializer; -import org.apache.tajo.storage.SerializerDeserializer; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.storage.rcfile.BytesRefArrayWritable; -import org.apache.tajo.storage.rcfile.ColumnProjectionUtils; -import org.apache.tajo.util.TUtil; - -import java.io.IOException; -import java.util.ArrayList; - -public class RCFileScanner extends FileScannerV2 { - private static final Log LOG = LogFactory.getLog(RCFileScanner.class); - public static final String SERDE = "rcfile.serde"; - public static final String NULL = "rcfile.null"; - - private RCFile.Reader in; - private long start; - private long end; - private boolean more = true; - private LongWritable key; - private BytesRefArrayWritable column; - private Integer [] projectionMap; - private ScheduledInputStream sin; - private boolean first = true; - private int maxBytesPerSchedule; - private SerializerDeserializer serde; - private byte[] nullChars; - private Object lock = new Object(); - - public RCFileScanner(final Configuration conf, final Schema schema, final TableMeta meta, final FileFragment fragment) - throws IOException { - super(conf, meta, schema, fragment); - - this.start = fragment.getStartKey(); - this.end = start + fragment.getEndKey(); - key = new LongWritable(); - column = new BytesRefArrayWritable(); - - String nullCharacters = StringEscapeUtils.unescapeJava(this.meta.getOption(NULL, NullDatum.DEFAULT_TEXT)); - if (StringUtils.isEmpty(nullCharacters)) { - nullChars = NullDatum.get().asTextBytes(); - } else { - nullChars = nullCharacters.getBytes(); - } - } - - @Override - protected Tuple nextTuple() throws IOException { - if(first) { - first = false; - if (start > in.getPosition()) { - in.sync(start); // sync to start - } - this.start = in.getPosition(); - more = start < end; - if(!more) { - return null; - } - } - - more = next(key); - - if (more) { - column.clear(); - in.getCurrentRow(column); - } - - if(more) { - Tuple tuple = makeTuple(); - return tuple; - } else { - close(); - return null; - } - } - - private Tuple makeTuple() throws IOException { - Tuple tuple = new VTuple(schema.size()); - synchronized (lock) { - column.resetValid(schema.size()); - int tid; // target column id - for (int i = 0; i < projectionMap.length; i++) { - tid = projectionMap[i]; - - byte[] bytes = column.get(tid).getBytesCopy(); - Datum datum = serde.deserialize(targets[i], bytes, 0, bytes.length, nullChars); - tuple.put(tid, datum); - } - } - return tuple; - } - - @Override - public void init() throws IOException { - if (targets == null) { - targets = schema.toArray(); - } - - prepareProjection(targets); - - super.init(); - } - - private void prepareProjection(Column[] targets) { - projectionMap = new Integer[targets.length]; - int tid; - for (int i = 0; i < targets.length; i++) { - tid = schema.getColumnIdByName(targets[i].getSimpleName()); - projectionMap[i] = tid; - } - ArrayList projectionIdList = new ArrayList(TUtil.newList(projectionMap)); - ColumnProjectionUtils.setReadColumnIDs(conf, projectionIdList); - } - - @Override - public void close() throws IOException { - if(closed.get()) { - return; - } - try { - if(in != null) { - in.close(); - in = null; - sin = null; - } - } catch (Exception e) { - LOG.warn(e.getMessage(), e); - } - - if(column != null) { - column.clear(); - column = null; - } - super.close(); - } - - private boolean next(LongWritable key) throws IOException { - if (!more) { - return false; - } - - more = in.next(key); - if (!more) { - return false; - } - - long lastSeenSyncPos = in.lastSeenSyncPos(); - if (lastSeenSyncPos >= end) { - more = false; - return more; - } - return more; - } - - @Override - protected boolean initFirstScan(int maxBytesPerSchedule) throws IOException { - synchronized(lock) { - first = true; - this.maxBytesPerSchedule = maxBytesPerSchedule; - if(sin == null) { - sin = new ScheduledInputStream( - fragment.getPath(), - fs.open(fragment.getPath()), - fragment.getStartKey(), - fragment.getEndKey(), - fs.getLength(fragment.getPath())); - - this.in = new RCFile.Reader(fragment.getPath(), sin, fs, fs.getConf()); - - Text text = this.in.getMetadata().get(new Text(SERDE)); - - try { - String serdeClass; - if(text != null && !text.toString().isEmpty()){ - serdeClass = text.toString(); - } else{ - serdeClass = this.meta.getOption(SERDE, BinarySerializerDeserializer.class.getName()); - } - serde = (SerializerDeserializer) Class.forName(serdeClass).newInstance(); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new IOException(e); - } - } - } - return true; - } - - @Override - public boolean isStopScanScheduling() { - if(sin != null && sin.isEndOfStream()) { - return true; - } else { - return false; - } - } - - @Override - protected boolean scanNext(int length) throws IOException { - synchronized(lock) { - if(isClosed()) { - return false; - } - return sin.readNext(length); - } - } - - - @Override - public boolean isFetchProcessing() { - //TODO row group size - if(sin != null && sin.getAvaliableSize() > maxBytesPerSchedule * 3) { - return true; - } else { - return false; - } - } - - @Override - protected long getFilePosition() throws IOException { - return in.getPosition(); - } - - @Override - public void scannerReset() { - if(in != null) { - try { - in.seek(0); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - } - if(sin != null) { - try { - sin.seek(0); - sin.reset(); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - } - } - - @Override - public boolean isProjectable() { - return true; - } - - @Override - public boolean isSelectable() { - return false; - } - - @Override - public boolean isSplittable(){ - return true; - } - - @Override - protected long[] reportReadBytes() { - if(sin == null) { - return new long[]{0, 0}; - } else { - return new long[]{sin.getTotalReadBytesForFetch(), sin.getTotalReadBytesFromDisk()}; - } - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java deleted file mode 100644 index fe852a0fc7..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/ScanScheduler.java +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tajo.storage.v2.StorageManagerV2.StorgaeManagerContext; - -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.apache.tajo.conf.TajoConf.ConfVars; - -public final class ScanScheduler extends Thread { - private static final Log LOG = LogFactory.getLog(ScanScheduler.class); - - private final Object scanQueueLock; - private StorgaeManagerContext context; - - private Map requestMap = new HashMap(); - - private final Map diskFileScannerMap = new HashMap(); - - private Map diskDeviceInfoMap = new HashMap(); - - private SortedSet diskMountInfos = new TreeSet(); - - private AtomicBoolean stopped = new AtomicBoolean(false); - - private Random rand = new Random(System.currentTimeMillis()); - - private Thread schedulerStatusReportThread; - - public ScanScheduler(StorgaeManagerContext context) { - this.context = context; - this.scanQueueLock = context.getScanQueueLock(); - - try { - List deviceInfos = DiskUtil.getDiskDeviceInfos(); - if (deviceInfos.size() == 0) { - deviceInfos = DiskUtil.getDefaultDiskDeviceInfos(); - } - for(DiskDeviceInfo eachInfo: deviceInfos) { - LOG.info("Create DiskScanQueue:" + eachInfo.getName()); - diskDeviceInfoMap.put(eachInfo.getId(), eachInfo); - - diskMountInfos.addAll(eachInfo.getMountInfos()); - } - - initFileScanners(); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - } - - final int reportInterval = context.getConf().getIntVar(ConfVars.STORAGE_MANAGER_DISK_SCHEDULER_REPORT_INTERVAL); - if(reportInterval > 0) { - schedulerStatusReportThread = new Thread() { - public void run() { - while (true) { - try { - Thread.sleep(reportInterval); - } catch (InterruptedException e) { - break; - } - synchronized (diskFileScannerMap) { - for (DiskFileScanScheduler eachScheduler : diskFileScannerMap - .values()) { - eachScheduler.printDiskSchedulerInfo(); - } - } - } - } - }; - - schedulerStatusReportThread.start(); - } - } - - public void run() { - synchronized(scanQueueLock) { - while(!stopped.get()) { - FileScannerV2 fileScannerV2 = context.getScanQueue().poll(); - if(fileScannerV2 == null) { - try { - scanQueueLock.wait(); - } catch (InterruptedException e) { - break; - } - } else { - int diskId = fileScannerV2.getDiskId(); - - int emptyDiskId = findEmptyDisk(); - if(emptyDiskId < 0) { - if(diskId < 0 || diskId >= diskDeviceInfoMap.size()) { - diskId = findDiskPartitionPath(fileScannerV2.getPath().toString()); - if(diskId < 0) { - - diskId = findMinQueueDisk(); - if(diskId < 0) { - diskId = rand.nextInt(diskDeviceInfoMap.size()); - } - } - } - } else { - diskId = emptyDiskId; - } - synchronized(diskFileScannerMap) { - requestMap.put(fileScannerV2.getId(), fileScannerV2); - DiskFileScanScheduler diskScheduler = diskFileScannerMap.get(diskId); - fileScannerV2.setAllocatedDiskId(diskId); - diskScheduler.requestScanFile(fileScannerV2); - } - } - } - } - } - - private int findEmptyDisk() { - synchronized(diskFileScannerMap) { - for(DiskFileScanScheduler eachDiskScanner: diskFileScannerMap.values()) { - int queueSize = eachDiskScanner.getTotalQueueSize(); - if(queueSize == 0) { - return eachDiskScanner.getDiskId(); - } - } - return -1; - } - } - - private int findMinQueueDisk() { - int minValue = Integer.MAX_VALUE; - int minId = -1; - synchronized(diskFileScannerMap) { - for(DiskFileScanScheduler eachDiskScanner: diskFileScannerMap.values()) { - int queueSize = eachDiskScanner.getTotalQueueSize(); - if(queueSize <= minValue) { - minValue = queueSize; - minId = eachDiskScanner.getDiskId(); - } - } - } - - return minId; - } - - private int findDiskPartitionPath(String fullPath) { - for (DiskMountInfo eachMountInfo : diskMountInfos) { - if (fullPath.indexOf(eachMountInfo.getMountPath()) == 0) { - return eachMountInfo.getDeviceId(); - } - } - - return -1; - } - - public void incrementReadBytes(int diskId, long[] readBytes) { - diskFileScannerMap.get(diskId).incrementReadBytes(readBytes); - } - - private void initFileScanners() { - for(Integer eachId: diskDeviceInfoMap.keySet()) { - DiskFileScanScheduler scanner = new DiskFileScanScheduler(context, diskDeviceInfoMap.get(eachId)); - scanner.start(); - - diskFileScannerMap.put(eachId, scanner); - } - } - - public void stopScheduler() { - stopped.set(true); - for(DiskFileScanScheduler eachDiskScanner: diskFileScannerMap.values()) { - eachDiskScanner.stopScan(); - } - this.interrupt(); - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/ScheduledInputStream.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/ScheduledInputStream.java deleted file mode 100644 index 12b984edb8..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/ScheduledInputStream.java +++ /dev/null @@ -1,513 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.Seekable; - -import java.io.*; -import java.util.LinkedList; -import java.util.Queue; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -public class ScheduledInputStream extends InputStream implements Seekable, Closeable, DataInput { - private static final Log LOG = LogFactory.getLog(ScheduledInputStream.class); - - private FSDataInputStream originStream; - - private int currentScanIndex; - - private Queue dataQueue = new LinkedList(); - - private ScanData currentScanData; - - private AtomicBoolean closed = new AtomicBoolean(false); - - private boolean eof = false; - - private long pos; - - private AtomicInteger avaliableSize = new AtomicInteger(0); - - private long fileLen; - - private long startOffset; - - private long length; - - private long endOffset; - - private boolean endOfStream = false; - - private Path file; - - private byte readLongBuffer[] = new byte[8]; - - private AtomicLong totalReadBytesForFetch = new AtomicLong(0); - - private AtomicLong totalReadBytesFromDisk = new AtomicLong(0); - - public ScheduledInputStream(Path file, FSDataInputStream originStream, - long startOffset, long length, long fileLen) throws IOException { - this.originStream = originStream; - this.startOffset = startOffset; - this.length = length; - this.endOffset = startOffset + length; - this.fileLen = fileLen; - this.file = file; - this.pos = this.originStream.getPos(); - - LOG.info("Open:" + toString()); - } - - public int getAvaliableSize() { - return avaliableSize.get(); - } - - public String toString() { - return file.getName() + ":" + startOffset + ":" + length; - } - public boolean readNext(int length) throws IOException { - return readNext(length, false); - } - - public boolean readNext(int length, boolean ignoreEOS) throws IOException { - synchronized(dataQueue) { - if(closed.get() || (!ignoreEOS && endOfStream)) { - return false; - } - int bufLength = ignoreEOS ? length : (int)Math.min(length, endOffset - originStream.getPos()); - bufLength = (int)Math.min(bufLength, fileLen - originStream.getPos()); - if(bufLength == 0) { - return false; - } - byte[] buf = new byte[bufLength]; - - try { - originStream.readFully(buf); - } catch (EOFException e) { - LOG.error(e.getMessage(), e); - throw e; - } catch (Exception e) { - throw new IOException(e.getMessage(), e); - } - - if(originStream.getPos() == fileLen) { - LOG.info("EOF:" + toString()); - eof = true; - } - if(!ignoreEOS && originStream.getPos() >= endOffset) { - LOG.info("EndOfStream:" + toString()); - endOfStream = true; - } - - if(currentScanData == null) { - currentScanData = new ScanData(buf, bufLength); - currentScanIndex = 0; - } else { - dataQueue.offer(new ScanData(buf, bufLength)); - } - - avaliableSize.addAndGet(bufLength); - - if(LOG.isDebugEnabled()) { - LOG.debug("Add DataQueue: queue=" + dataQueue.size() + - ", avaliable Size=" + avaliableSize.get() + ", pos=" + getPos() + - ", originPos=" + originStream.getPos() + ",endOfStream=" + endOfStream + - ", bufLength=" + bufLength + ",ignoreEOS=" + ignoreEOS); - } - - totalReadBytesFromDisk.addAndGet(bufLength); - dataQueue.notifyAll(); - } - return !eof; - } - - static class ScanData { - byte[] data; - int length; - public ScanData(byte[] buf, int length) { - this.data = buf; - this.length = length; - } - - @Override - public String toString() { - return "length=" + length; - } - } - - @Override - public void seek(long pos) throws IOException { - synchronized(dataQueue) { - dataQueue.clear(); - currentScanData = null; - currentScanIndex = 0; - avaliableSize.set(0); - originStream.seek(pos); - this.pos = pos; - } - } - - @Override - public long getPos() throws IOException { - return this.pos; - } - - public long getOriginStreamPos() { - try { - return this.originStream.getPos(); - } catch (IOException e) { - e.printStackTrace(); - return 0; - } - } - - @Override - public boolean seekToNewSource(long targetPos) throws IOException { - synchronized(dataQueue) { - dataQueue.clear(); - currentScanData = null; - currentScanIndex = 0; - avaliableSize.set(0); - boolean result = originStream.seekToNewSource(targetPos); - - this.pos = originStream.getPos(); - return result; - } - } - - @Override - public int read() throws IOException { - if(noMoreData()) { - return -1; - } - if(currentScanData == null || currentScanIndex >= currentScanData.length) { - synchronized(dataQueue) { - if(dataQueue.isEmpty()) { - if(endOfStream) { - readNext(64 * 1024, true); - } else { - try { - dataQueue.wait(); - if(eof && dataQueue.isEmpty() && currentScanIndex > 0) { - //no more data - return -1; - } - } catch (InterruptedException e) { - } - } - } - if(!dataQueue.isEmpty() && currentScanIndex > 0) { - currentScanData = dataQueue.poll(); - currentScanIndex = 0; - } - } - } - - this.pos++; - avaliableSize.decrementAndGet(); - totalReadBytesForFetch.incrementAndGet(); - - return currentScanData.data[currentScanIndex++] & 0xff; - } - - private boolean noMoreData() { - return closed.get(); - } - - public int read(byte b[], int off, int len) throws IOException { - if(noMoreData()) { - return -1; - } - if (b == null) { - throw new NullPointerException(); - } else if (off < 0 || len < 0 || len > b.length - off) { - throw new IndexOutOfBoundsException(); - } else if (len == 0) { - return 0; - } - if(currentScanData == null) { - synchronized(dataQueue) { - if(dataQueue.isEmpty()) { - if(endOfStream) { - readNext(64 * 1024, true); - } else { - try { - dataQueue.wait(); - if(noMoreData()) { - return -1; - } - } catch (InterruptedException e) { - } - } - } - if(!dataQueue.isEmpty() && currentScanIndex > 0) { - currentScanData = dataQueue.poll(); - currentScanIndex = 0; - } - } - } - - int numRemainBytes = currentScanData.length - currentScanIndex; - if(numRemainBytes > len) { - System.arraycopy(currentScanData.data, currentScanIndex, b, off, len); - currentScanIndex += len; - avaliableSize.addAndGet(0 - len); - pos += len; - - totalReadBytesForFetch.addAndGet(len); - return len; - } else { - int offset = off; - int length = 0; - int numCopyBytes = numRemainBytes; - while(true) { - synchronized(dataQueue) { - if(numCopyBytes == 0 && eof && dataQueue.isEmpty()) { - return -1; - } - } - System.arraycopy(currentScanData.data, currentScanIndex, b, offset, numCopyBytes); - currentScanIndex += numCopyBytes; - offset += numCopyBytes; - length += numCopyBytes; - if(length >= len) { - break; - } - synchronized(dataQueue) { - if(dataQueue.isEmpty()) { - if(eof) { - break; - } - if(endOfStream) { - readNext(64 * 1024, true); - } else { - try { - dataQueue.wait(); - } catch (InterruptedException e) { - } - } - } - if(eof && dataQueue.isEmpty()) { - break; - } - if(!dataQueue.isEmpty() && currentScanIndex > 0) { - currentScanData = dataQueue.poll(); - currentScanIndex = 0; - } - if(currentScanData == null) { - break; - } - } - if(currentScanData.length >= (len - length)) { - numCopyBytes = (len - length); - } else { - numCopyBytes = currentScanData.length; - } - } //end of while - this.pos += length; - avaliableSize.addAndGet(0 - length); - - totalReadBytesForFetch.addAndGet(length); - return length; - } - } - - public long getTotalReadBytesForFetch() { - return totalReadBytesForFetch.get(); - } - - public long getTotalReadBytesFromDisk() { - return totalReadBytesFromDisk.get(); - } - - @Override - public void close() throws IOException { - LOG.info("Close:" + toString()); - synchronized(dataQueue) { - if(closed.get()) { - return; - } - closed.set(true); - originStream.close(); - dataQueue.clear(); - currentScanIndex = 0; - super.close(); - } - } - - @Override - public void readFully(byte[] b) throws IOException { - readFully(b, 0, b.length); - } - - @Override - public void readFully(byte[] b, int off, int len) throws IOException { - if (len < 0) { - throw new IndexOutOfBoundsException(); - } - int n = 0; - while (n < len) { - int count = read(b, off + n, len - n); - if (count < 0) { - throw new EOFException(); - } - n += count; - } - } - - @Override - public int skipBytes(int bytes) throws IOException { - int skipTotal = 0; - int currentPos = 0; - - while ((skipTotal 0)) { - skipTotal += currentPos; - } - - return skipTotal; - } - - @Override - public boolean readBoolean() throws IOException { - int val = read(); - if (val < 0) { - throw new EOFException(); - } - return (val != 0); - } - - @Override - public byte readByte() throws IOException { - int val = read(); - if (val < 0) { - throw new EOFException(); - } - return (byte)(val); - } - - @Override - public int readUnsignedByte() throws IOException { - int val = read(); - if (val < 0) { - throw new EOFException(); - } - return val; - } - - @Override - public short readShort() throws IOException { - int val1 = read(); - int val2 = read(); - if ((val1 | val2) < 0) { - throw new EOFException(); - } - return (short)((val1 << 8) + (val2 << 0)); - } - - @Override - public int readUnsignedShort() throws IOException { - int val1 = read(); - int val2 = read(); - if ((val1 | val2) < 0) { - throw new EOFException(); - } - return (val1 << 8) + (val2 << 0); - } - - @Override - public char readChar() throws IOException { - int val1 = read(); - int val2 = read(); - if ((val1 | val2) < 0) { - throw new EOFException(); - } - return (char)((val1 << 8) + (val2 << 0)); - } - - @Override - public int readInt() throws IOException { - int val1 = read(); - int val2 = read(); - int val3 = read(); - int val4 = read(); - if ((val1 | val2 | val3 | val4) < 0) { - throw new EOFException(); - } - return ((val1 << 24) + (val2 << 16) + (val3 << 8) + (val4 << 0)); - } - - @Override - public long readLong() throws IOException { - readFully(readLongBuffer, 0, 8); - return (((long) readLongBuffer[0] << 56) + - ((long)(readLongBuffer[1] & 255) << 48) + - ((long)(readLongBuffer[2] & 255) << 40) + - ((long)(readLongBuffer[3] & 255) << 32) + - ((long)(readLongBuffer[4] & 255) << 24) + - ((readLongBuffer[5] & 255) << 16) + - ((readLongBuffer[6] & 255) << 8) + - ((readLongBuffer[7] & 255) << 0)); - } - - @Override - public float readFloat() throws IOException { - return Float.intBitsToFloat(readInt()); - } - - @Override - public double readDouble() throws IOException { - return Double.longBitsToDouble(readLong()); - } - - @Override - public String readLine() throws IOException { - throw new IOException("Unsupported operation: readLine"); - } - - @Override - public String readUTF() throws IOException { - throw new IOException("Unsupported operation: readUTF"); - } - - public boolean isEOF() { - return eof; - } - - public boolean isEndOfStream() { - return endOfStream; - } - - public void reset() { - synchronized(dataQueue) { - endOfStream = false; - eof = false; - closed.set(false); - dataQueue.clear(); - currentScanIndex = 0; - currentScanData = null; - } - } -} diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java b/tajo-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java deleted file mode 100644 index 2fd4a99f52..0000000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/v2/StorageManagerV2.java +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.storage.AbstractStorageManager; -import org.apache.tajo.storage.NullScanner; -import org.apache.tajo.storage.Scanner; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.storage.fragment.Fragment; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.Queue; - -public final class StorageManagerV2 extends AbstractStorageManager { - private final Log LOG = LogFactory.getLog(StorageManagerV2.class); - - private Queue scanQueue = new LinkedList(); - - private Object scanQueueLock = new Object(); - - private Object scanDataLock = new Object(); - - private ScanScheduler scanScheduler; - - private StorgaeManagerContext context; - - public StorageManagerV2(TajoConf conf) throws IOException { - super(conf); - context = new StorgaeManagerContext(); - scanScheduler = new ScanScheduler(context); - scanScheduler.start(); - LOG.info("StorageManager v2 started..."); - } - - @Override - public Class getScannerClass(CatalogProtos.StoreType storeType) throws IOException { - Class scannerClass; - - String handlerName = storeType.name().toLowerCase(); - String handlerNameKey = handlerName + "_v2"; - - scannerClass = SCANNER_HANDLER_CACHE.get(handlerNameKey); - if (scannerClass == null) { - scannerClass = conf.getClass(String.format("tajo.storage.scanner-handler.v2.%s.class", - storeType.name().toLowerCase()), null, Scanner.class); - SCANNER_HANDLER_CACHE.put(handlerNameKey, scannerClass); - } - - return scannerClass; - } - - @Override - public Scanner getScanner(TableMeta meta, Schema schema, Fragment fragment, Schema target) throws IOException { - if (fragment instanceof FileFragment) { - FileFragment fileFragment = (FileFragment)fragment; - if (fileFragment.getEndKey() == 0) { - Scanner scanner = new NullScanner(conf, schema, meta, fileFragment); - scanner.setTarget(target.toArray()); - - return scanner; - } - } - - Scanner scanner; - - Class scannerClass = getScannerClass(meta.getStoreType()); - if (scannerClass == null) { - throw new IOException("Unknown Storage Type: " + meta.getStoreType()); - } - - scanner = newScannerInstance(scannerClass, conf, schema, meta, fragment); - if (scanner.isProjectable()) { - scanner.setTarget(target.toArray()); - } - - if(scanner instanceof FileScannerV2) { - ((FileScannerV2)scanner).setStorageManagerContext(context); - } - return scanner; - } - - public void requestFileScan(FileScannerV2 fileScanner) { - synchronized(scanQueueLock) { - scanQueue.offer(fileScanner); - - scanQueueLock.notifyAll(); - } - } - - public StorgaeManagerContext getContext() { - return context; - } - - public class StorgaeManagerContext { - public Object getScanQueueLock() { - return scanQueueLock; - } - - public Object getScanDataLock() { - return scanDataLock; - } - - public Queue getScanQueue() { - return scanQueue; - } - - public int getMaxReadBytesPerScheduleSlot() { - return conf.getIntVar(TajoConf.ConfVars.STORAGE_MANAGER_DISK_SCHEDULER_MAX_READ_BYTES_PER_SLOT); - } - - public void requestFileScan(FileScannerV2 fileScanner) { - StorageManagerV2.this.requestFileScan(fileScanner); - } - - public TajoConf getConf() { - return conf; - } - - public void incrementReadBytes(int diskId, long[] readBytes) { - scanScheduler.incrementReadBytes(diskId, readBytes); - } - } - - public void stop() { - if(scanScheduler != null) { - scanScheduler.stopScheduler(); - } - } -} diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java index cae03570f0..212f374e8c 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestCompressionStorages.java @@ -124,7 +124,7 @@ public void testSplitCompressionData() throws IOException { meta.putOption("compression.codec", BZip2Codec.class.getCanonicalName()); Path tablePath = new Path(testDir, "SplitCompression"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); appender.enableStats(); appender.init(); @@ -156,7 +156,7 @@ public void testSplitCompressionData() throws IOException { tablets[0] = new FileFragment("SplitCompression", tablePath, 0, randomNum); tablets[1] = new FileFragment("SplitCompression", tablePath, randomNum, (fileLen - randomNum)); - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema); + Scanner scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema); assertTrue(scanner.isSplittable()); scanner.init(); int tupleCnt = 0; @@ -166,7 +166,7 @@ public void testSplitCompressionData() throws IOException { } scanner.close(); - scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema); + scanner = StorageManager.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema); assertTrue(scanner.isSplittable()); scanner.init(); while ((tuple = scanner.next()) != null) { @@ -191,7 +191,7 @@ private void storageCompressionTest(StoreType storeType, Class 0; i--) { @@ -853,7 +853,7 @@ public void testFindNextKeyValueDescOrder() throws IOException { meta = CatalogUtil.newTableMeta(storeType); Path tablePath = new Path(testDir, "testFindNextKeyValueDescOrder_" + storeType); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; @@ -888,7 +888,7 @@ public void testFindNextKeyValueDescOrder() throws IOException { creater.setLoadNum(LOAD_NUM); creater.open(); - SeekableScanner scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema); + SeekableScanner scanner = StorageManager.getSeekableScanner(conf, meta, schema, tablet, schema); scanner.init(); Tuple keyTuple; @@ -916,7 +916,7 @@ public void testFindNextKeyValueDescOrder() throws IOException { assertEquals(keySchema, reader.getKeySchema()); assertEquals(comp, reader.getComparator()); - scanner = StorageManagerFactory.getSeekableScanner(conf, meta, schema, tablet, schema); + scanner = StorageManager.getSeekableScanner(conf, meta, schema, tablet, schema); scanner.init(); Tuple result; diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java b/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java index 53a2531a7d..78b16c3804 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/index/TestSingleCSVFileBSTIndex.java @@ -78,7 +78,7 @@ public void testFindValueInSingleCSV() throws IOException { Path tablePath = StorageUtil.concatPath(testDir, "testFindValueInSingleCSV", "table.csv"); fs.mkdirs(tablePath.getParent()); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; for (int i = 0; i < TUPLE_NUM; i++) { @@ -167,7 +167,7 @@ public void testFindNextKeyValueInSingleCSV() throws IOException { Path tablePath = StorageUtil.concatPath(testDir, "testFindNextKeyValueInSingleCSV", "table1.csv"); fs.mkdirs(tablePath.getParent()); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); + Appender appender = StorageManager.getStorageManager(conf).getAppender(meta, schema, tablePath); appender.init(); Tuple tuple; for(int i = 0 ; i < TUPLE_NUM; i ++ ) { diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java b/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java deleted file mode 100644 index 7e95b8b5b6..0000000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVCompression.java +++ /dev/null @@ -1,213 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.compress.*; -import org.apache.hadoop.util.NativeCodeLoader; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.util.CommonTestingUtil; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; - -import static org.junit.Assert.assertEquals; - -@RunWith(Parameterized.class) -public class TestCSVCompression { - private TajoConf conf; - private static String TEST_PATH = "target/test-data/v2/TestCSVCompression"; - - private CatalogProtos.StoreType storeType; - private Path testDir; - private FileSystem fs; - - public TestCSVCompression(CatalogProtos.StoreType type) throws IOException { - this.storeType = type; - conf = new TajoConf(); - conf.setBoolVar(TajoConf.ConfVars.STORAGE_MANAGER_VERSION_2, true); - - testDir = CommonTestingUtil.getTestDir(TEST_PATH); - fs = testDir.getFileSystem(conf); - } - - @Parameterized.Parameters - public static Collection generateParameters() { - return Arrays.asList(new Object[][]{ - {CatalogProtos.StoreType.CSV} - }); - } - - @Test - public void testDeflateCodecCompressionData() throws IOException { - storageCompressionTest(storeType, DeflateCodec.class); - } - - @Test - public void testGzipCodecCompressionData() throws IOException { - storageCompressionTest(storeType, GzipCodec.class); - } - - @Test - public void testSnappyCodecCompressionData() throws IOException { - if (SnappyCodec.isNativeCodeLoaded()) { - storageCompressionTest(storeType, SnappyCodec.class); - } - } - - @Test - public void testBzip2CodecCompressionData() throws IOException { - storageCompressionTest(storeType, BZip2Codec.class); - } - - @Test - public void testLz4CodecCompressionData() throws IOException { - if(NativeCodeLoader.isNativeCodeLoaded() && Lz4Codec.isNativeCodeLoaded()) - storageCompressionTest(storeType, Lz4Codec.class); - } - - // TODO - See https://issues.apache.org/jira/browse/HADOOP-9622 - //@Test - public void testSplitCompressionData() throws IOException { - - Schema schema = new Schema(); - schema.addColumn("id", TajoDataTypes.Type.INT4); - schema.addColumn("age", TajoDataTypes.Type.INT8); - - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV); - meta.putOption("compression.codec", BZip2Codec.class.getCanonicalName()); - - Path tablePath = new Path(testDir, "SplitCompression"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); - appender.enableStats(); - - appender.init(); - - String extention = ""; - if (appender instanceof CSVFile.CSVAppender) { - extention = ((CSVFile.CSVAppender) appender).getExtension(); - } - - int tupleNum = 100000; - VTuple vTuple; - - for (int i = 0; i < tupleNum; i++) { - vTuple = new VTuple(2); - vTuple.put(0, DatumFactory.createInt4(i + 1)); - vTuple.put(1, DatumFactory.createInt8(25l)); - appender.addTuple(vTuple); - } - appender.close(); - - TableStats stat = appender.getStats(); - assertEquals(tupleNum, stat.getNumRows().longValue()); - tablePath = tablePath.suffix(extention); - - FileStatus status = fs.getFileStatus(tablePath); - long fileLen = status.getLen(); - long randomNum = (long) (Math.random() * fileLen) + 1; - - FileFragment[] tablets = new FileFragment[2]; - tablets[0] = new FileFragment("SplitCompression", tablePath, 0, randomNum); - tablets[1] = new FileFragment("SplitCompression", tablePath, randomNum, (fileLen - randomNum)); - - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema); - scanner.init(); - int tupleCnt = 0; - Tuple tuple; - while ((tuple = scanner.next()) != null) { - tupleCnt++; - } - scanner.close(); - - scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema); - scanner.init(); - while ((tuple = scanner.next()) != null) { - tupleCnt++; - } - - scanner.close(); - assertEquals(tupleNum, tupleCnt); - } - - private void storageCompressionTest(CatalogProtos.StoreType storeType, Class codec) - throws IOException { - Schema schema = new Schema(); - schema.addColumn("id", TajoDataTypes.Type.INT4); - schema.addColumn("age", TajoDataTypes.Type.INT8); - - TableMeta meta = CatalogUtil.newTableMeta(storeType); - meta.putOption("compression.codec", codec.getCanonicalName()); - - String fileName = "Compression_" + codec.getSimpleName(); - Path tablePath = new Path(testDir, fileName); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); - appender.enableStats(); - - appender.init(); - - String extension = ""; - if (appender instanceof CSVFile.CSVAppender) { - extension = ((CSVFile.CSVAppender) appender).getExtension(); - } - - int tupleNum = 10000; - VTuple vTuple; - - for (int i = 0; i < tupleNum; i++) { - vTuple = new VTuple(2); - vTuple.put(0, DatumFactory.createInt4(i + 1)); - vTuple.put(1, DatumFactory.createInt8(25l)); - appender.addTuple(vTuple); - } - appender.close(); - - TableStats stat = appender.getStats(); - assertEquals(tupleNum, stat.getNumRows().longValue()); - tablePath = tablePath.suffix(extension); - FileStatus status = fs.getFileStatus(tablePath); - long fileLen = status.getLen(); - FileFragment[] tablets = new FileFragment[1]; - tablets[0] = new FileFragment(fileName, tablePath, 0, fileLen); - - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema); - scanner.init(); - int tupleCnt = 0; - while (scanner.next() != null) { - tupleCnt++; - } - scanner.close(); - assertEquals(tupleCnt, tupleNum); - } -} diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java b/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java deleted file mode 100644 index c356548d42..0000000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestCSVScanner.java +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.util.CommonTestingUtil; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.io.IOException; - -import static org.apache.tajo.conf.TajoConf.ConfVars; -import static org.junit.Assert.assertEquals; - -public class TestCSVScanner { - private TajoConf conf; - private static String TEST_PATH = "target/test-data/v2/TestCSVScanner"; - AbstractStorageManager sm = null; - private Path testDir; - private FileSystem fs; - - @Before - public void setUp() throws Exception { - conf = new TajoConf(); - conf.setBoolVar(ConfVars.STORAGE_MANAGER_VERSION_2, true); - testDir = CommonTestingUtil.getTestDir(TEST_PATH); - fs = testDir.getFileSystem(conf); - sm = StorageManagerFactory.getStorageManager(conf, testDir); - } - - @After - public void tearDown() throws Exception { - } - - @Test - public final void testGetScannerAndAppender() throws IOException { - Schema schema = new Schema(); - schema.addColumn("id", TajoDataTypes.Type.INT4); - schema.addColumn("age", TajoDataTypes.Type.INT4); - schema.addColumn("name", TajoDataTypes.Type.TEXT); - - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV); - - Tuple[] tuples = new Tuple[4]; - for(int i=0; i < tuples.length; i++) { - tuples[i] = new VTuple(3); - tuples[i].put(new Datum[] { - DatumFactory.createInt4(i), - DatumFactory.createInt4(i + 32), - DatumFactory.createText("name" + i)}); - } - - Path path = StorageUtil.concatPath(testDir, "testGetScannerAndAppender", "table.csv"); - fs.mkdirs(path.getParent()); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, path); - appender.init(); - for(Tuple t : tuples) { - appender.addTuple(t); - } - appender.close(); - - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getFileScanner(meta, schema, path); - scanner.init(); - int i=0; - Tuple tuple = null; - while( (tuple = scanner.next()) != null) { - i++; - } - assertEquals(4,i); - } - - @Test - public final void testPartitionFile() throws IOException { - Schema schema = new Schema(); - schema.addColumn("key", TajoDataTypes.Type.TEXT); - schema.addColumn("age", TajoDataTypes.Type.INT4); - schema.addColumn("name", TajoDataTypes.Type.TEXT); - - TableMeta meta = CatalogUtil.newTableMeta(CatalogProtos.StoreType.CSV); - - - Path path = StorageUtil.concatPath(testDir, "testPartitionFile", "table.csv"); - fs.mkdirs(path.getParent()); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, path); - appender.init(); - - String keyValue = ""; - for(int i = 0; i < 100; i++) { - keyValue += "0123456789"; - } - keyValue = "key_" + keyValue + "_"; - - String nameValue = ""; - for(int i = 0; i < 100; i++) { - nameValue += "0123456789"; - } - nameValue = "name_" + nameValue + "_"; - - int numTuples = 100000; - for(int i = 0; i < numTuples; i++) { - Tuple tuple = new VTuple(3); - tuple.put(new Datum[] { - DatumFactory.createText(keyValue + i), - DatumFactory.createInt4(i + 32), - DatumFactory.createText(nameValue + i)}); - appender.addTuple(tuple); - } - appender.close(); - - long fileLength = fs.getLength(path); - long totalTupleCount = 0; - - int scanCount = 0; - Tuple startTuple = null; - Tuple lastTuple = null; - while(true) { - long startOffset = (64 * 1024 * 1024) * scanCount; - long length = Math.min(64 * 1024 * 1024, fileLength - startOffset); - - FileFragment fragment = new FileFragment("Test", path, startOffset, length, null); - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, schema); - scanner.init(); - Tuple tuple = null; - while( (tuple = scanner.next()) != null) { - if(startTuple == null) { - startTuple = tuple; - } - lastTuple = tuple; - totalTupleCount++; - } - scanCount++; - if(length < 64 * 1024 * 1024) { - break; - } - } - assertEquals(numTuples, totalTupleCount); - assertEquals(keyValue + 0, startTuple.get(0).toString()); - assertEquals(keyValue + (numTuples - 1), lastTuple.get(0).toString()); - } -} diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java deleted file mode 100644 index 357dadb04f..0000000000 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/v2/TestStorages.java +++ /dev/null @@ -1,292 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage.v2; - -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.storage.*; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.storage.rcfile.RCFile; -import org.apache.tajo.util.CommonTestingUtil; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; - -import static org.apache.tajo.conf.TajoConf.ConfVars; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -@RunWith(Parameterized.class) -public class TestStorages { - private TajoConf conf; - private static String TEST_PATH = "target/test-data/v2/TestStorages"; - - private static String TEST_PROJECTION_AVRO_SCHEMA = - "{\n" + - " \"type\": \"record\",\n" + - " \"namespace\": \"org.apache.tajo\",\n" + - " \"name\": \"testProjection\",\n" + - " \"fields\": [\n" + - " { \"name\": \"id\", \"type\": \"int\" },\n" + - " { \"name\": \"age\", \"type\": \"long\" },\n" + - " { \"name\": \"score\", \"type\": \"float\" }\n" + - " ]\n" + - "}\n"; - - private static String TEST_VARIOUS_TYPES_AVRO_SCHEMA = - "{\n" + - " \"type\": \"record\",\n" + - " \"namespace\": \"org.apache.tajo\",\n" + - " \"name\": \"testVariousTypes\",\n" + - " \"fields\": [\n" + - " { \"name\": \"col1\", \"type\": \"boolean\" },\n" + - " { \"name\": \"col2\", \"type\": \"int\" },\n" + - " { \"name\": \"col3\", \"type\": \"string\" },\n" + - " { \"name\": \"col4\", \"type\": \"int\" },\n" + - " { \"name\": \"col5\", \"type\": \"int\" },\n" + - " { \"name\": \"col6\", \"type\": \"long\" },\n" + - " { \"name\": \"col7\", \"type\": \"float\" },\n" + - " { \"name\": \"col8\", \"type\": \"double\" },\n" + - " { \"name\": \"col9\", \"type\": \"string\" },\n" + - " { \"name\": \"col10\", \"type\": \"bytes\" },\n" + - " { \"name\": \"col11\", \"type\": \"bytes\" },\n" + - " { \"name\": \"col12\", \"type\": \"null\" }\n" + - " ]\n" + - "}\n"; - - private StoreType storeType; - private boolean splitable; - private boolean statsable; - private Path testDir; - private FileSystem fs; - - public TestStorages(StoreType type, boolean splitable, boolean statsable) throws IOException { - this.storeType = type; - this.splitable = splitable; - this.statsable = statsable; - - conf = new TajoConf(); - conf.setBoolVar(ConfVars.STORAGE_MANAGER_VERSION_2, true); - - if (storeType == StoreType.RCFILE) { - conf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, 100); - } - - - testDir = CommonTestingUtil.getTestDir(TEST_PATH); - fs = testDir.getFileSystem(conf); - } - - @Parameterized.Parameters - public static Collection generateParameters() { - return Arrays.asList(new Object[][]{ - {StoreType.CSV, true, true}, - {StoreType.RCFILE, true, true}, - {StoreType.TREVNI, false, true}, - {StoreType.PARQUET, false, false}, - {StoreType.AVRO, false, false}, - {StoreType.RAW, false, false}, - }); - } - - @Test - public void testSplitable() throws IOException { - if (splitable) { - Schema schema = new Schema(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - - TableMeta meta = CatalogUtil.newTableMeta(storeType); - meta.setOptions(StorageUtil.newPhysicalProperties(storeType)); - Path tablePath = new Path(testDir, "Splitable.data"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); - appender.enableStats(); - appender.init(); - int tupleNum = 10000; - VTuple vTuple; - - for(int i = 0; i < tupleNum; i++) { - vTuple = new VTuple(2); - vTuple.put(0, DatumFactory.createInt4(i + 1)); - vTuple.put(1, DatumFactory.createInt8(25l)); - appender.addTuple(vTuple); - } - appender.close(); - TableStats stat = appender.getStats(); - assertEquals(tupleNum, stat.getNumRows().longValue()); - - FileStatus status = fs.getFileStatus(tablePath); - long fileLen = status.getLen(); - long randomNum = (long) (Math.random() * fileLen) + 1; - - FileFragment[] tablets = new FileFragment[2]; - tablets[0] = new FileFragment("Splitable", tablePath, 0, randomNum); - tablets[1] = new FileFragment("Splitable", tablePath, randomNum, (fileLen - randomNum)); - - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[0], schema); - scanner.init(); - int tupleCnt = 0; - while (scanner.next() != null) { - tupleCnt++; - } - scanner.close(); - - scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, tablets[1], schema); - scanner.init(); - while (scanner.next() != null) { - tupleCnt++; - } - scanner.close(); - - assertEquals(tupleNum, tupleCnt); - } - } - - @Test - public void testProjection() throws IOException { - Schema schema = new Schema(); - schema.addColumn("id", Type.INT4); - schema.addColumn("age", Type.INT8); - schema.addColumn("score", Type.FLOAT4); - - TableMeta meta = CatalogUtil.newTableMeta(storeType); - meta.setOptions(StorageUtil.newPhysicalProperties(storeType)); - if (storeType == StoreType.AVRO) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, - TEST_PROJECTION_AVRO_SCHEMA); - } - - Path tablePath = new Path(testDir, "testProjection.data"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); - appender.init(); - int tupleNum = 10000; - VTuple vTuple; - - for(int i = 0; i < tupleNum; i++) { - vTuple = new VTuple(3); - vTuple.put(0, DatumFactory.createInt4(i + 1)); - vTuple.put(1, DatumFactory.createInt8(i + 2)); - vTuple.put(2, DatumFactory.createFloat4(i + 3)); - appender.addTuple(vTuple); - } - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - FileFragment fragment = new FileFragment("testReadAndWrite", tablePath, 0, status.getLen()); - - Schema target = new Schema(); - target.addColumn("age", Type.INT8); - target.addColumn("score", Type.FLOAT4); - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment, target); - scanner.init(); - int tupleCnt = 0; - Tuple tuple; - while ((tuple = scanner.next()) != null) { - if (storeType == StoreType.RCFILE - || storeType == StoreType.TREVNI - || storeType == StoreType.CSV - || storeType == StoreType.PARQUET - || storeType == StoreType.AVRO) { - assertTrue(tuple.get(0) == null || tuple.get(0) instanceof NullDatum); - } - assertTrue(tupleCnt + 2 == tuple.get(1).asInt8()); - assertTrue(tupleCnt + 3 == tuple.get(2).asFloat4()); - tupleCnt++; - } - scanner.close(); - - assertEquals(tupleNum, tupleCnt); - } - - @Test - public void testVariousTypes() throws IOException { - Schema schema = new Schema(); - schema.addColumn("col1", Type.BOOLEAN); - schema.addColumn("col2", Type.BIT); - schema.addColumn("col3", Type.CHAR, 7); - schema.addColumn("col4", Type.INT2); - schema.addColumn("col5", Type.INT4); - schema.addColumn("col6", Type.INT8); - schema.addColumn("col7", Type.FLOAT4); - schema.addColumn("col8", Type.FLOAT8); - schema.addColumn("col9", Type.TEXT); - schema.addColumn("col10", Type.BLOB); - schema.addColumn("col11", Type.INET4); - schema.addColumn("col12", Type.NULL_TYPE); - - KeyValueSet options = new KeyValueSet(); - TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(StorageUtil.newPhysicalProperties(storeType)); - if (storeType == StoreType.AVRO) { - meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, - TEST_VARIOUS_TYPES_AVRO_SCHEMA); - } - - Path tablePath = new Path(testDir, "testVariousTypes.data"); - Appender appender = StorageManagerFactory.getStorageManager(conf).getAppender(meta, schema, tablePath); - appender.init(); - - Tuple tuple = new VTuple(12); - tuple.put(new Datum[] { - DatumFactory.createBool(true), - DatumFactory.createBit((byte) 0x99), - DatumFactory.createChar("hyunsik"), - DatumFactory.createInt2((short) 17), - DatumFactory.createInt4(59), - DatumFactory.createInt8(23l), - DatumFactory.createFloat4(77.9f), - DatumFactory.createFloat8(271.9f), - DatumFactory.createText("hyunsik"), - DatumFactory.createBlob("hyunsik".getBytes()), - DatumFactory.createInet4("192.168.0.1"), - NullDatum.get() - }); - appender.addTuple(tuple); - appender.flush(); - appender.close(); - - FileStatus status = fs.getFileStatus(tablePath); - FileFragment fragment = new FileFragment("table", tablePath, 0, status.getLen()); - Scanner scanner = StorageManagerFactory.getStorageManager(conf).getScanner(meta, schema, fragment); - scanner.init(); - - Tuple retrieved; - while ((retrieved=scanner.next()) != null) { - for (int i = 0; i < tuple.size(); i++) { - assertEquals(tuple.get(i), retrieved.get(i)); - } - } - } -} diff --git a/tajo-storage/src/test/resources/storage-default.xml b/tajo-storage/src/test/resources/storage-default.xml index a1cc2ff157..6bfc902411 100644 --- a/tajo-storage/src/test/resources/storage-default.xml +++ b/tajo-storage/src/test/resources/storage-default.xml @@ -20,23 +20,6 @@ --> - - tajo.storage.manager.v2 - false - - - - tajo.storage.manager.maxReadBytes - 8388608 - - - - - tajo.storage.manager.concurrency.perDisk - 1 - - - fs.s3.impl org.apache.tajo.storage.s3.SmallBlockS3FileSystem @@ -88,81 +71,41 @@ org.apache.tajo.storage.CSVFile$CSVScanner - - tajo.storage.scanner-handler.v2.csv.class - org.apache.tajo.storage.v2.CSVFileScanner - - tajo.storage.scanner-handler.raw.class org.apache.tajo.storage.RawFile$RawFileScanner - - tajo.storage.scanner-handler.v2.raw.class - org.apache.tajo.storage.RawFile$RawFileScanner - - tajo.storage.scanner-handler.rcfile.class org.apache.tajo.storage.rcfile.RCFile$RCFileScanner - - tajo.storage.scanner-handler.v2.rcfile.class - org.apache.tajo.storage.v2.RCFileScanner - - tajo.storage.scanner-handler.rowfile.class org.apache.tajo.storage.RowFile$RowFileScanner - - tajo.storage.scanner-handler.v2.rowfile.class - org.apache.tajo.storage.RowFile$RowFileScanner - - tajo.storage.scanner-handler.trevni.class org.apache.tajo.storage.trevni.TrevniScanner - - tajo.storage.scanner-handler.v2.trevni.class - org.apache.tajo.storage.trevni.TrevniScanner - - tajo.storage.scanner-handler.parquet.class org.apache.tajo.storage.parquet.ParquetScanner - - tajo.storage.scanner-handler.v2.parquet.class - org.apache.tajo.storage.parquet.ParquetScanner - - tajo.storage.scanner-handler.sequencefile.class org.apache.tajo.storage.sequencefile.SequenceFileScanner - - tajo.storage.scanner-handler.v2.sequencefile.class - org.apache.tajo.storage.sequencefile.SequenceFileScanner - - tajo.storage.scanner-handler.avro.class org.apache.tajo.storage.avro.AvroScanner - - tajo.storage.scanner-handler.v2.avro.class - org.apache.tajo.storage.avro.AvroScanner - - tajo.storage.appender-handler From b143f991242b79fa8479148cd79fad7d4f8f2146 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sat, 25 Oct 2014 10:54:12 -0700 Subject: [PATCH 08/15] TAJO-1125: Separate logical plan and optimizer into a maven module. Closes #210 --- CHANGES | 3 + pom.xml | 1 + .../apache/tajo/catalog/CatalogService.java | 0 .../org/apache/tajo/catalog/CatalogUtil.java | 29 ++ .../apache/tajo/storage/StorageConstants.java | 21 +- .../java/org/apache/tajo/storage/Tuple.java | 0 .../java/org/apache/tajo/storage/VTuple.java | 0 .../tajo/util}/graph/DirectedGraph.java | 2 +- .../tajo/util}/graph/DirectedGraphCursor.java | 2 +- .../util}/graph/DirectedGraphVisitor.java | 2 +- .../org/apache/tajo/util}/graph/Graph.java | 2 +- .../tajo/util}/graph/SimpleDirectedGraph.java | 2 +- .../util}/graph/SimpleUndirectedGraph.java | 2 +- .../tajo/util}/graph/UndirectedGraph.java | 2 +- .../util/graph}/TestSimpleDirectedGraph.java | 6 +- .../graph}/TestSimpleUndirectedGraph.java | 6 +- tajo-core/pom.xml | 6 +- .../tajo/engine/codegen/CaseWhenEmitter.java | 8 +- .../codegen/CaseWhenSwitchGenerator.java | 2 +- .../tajo/engine/codegen/CompilationError.java | 2 +- .../tajo/engine/codegen/EvalCodeEmitter.java | 2 +- .../engine/codegen/EvalCodeGenContext.java | 2 +- .../engine/codegen/EvalCodeGenerator.java | 6 +- .../engine/codegen/ExecutorPreCompiler.java | 12 +- .../codegen/LegacyFunctionBindingEmitter.java | 4 +- .../codegen/ScalarFunctionBindingEmitter.java | 4 +- .../engine/codegen/TajoGeneratorAdapter.java | 8 +- .../engine/codegen/VariablesPreBuilder.java | 2 +- .../engine/function/builtin/AvgDouble.java | 4 +- .../engine/function/builtin/AvgFloat.java | 2 +- .../tajo/engine/function/builtin/AvgInt.java | 2 +- .../tajo/engine/function/builtin/AvgLong.java | 4 +- .../engine/function/builtin/Coalesce.java | 2 +- .../engine/function/builtin/CountRows.java | 4 +- .../engine/function/builtin/CountValue.java | 2 +- .../function/builtin/CountValueDistinct.java | 2 +- .../tajo/engine/function/builtin/Date.java | 2 +- .../engine/function/builtin/MaxDouble.java | 4 +- .../engine/function/builtin/MaxFloat.java | 4 +- .../tajo/engine/function/builtin/MaxInt.java | 4 +- .../tajo/engine/function/builtin/MaxLong.java | 4 +- .../engine/function/builtin/MaxString.java | 4 +- .../engine/function/builtin/MinDouble.java | 4 +- .../engine/function/builtin/MinFloat.java | 4 +- .../tajo/engine/function/builtin/MinInt.java | 4 +- .../tajo/engine/function/builtin/MinLong.java | 4 +- .../engine/function/builtin/MinString.java | 4 +- .../engine/function/builtin/RandomInt.java | 2 +- .../tajo/engine/function/builtin/Sleep.java | 2 +- .../engine/function/builtin/SumDouble.java | 4 +- .../function/builtin/SumDoubleDistinct.java | 4 +- .../engine/function/builtin/SumFloat.java | 4 +- .../function/builtin/SumFloatDistinct.java | 4 +- .../tajo/engine/function/builtin/SumInt.java | 4 +- .../function/builtin/SumIntDistinct.java | 4 +- .../tajo/engine/function/builtin/SumLong.java | 4 +- .../function/builtin/SumLongDistinct.java | 4 +- .../engine/function/datetime/AddDays.java | 2 +- .../engine/function/datetime/AddMonths.java | 2 +- .../engine/function/datetime/CurrentDate.java | 2 +- .../engine/function/datetime/CurrentTime.java | 2 +- .../function/datetime/DatePartFromDate.java | 2 +- .../function/datetime/DatePartFromTime.java | 2 +- .../datetime/DatePartFromTimestamp.java | 2 +- .../DateTimePartFromUnixTimestamp.java | 2 +- .../function/datetime/NowTimestamp.java | 2 +- .../function/datetime/ToCharTimestamp.java | 4 +- .../tajo/engine/function/datetime/ToDate.java | 2 +- .../function/datetime/ToTimestampInt.java | 2 +- .../function/datetime/ToTimestampText.java | 2 +- .../function/geoip/GeoIPCountryInet4.java | 2 +- .../function/geoip/GeoIPCountryText.java | 2 +- .../function/geoip/GeoIPInCountryInet4.java | 2 +- .../function/geoip/GeoIPInCountryText.java | 2 +- .../tajo/engine/function/math/AbsDouble.java | 2 +- .../tajo/engine/function/math/AbsFloat.java | 2 +- .../tajo/engine/function/math/AbsInt.java | 2 +- .../tajo/engine/function/math/AbsLong.java | 2 +- .../tajo/engine/function/math/Acos.java | 2 +- .../tajo/engine/function/math/Asin.java | 2 +- .../tajo/engine/function/math/Atan.java | 2 +- .../tajo/engine/function/math/Atan2.java | 2 +- .../tajo/engine/function/math/Cbrt.java | 2 +- .../tajo/engine/function/math/Ceil.java | 2 +- .../apache/tajo/engine/function/math/Cos.java | 2 +- .../tajo/engine/function/math/Degrees.java | 2 +- .../apache/tajo/engine/function/math/Div.java | 2 +- .../apache/tajo/engine/function/math/Exp.java | 2 +- .../tajo/engine/function/math/Floor.java | 2 +- .../apache/tajo/engine/function/math/Mod.java | 2 +- .../apache/tajo/engine/function/math/Pi.java | 2 +- .../apache/tajo/engine/function/math/Pow.java | 3 +- .../tajo/engine/function/math/Radians.java | 2 +- .../tajo/engine/function/math/Round.java | 2 +- .../engine/function/math/RoundFloat8.java | 2 +- .../tajo/engine/function/math/Sign.java | 2 +- .../apache/tajo/engine/function/math/Sin.java | 2 +- .../tajo/engine/function/math/Sqrt.java | 2 +- .../apache/tajo/engine/function/math/Tan.java | 2 +- .../tajo/engine/function/string/Ascii.java | 2 +- .../tajo/engine/function/string/BTrim.java | 4 +- .../engine/function/string/BitLength.java | 2 +- .../engine/function/string/CharLength.java | 2 +- .../tajo/engine/function/string/Chr.java | 2 +- .../tajo/engine/function/string/Concat.java | 2 +- .../engine/function/string/Concat_ws.java | 2 +- .../tajo/engine/function/string/Decode.java | 2 +- .../tajo/engine/function/string/Digest.java | 2 +- .../tajo/engine/function/string/Encode.java | 2 +- .../engine/function/string/FindInSet.java | 2 +- .../tajo/engine/function/string/InitCap.java | 2 +- .../tajo/engine/function/string/LTrim.java | 4 +- .../tajo/engine/function/string/Left.java | 2 +- .../tajo/engine/function/string/Length.java | 2 +- .../tajo/engine/function/string/Locate.java | 2 +- .../tajo/engine/function/string/Lower.java | 2 +- .../tajo/engine/function/string/Lpad.java | 4 +- .../tajo/engine/function/string/Md5.java | 2 +- .../engine/function/string/OctetLength.java | 2 +- .../engine/function/string/QuoteIdent.java | 2 +- .../tajo/engine/function/string/RTrim.java | 4 +- .../engine/function/string/RegexpReplace.java | 11 +- .../tajo/engine/function/string/Repeat.java | 2 +- .../tajo/engine/function/string/Reverse.java | 2 +- .../tajo/engine/function/string/Right.java | 2 +- .../tajo/engine/function/string/Rpad.java | 4 +- .../engine/function/string/SplitPart.java | 2 +- .../tajo/engine/function/string/StrPos.java | 2 +- .../tajo/engine/function/string/StrPosb.java | 2 +- .../tajo/engine/function/string/Substr.java | 2 +- .../tajo/engine/function/string/ToBin.java | 2 +- .../tajo/engine/function/string/ToHex.java | 2 +- .../tajo/engine/function/string/Upper.java | 2 +- .../tajo/engine/function/window/Rank.java | 4 +- .../engine/function/window/RowNumber.java | 4 +- .../tajo/engine/json/CoreGsonHelper.java | 12 +- .../tajo/engine/parser/SQLSyntaxError.java | 2 +- .../BroadcastJoinMarkCandidateVisitor.java | 11 +- .../planner/BroadcastJoinPlanVisitor.java | 11 +- .../tajo/engine/planner/PhysicalPlanner.java | 2 +- .../engine/planner/PhysicalPlannerImpl.java | 5 +- .../apache/tajo/engine/planner/Projector.java | 6 +- .../engine/planner/global/DataChannel.java | 4 +- .../engine/planner/global/ExecutionBlock.java | 2 +- .../engine/planner/global/GlobalPlanner.java | 22 +- .../engine/planner/global/MasterPlan.java | 12 +- .../builder/DistinctGroupbyBuilder.java | 27 +- .../planner/physical/AggregationExec.java | 4 +- .../engine/planner/physical/BNLJoinExec.java | 6 +- .../planner/physical/BSTIndexScanExec.java | 4 +- .../physical/ColPartitionStoreExec.java | 8 +- .../DistinctGroupbyFirstAggregationExec.java | 8 +- .../DistinctGroupbyHashAggregationExec.java | 8 +- .../DistinctGroupbySecondAggregationExec.java | 32 +- .../DistinctGroupbySortAggregationExec.java | 4 +- .../DistinctGroupbyThirdAggregationExec.java | 11 +- .../engine/planner/physical/EvalExprExec.java | 4 +- .../planner/physical/ExternalSortExec.java | 2 +- .../planner/physical/HashAggregateExec.java | 4 +- .../HashBasedColPartitionStoreExec.java | 2 +- .../physical/HashFullOuterJoinExec.java | 8 +- .../engine/planner/physical/HashJoinExec.java | 8 +- .../physical/HashLeftAntiJoinExec.java | 2 +- .../physical/HashLeftOuterJoinExec.java | 12 +- .../physical/HashLeftSemiJoinExec.java | 2 +- .../physical/HashShuffleFileWriteExec.java | 2 +- .../engine/planner/physical/HavingExec.java | 4 +- .../engine/planner/physical/LimitExec.java | 2 +- .../engine/planner/physical/MemSortExec.java | 2 +- .../physical/MergeFullOuterJoinExec.java | 7 +- .../planner/physical/MergeJoinExec.java | 7 +- .../engine/planner/physical/NLJoinExec.java | 4 +- .../planner/physical/NLLeftOuterJoinExec.java | 4 +- .../physical/PartitionMergeScanExec.java | 4 +- .../planner/physical/PhysicalPlanUtil.java | 123 +++++++- .../planner/physical/ProjectionExec.java | 2 +- .../physical/RangeShuffleFileWriteExec.java | 2 +- .../physical/RightOuterMergeJoinExec.java | 7 +- .../planner/physical/SelectionExec.java | 4 +- .../engine/planner/physical/SeqScanExec.java | 19 +- .../planner/physical/SortAggregateExec.java | 4 +- .../SortBasedColPartitionStoreExec.java | 2 +- .../planner/physical/StoreTableExec.java | 4 +- .../engine/planner/physical/UnionExec.java | 2 +- .../planner/physical/WindowAggExec.java | 8 +- .../tajo/engine/query/QueryContext.java | 16 +- .../apache/tajo/engine/utils/TupleUtil.java | 66 +--- .../utils/test/ErrorInjectionRewriter.java | 6 +- .../org/apache/tajo/master/GlobalEngine.java | 24 +- .../master/NonForwardQueryResultScanner.java | 7 +- .../apache/tajo/master/querymaster/Query.java | 6 +- .../master/querymaster/QueryInProgress.java | 2 +- .../master/querymaster/QueryJobManager.java | 2 +- .../master/querymaster/QueryMasterTask.java | 14 +- .../tajo/master/querymaster/QueryUnit.java | 2 +- .../master/querymaster/Repartitioner.java | 20 +- .../tajo/master/querymaster/SubQuery.java | 7 +- .../java/org/apache/tajo/util/IndexUtil.java | 8 +- .../worker/ExecutionBlockSharedResource.java | 6 +- .../org/apache/tajo/worker/FetchImpl.java | 15 +- .../apache/tajo/worker/TajoQueryEngine.java | 2 +- .../java/org/apache/tajo/worker/Task.java | 5 +- .../tajo/worker/TaskAttemptContext.java | 2 +- .../src/main/proto/TajoWorkerProtocol.proto | 14 +- .../resources/webapps/worker/querytasks.jsp | 2 +- .../org/apache/tajo/QueryTestCaseBase.java | 5 +- .../apache/tajo/engine/eval/ExprTestBase.java | 14 +- .../apache/tajo/engine/eval/TestEvalTree.java | 1 + .../tajo/engine/eval/TestEvalTreeUtil.java | 24 +- .../tajo/engine/function/TestAggFunction.java | 1 + .../engine/planner/TestLogicalOptimizer.java | 24 +- .../tajo/engine/planner/TestLogicalPlan.java | 15 +- .../engine/planner/TestLogicalPlanner.java | 32 +- .../tajo/engine/planner/TestPlannerUtil.java | 24 +- .../engine/planner/TestQueryValidation.java | 1 + .../planner/TestUniformRangePartition.java | 1 + .../planner/global/TestBroadcastJoinPlan.java | 11 +- .../engine/planner/global/TestMasterPlan.java | 6 +- .../planner/physical/TestBNLJoinExec.java | 9 +- .../planner/physical/TestBSTIndexExec.java | 10 +- .../physical/TestExternalSortExec.java | 5 +- .../physical/TestFullOuterHashJoinExec.java | 9 +- .../physical/TestFullOuterMergeJoinExec.java | 9 +- .../physical/TestHashAntiJoinExec.java | 6 +- .../planner/physical/TestHashJoinExec.java | 9 +- .../physical/TestHashSemiJoinExec.java | 6 +- .../physical/TestLeftOuterHashJoinExec.java | 9 +- .../physical/TestLeftOuterNLJoinExec.java | 6 +- .../planner/physical/TestMergeJoinExec.java | 10 +- .../planner/physical/TestNLJoinExec.java | 6 +- .../planner/physical/TestPhysicalPlanner.java | 15 +- .../TestProgressExternalSortExec.java | 6 +- .../physical/TestRightOuterHashJoinExec.java | 6 +- .../physical/TestRightOuterMergeJoinExec.java | 9 +- .../engine/planner/physical/TestSortExec.java | 4 +- .../tajo/engine/query/TestJoinBroadcast.java | 2 +- .../engine/query/TestTablePartitions.java | 14 +- .../tajo/engine/util/TestTupleUtil.java | 24 +- .../tajo/master/TestExecutionBlockCursor.java | 6 +- .../apache/tajo/master/TestGlobalPlanner.java | 10 +- .../apache/tajo/master/TestRepartitioner.java | 13 +- .../master/querymaster/TestKillQuery.java | 6 +- .../worker/TestRangeRetrieverHandler.java | 6 +- tajo-plan/pom.xml | 288 ++++++++++++++++++ .../org/apache/tajo/plan}/ExprAnnotator.java | 19 +- .../org/apache/tajo/plan}/ExprNormalizer.java | 9 +- .../plan}/IllegalQueryStatusException.java | 2 +- .../tajo/plan}/InvalidQueryException.java | 2 +- .../apache/tajo/plan}/LogicalOptimizer.java | 33 +- .../org/apache/tajo/plan}/LogicalPlan.java | 25 +- .../tajo/plan}/LogicalPlanPreprocessor.java | 34 ++- .../org/apache/tajo/plan}/LogicalPlanner.java | 58 ++-- .../apache/tajo/plan}/NamedExprsManager.java | 8 +- .../org/apache/tajo/plan}/PlanString.java | 4 +- .../apache/tajo/plan}/PlanningException.java | 2 +- .../java/org/apache/tajo/plan}/Target.java | 13 +- .../apache/tajo/plan}/TypeDeterminant.java | 5 +- .../tajo/plan/algebra}/AlgebraVisitor.java | 3 +- .../algebra}/AmbiguousFieldException.java | 4 +- .../plan/algebra}/BaseAlgebraVisitor.java | 3 +- .../tajo/plan/annotator}/Prioritized.java | 2 +- .../expr}/AggregationFunctionCallEval.java | 6 +- .../tajo/plan/expr}/AlgebraicException.java | 2 +- .../apache/tajo/plan/expr}/AlgebraicUtil.java | 2 +- .../tajo/plan/expr}/BasicEvalNodeVisitor.java | 2 +- .../tajo/plan/expr}/BetweenPredicateEval.java | 2 +- .../apache/tajo/plan/expr}/BinaryEval.java | 2 +- .../apache/tajo/plan/expr}/CaseWhenEval.java | 6 +- .../org/apache/tajo/plan/expr}/CastEval.java | 2 +- .../org/apache/tajo/plan/expr}/ConstEval.java | 2 +- .../org/apache/tajo/plan/expr}/EvalNode.java | 6 +- .../tajo/plan/expr}/EvalNodeVisitor.java | 2 +- .../tajo/plan/expr}/EvalNodeVisitor2.java | 2 +- .../tajo/plan/expr}/EvalTreeFactory.java | 2 +- .../apache/tajo/plan/expr}/EvalTreeUtil.java | 10 +- .../org/apache/tajo/plan/expr}/EvalType.java | 2 +- .../org/apache/tajo/plan/expr}/FieldEval.java | 2 +- .../apache/tajo/plan/expr}/FunctionEval.java | 2 +- .../tajo/plan/expr}/GeneralFunctionEval.java | 4 +- .../org/apache/tajo/plan/expr}/InEval.java | 2 +- .../tajo/plan/expr}/InvalidEvalException.java | 2 +- .../apache/tajo/plan/expr}/IsNullEval.java | 2 +- .../tajo/plan/expr}/LikePredicateEval.java | 2 +- .../org/apache/tajo/plan/expr}/NotEval.java | 2 +- .../tajo/plan/expr}/PartialBinaryExpr.java | 2 +- .../plan/expr}/PatternMatchPredicateEval.java | 2 +- .../tajo/plan/expr}/RegexPredicateEval.java | 2 +- .../tajo/plan/expr}/RowConstantEval.java | 2 +- .../apache/tajo/plan/expr}/SignedEval.java | 2 +- .../plan/expr}/SimilarToPredicateEval.java | 2 +- .../plan/expr}/SimpleEvalNodeVisitor.java | 2 +- .../org/apache/tajo/plan/expr}/UnaryEval.java | 2 +- .../tajo/plan/expr}/WindowFunctionEval.java | 8 +- .../EvalTreeOptimizationRule.java | 7 +- .../plan/exprrewrite}/EvalTreeOptimizer.java | 7 +- .../exprrewrite}/rules/ConstantFolding.java | 10 +- .../rules/ConstantPropagation.java | 10 +- .../tajo/plan}/function/AggFunction.java | 2 +- .../tajo/plan}/function/FunctionContext.java | 2 +- .../tajo/plan}/function/GeneralFunction.java | 6 +- .../tajo/plan}/function/WindowAggFunc.java | 2 +- .../org/apache/tajo/plan/joinorder}/Edge.java | 4 +- .../tajo/plan/joinorder}/FoundJoinOrder.java | 4 +- .../GreedyHeuristicJoinOrderAlgorithm.java | 19 +- .../apache/tajo/plan/joinorder}/JoinEdge.java | 6 +- .../tajo/plan/joinorder}/JoinGraph.java | 24 +- .../plan/joinorder}/JoinOrderAlgorithm.java | 10 +- .../tajo/plan}/logical/AlterTableNode.java | 4 +- .../plan/logical}/AlterTablespaceNode.java | 9 +- .../apache/tajo/plan}/logical/BinaryNode.java | 2 +- .../plan}/logical/CreateDatabaseNode.java | 4 +- .../tajo/plan}/logical/CreateTableNode.java | 6 +- .../plan}/logical/DistinctGroupbyNode.java | 14 +- .../tajo/plan}/logical/DropDatabaseNode.java | 4 +- .../tajo/plan}/logical/DropTableNode.java | 4 +- .../tajo/plan}/logical/EvalExprNode.java | 8 +- .../apache/tajo/plan}/logical/ExceptNode.java | 4 +- .../tajo/plan/logical}/GroupElement.java | 6 +- .../tajo/plan}/logical/GroupbyNode.java | 12 +- .../apache/tajo/plan}/logical/HavingNode.java | 6 +- .../tajo/plan}/logical/IndexScanNode.java | 6 +- .../apache/tajo/plan}/logical/InsertNode.java | 4 +- .../tajo/plan}/logical/IntersectNode.java | 4 +- .../apache/tajo/plan}/logical/JoinNode.java | 12 +- .../apache/tajo/plan}/logical/LimitNode.java | 4 +- .../tajo/plan}/logical/LogicalNode.java | 10 +- .../plan}/logical/LogicalNodeVisitor.java | 2 +- .../tajo/plan}/logical/LogicalRootNode.java | 4 +- .../plan/logical}/NoSuchColumnException.java | 15 +- .../apache/tajo/plan}/logical/NodeType.java | 4 +- .../logical/PartitionedTableScanNode.java | 8 +- .../plan}/logical/PersistentStoreNode.java | 4 +- .../tajo/plan}/logical/Projectable.java | 12 +- .../tajo/plan}/logical/ProjectionNode.java | 8 +- .../tajo/plan}/logical/RelationNode.java | 2 +- .../apache/tajo/plan}/logical/ScanNode.java | 12 +- .../tajo/plan}/logical/SelectableNode.java | 4 +- .../tajo/plan}/logical/SelectionNode.java | 6 +- .../plan}/logical/ShuffleFileWriteNode.java | 11 +- .../apache/tajo/plan}/logical/SortNode.java | 4 +- .../tajo/plan}/logical/StoreTableNode.java | 4 +- .../tajo/plan}/logical/TableSubQueryNode.java | 10 +- .../tajo/plan}/logical/TruncateTableNode.java | 4 +- .../apache/tajo/plan}/logical/UnaryNode.java | 2 +- .../apache/tajo/plan}/logical/UnionNode.java | 4 +- .../tajo/plan}/logical/WindowAggNode.java | 13 +- .../apache/tajo/plan}/logical/WindowSpec.java | 4 +- .../tajo/plan}/nameresolver/NameResolver.java | 14 +- .../plan}/nameresolver/NameResolvingMode.java | 2 +- .../plan}/nameresolver/ResolverByLegacy.java | 14 +- .../plan}/nameresolver/ResolverByRels.java | 8 +- .../ResolverByRelsAndSubExprs.java | 8 +- .../ResolverBySubExprsAndRels.java | 8 +- .../rewrite/BasicQueryRewriteEngine.java | 6 +- .../plan}/rewrite/QueryRewriteEngine.java | 6 +- .../tajo/plan}/rewrite/RewriteRule.java | 6 +- .../rewrite/rules}/FilterPushDownRule.java | 14 +- .../rules}/PartitionedTableRewriter.java | 89 +++++- .../rules}/ProjectionPushDownRule.java | 17 +- .../tajo/plan/serder}/EvalNodeAdapter.java | 6 +- .../serder}/EvalTreeProtoDeserializer.java | 13 +- .../plan/serder}/EvalTreeProtoSerializer.java | 9 +- .../tajo/plan/serder}/LogicalNodeAdapter.java | 6 +- .../tajo/plan/serder/PlanGsonHelper.java | 90 ++++++ .../apache/tajo/plan/util}/ExprFinder.java | 4 +- .../apache/tajo/plan/util}/PlannerUtil.java | 124 +------- .../apache/tajo/plan/util}/SchemaUtil.java | 2 +- .../tajo/plan/verifier}/ExprsVerifier.java | 7 +- .../plan/verifier}/LogicalPlanVerifier.java | 18 +- .../verifier}/PreLogicalPlanVerifier.java | 23 +- .../plan/verifier}/VerificationState.java | 2 +- .../tajo/plan/verifier}/VerifyException.java | 4 +- .../visitor}/BasicLogicalPlanVisitor.java | 6 +- .../visitor}/ExplainLogicalPlanVisitor.java | 7 +- .../plan/visitor}/LogicalPlanVisitor.java | 6 +- .../plan/visitor}/SimpleAlgebraVisitor.java | 4 +- .../src/main/proto/Plan.proto | 17 +- .../apache/tajo/plan}/TestExprAnnotator.java | 2 +- .../apache/tajo/plan}/TestLogicalNode.java | 14 +- tajo-project/pom.xml | 5 + .../org/apache/tajo/storage/StorageUtil.java | 20 -- .../apache/tajo/storage/TestMergeScanner.java | 2 +- .../org/apache/tajo/storage/TestStorages.java | 6 +- .../java/org/apache/tajo/storage/Tuple.java | 83 ----- 384 files changed, 1798 insertions(+), 1331 deletions(-) rename tajo-catalog/{tajo-catalog-client => tajo-catalog-common}/src/main/java/org/apache/tajo/catalog/CatalogService.java (100%) rename {tajo-storage => tajo-common}/src/main/java/org/apache/tajo/storage/StorageConstants.java (80%) rename {tajo-storage => tajo-common}/src/main/java/org/apache/tajo/storage/Tuple.java (100%) rename {tajo-storage => tajo-common}/src/main/java/org/apache/tajo/storage/VTuple.java (100%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-common/src/main/java/org/apache/tajo/util}/graph/DirectedGraph.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-common/src/main/java/org/apache/tajo/util}/graph/DirectedGraphCursor.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-common/src/main/java/org/apache/tajo/util}/graph/DirectedGraphVisitor.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-common/src/main/java/org/apache/tajo/util}/graph/Graph.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-common/src/main/java/org/apache/tajo/util}/graph/SimpleDirectedGraph.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-common/src/main/java/org/apache/tajo/util}/graph/SimpleUndirectedGraph.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-common/src/main/java/org/apache/tajo/util}/graph/UndirectedGraph.java (95%) rename {tajo-core/src/test/java/org/apache/tajo/engine/planner => tajo-common/src/test/java/org/apache/tajo/util/graph}/TestSimpleDirectedGraph.java (92%) rename {tajo-core/src/test/java/org/apache/tajo/engine/planner => tajo-common/src/test/java/org/apache/tajo/util/graph}/TestSimpleUndirectedGraph.java (94%) create mode 100644 tajo-plan/pom.xml rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/ExprAnnotator.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/ExprNormalizer.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/exception => tajo-plan/src/main/java/org/apache/tajo/plan}/IllegalQueryStatusException.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/exception => tajo-plan/src/main/java/org/apache/tajo/plan}/InvalidQueryException.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/LogicalOptimizer.java (91%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/LogicalPlan.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/LogicalPlanPreprocessor.java (94%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/LogicalPlanner.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/NamedExprsManager.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/PlanString.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/PlanningException.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/Target.java (91%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/TypeDeterminant.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/algebra}/AlgebraVisitor.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/exception => tajo-plan/src/main/java/org/apache/tajo/plan/algebra}/AmbiguousFieldException.java (92%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/algebra}/BaseAlgebraVisitor.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval => tajo-plan/src/main/java/org/apache/tajo/plan/annotator}/Prioritized.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/AggregationFunctionCallEval.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/AlgebraicException.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/AlgebraicUtil.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/BasicEvalNodeVisitor.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/BetweenPredicateEval.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/BinaryEval.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/CaseWhenEval.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/CastEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/ConstEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/EvalNode.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/EvalNodeVisitor.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/EvalNodeVisitor2.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/EvalTreeFactory.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/EvalTreeUtil.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/EvalType.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/FieldEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/FunctionEval.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/GeneralFunctionEval.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/InEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/InvalidEvalException.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/IsNullEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/LikePredicateEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/NotEval.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/PartialBinaryExpr.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/PatternMatchPredicateEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/RegexPredicateEval.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/RowConstantEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/SignedEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/SimilarToPredicateEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/SimpleEvalNodeVisitor.java (99%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/UnaryEval.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/eval => tajo-plan/src/main/java/org/apache/tajo/plan/expr}/WindowFunctionEval.java (94%) rename {tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval => tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite}/EvalTreeOptimizationRule.java (84%) rename {tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval => tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite}/EvalTreeOptimizer.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval => tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite}/rules/ConstantFolding.java (91%) rename {tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval => tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite}/rules/ConstantPropagation.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine => tajo-plan/src/main/java/org/apache/tajo/plan}/function/AggFunction.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine => tajo-plan/src/main/java/org/apache/tajo/plan}/function/FunctionContext.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine => tajo-plan/src/main/java/org/apache/tajo/plan}/function/GeneralFunction.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine => tajo-plan/src/main/java/org/apache/tajo/plan}/function/WindowAggFunc.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join => tajo-plan/src/main/java/org/apache/tajo/plan/joinorder}/Edge.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join => tajo-plan/src/main/java/org/apache/tajo/plan/joinorder}/FoundJoinOrder.java (92%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join => tajo-plan/src/main/java/org/apache/tajo/plan/joinorder}/GreedyHeuristicJoinOrderAlgorithm.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join => tajo-plan/src/main/java/org/apache/tajo/plan/joinorder}/JoinEdge.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join => tajo-plan/src/main/java/org/apache/tajo/plan/joinorder}/JoinGraph.java (90%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join => tajo-plan/src/main/java/org/apache/tajo/plan/joinorder}/JoinOrderAlgorithm.java (85%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/AlterTableNode.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/logical}/AlterTablespaceNode.java (91%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/BinaryNode.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/CreateDatabaseNode.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/CreateTableNode.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/DistinctGroupbyNode.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/DropDatabaseNode.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/DropTableNode.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/EvalExprNode.java (91%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/ExceptNode.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/logical}/GroupElement.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/GroupbyNode.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/HavingNode.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/IndexScanNode.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/InsertNode.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/IntersectNode.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/JoinNode.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/LimitNode.java (94%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/LogicalNode.java (92%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/LogicalNodeVisitor.java (94%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/LogicalRootNode.java (92%) rename {tajo-core/src/main/java/org/apache/tajo/engine/exception => tajo-plan/src/main/java/org/apache/tajo/plan/logical}/NoSuchColumnException.java (67%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/NodeType.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/PartitionedTableScanNode.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/PersistentStoreNode.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/Projectable.java (81%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/ProjectionNode.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/RelationNode.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/ScanNode.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/SelectableNode.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/SelectionNode.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/ShuffleFileWriteNode.java (88%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/SortNode.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/StoreTableNode.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/TableSubQueryNode.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/TruncateTableNode.java (94%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/UnaryNode.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/UnionNode.java (91%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/WindowAggNode.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/logical/WindowSpec.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/nameresolver/NameResolver.java (96%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/nameresolver/NameResolvingMode.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/nameresolver/ResolverByLegacy.java (91%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/nameresolver/ResolverByRels.java (85%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/nameresolver/ResolverByRelsAndSubExprs.java (86%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/nameresolver/ResolverBySubExprsAndRels.java (86%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/rewrite/BasicQueryRewriteEngine.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/rewrite/QueryRewriteEngine.java (87%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan}/rewrite/RewriteRule.java (92%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite => tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules}/FilterPushDownRule.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite => tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules}/PartitionedTableRewriter.java (79%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite => tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules}/ProjectionPushDownRule.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/json => tajo-plan/src/main/java/org/apache/tajo/plan/serder}/EvalNodeAdapter.java (93%) rename {tajo-core/src/main/java/org/apache/tajo/engine/plan => tajo-plan/src/main/java/org/apache/tajo/plan/serder}/EvalTreeProtoDeserializer.java (95%) rename {tajo-core/src/main/java/org/apache/tajo/engine/plan => tajo-plan/src/main/java/org/apache/tajo/plan/serder}/EvalTreeProtoSerializer.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/json => tajo-plan/src/main/java/org/apache/tajo/plan/serder}/LogicalNodeAdapter.java (91%) create mode 100644 tajo-plan/src/main/java/org/apache/tajo/plan/serder/PlanGsonHelper.java rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/util}/ExprFinder.java (94%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/util}/PlannerUtil.java (84%) rename {tajo-core/src/main/java/org/apache/tajo/engine/utils => tajo-plan/src/main/java/org/apache/tajo/plan/util}/SchemaUtil.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/verifier}/ExprsVerifier.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/verifier}/LogicalPlanVerifier.java (94%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/verifier}/PreLogicalPlanVerifier.java (92%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/verifier}/VerificationState.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/exception => tajo-plan/src/main/java/org/apache/tajo/plan/verifier}/VerifyException.java (90%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/visitor}/BasicLogicalPlanVisitor.java (98%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/visitor}/ExplainLogicalPlanVisitor.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/visitor}/LogicalPlanVisitor.java (97%) rename {tajo-core/src/main/java/org/apache/tajo/engine/planner => tajo-plan/src/main/java/org/apache/tajo/plan/visitor}/SimpleAlgebraVisitor.java (98%) rename {tajo-core => tajo-plan}/src/main/proto/Plan.proto (94%) rename {tajo-core/src/test/java/org/apache/tajo/engine/planner => tajo-plan/src/test/java/org/apache/tajo/plan}/TestExprAnnotator.java (98%) rename {tajo-core/src/test/java/org/apache/tajo/engine/planner => tajo-plan/src/test/java/org/apache/tajo/plan}/TestLogicalNode.java (85%) delete mode 100644 tajo-yarn-pullserver/src/main/java/org/apache/tajo/storage/Tuple.java diff --git a/CHANGES b/CHANGES index e498d9e3e3..a9a31274dc 100644 --- a/CHANGES +++ b/CHANGES @@ -11,6 +11,9 @@ Release 0.9.1 - unreleased IMPROVEMENT + TAJO-1125: Separate logical plan and optimizer into a maven module. + (hyunsik) + TAJO-1092: Improve the function system to allow other function implementation types. (hyunsik) diff --git a/pom.xml b/pom.xml index a3bcf20116..3dca9c0d96 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,7 @@ tajo-project tajo-common tajo-algebra + tajo-plan tajo-core tajo-rpc tajo-catalog diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java similarity index 100% rename from tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/CatalogService.java rename to tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogService.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 7486eff968..8be4a8d35d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -30,6 +30,8 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; +import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; @@ -805,4 +807,31 @@ public static AlterTableDesc addNewColumn(String tableName, Column column, Alter TUtil.putToNestedMap(OPERATION_CASTING_MAP, Type.INET4, Type.INET4, Type.INET4); } + + // table default properties + public static final String BLOCK_SIZE = "parquet.block.size"; + public static final String PAGE_SIZE = "parquet.page.size"; + public static final String COMPRESSION = "parquet.compression"; + public static final String ENABLE_DICTIONARY = "parquet.enable.dictionary"; + public static final String VALIDATION = "parquet.validation"; + + public static KeyValueSet newPhysicalProperties(StoreType type) { + KeyValueSet options = new KeyValueSet(); + if (StoreType.CSV == type) { + options.set(StorageConstants.CSVFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + } else if (StoreType.RCFILE == type) { + options.set(StorageConstants.RCFILE_SERDE, StorageConstants.DEFAULT_BINARY_SERDE); + } else if (StoreType.SEQUENCEFILE == type) { + options.set(StorageConstants.SEQUENCEFILE_SERDE, StorageConstants.DEFAULT_TEXT_SERDE); + options.set(StorageConstants.SEQUENCEFILE_DELIMITER, StorageConstants.DEFAULT_FIELD_DELIMITER); + } else if (type == StoreType.PARQUET) { + options.set(BLOCK_SIZE, StorageConstants.PARQUET_DEFAULT_BLOCK_SIZE); + options.set(PAGE_SIZE, StorageConstants.PARQUET_DEFAULT_PAGE_SIZE); + options.set(COMPRESSION, StorageConstants.PARQUET_DEFAULT_COMPRESSION_CODEC_NAME); + options.set(ENABLE_DICTIONARY, StorageConstants.PARQUET_DEFAULT_IS_DICTIONARY_ENABLED); + options.set(VALIDATION, StorageConstants.PARQUET_DEFAULT_IS_VALIDATION_ENABLED); + } + + return options; + } } diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageConstants.java b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java similarity index 80% rename from tajo-storage/src/main/java/org/apache/tajo/storage/StorageConstants.java rename to tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java index 4b238f9ef3..a76e9ef74d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageConstants.java +++ b/tajo-common/src/main/java/org/apache/tajo/storage/StorageConstants.java @@ -18,11 +18,8 @@ package org.apache.tajo.storage; -import parquet.hadoop.ParquetWriter; -import parquet.hadoop.metadata.CompressionCodecName; - public class StorageConstants { - // table options + // table properties public static final String COMPRESSION_CODEC = "compression.codec"; public static final String COMPRESSION_TYPE = "compression.type"; @@ -39,8 +36,8 @@ public class StorageConstants { public static final String RCFILE_SERDE = "rcfile.serde"; public static final String DEFAULT_FIELD_DELIMITER = "|"; - public static final String DEFAULT_BINARY_SERDE = BinarySerializerDeserializer.class.getName(); - public static final String DEFAULT_TEXT_SERDE = TextSerializerDeserializer.class.getName(); + public static final String DEFAULT_BINARY_SERDE = "org.apache.tajo.storage.BinarySerializerDeserializer"; + public static final String DEFAULT_TEXT_SERDE = "org.apache.tajo.storage.TextSerializerDeserializer"; public static final String PARQUET_DEFAULT_BLOCK_SIZE; public static final String PARQUET_DEFAULT_PAGE_SIZE; @@ -51,16 +48,16 @@ public class StorageConstants { public static final String AVRO_SCHEMA_LITERAL = "avro.schema.literal"; public static final String AVRO_SCHEMA_URL = "avro.schema.url"; + public static final int DEFAULT_BLOCK_SIZE = 128 * 1024 * 1024; + public static final int DEFAULT_PAGE_SIZE = 1 * 1024 * 1024; static { - PARQUET_DEFAULT_BLOCK_SIZE = - Integer.toString(ParquetWriter.DEFAULT_BLOCK_SIZE); - PARQUET_DEFAULT_PAGE_SIZE = - Integer.toString(ParquetWriter.DEFAULT_PAGE_SIZE); + PARQUET_DEFAULT_BLOCK_SIZE = Integer.toString(DEFAULT_BLOCK_SIZE); + PARQUET_DEFAULT_PAGE_SIZE = Integer.toString(DEFAULT_PAGE_SIZE); // When parquet-hadoop 1.3.3 is available, this should be changed to // ParquetWriter.DEFAULT_COMPRESSION_CODEC_NAME. - PARQUET_DEFAULT_COMPRESSION_CODEC_NAME = - CompressionCodecName.UNCOMPRESSED.name().toLowerCase(); + PARQUET_DEFAULT_COMPRESSION_CODEC_NAME = "uncompressed"; + // CompressionCodecName.UNCOMPRESSED.name().toLowerCase(); // When parquet-hadoop 1.3.3 is available, this should be changed to // ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED. diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/Tuple.java b/tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/Tuple.java rename to tajo-common/src/main/java/org/apache/tajo/storage/Tuple.java diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/VTuple.java b/tajo-common/src/main/java/org/apache/tajo/storage/VTuple.java similarity index 100% rename from tajo-storage/src/main/java/org/apache/tajo/storage/VTuple.java rename to tajo-common/src/main/java/org/apache/tajo/storage/VTuple.java diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/DirectedGraph.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraph.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/DirectedGraph.java rename to tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraph.java index c6f4aaa9e7..5433ef577c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/DirectedGraph.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraph.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.graph; +package org.apache.tajo.util.graph; import org.apache.tajo.annotation.Nullable; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/DirectedGraphCursor.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphCursor.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/DirectedGraphCursor.java rename to tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphCursor.java index 7db9b8361e..9b15cfe466 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/DirectedGraphCursor.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphCursor.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.graph; +package org.apache.tajo.util.graph; import java.util.ArrayList; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/DirectedGraphVisitor.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphVisitor.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/DirectedGraphVisitor.java rename to tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphVisitor.java index 3deec7fdba..139c2b4da6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/DirectedGraphVisitor.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/DirectedGraphVisitor.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.graph; +package org.apache.tajo.util.graph; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/Graph.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/Graph.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/Graph.java rename to tajo-common/src/main/java/org/apache/tajo/util/graph/Graph.java index d380b94099..886b09dcfe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/Graph.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/Graph.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.graph; +package org.apache.tajo.util.graph; import org.apache.tajo.annotation.Nullable; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/SimpleDirectedGraph.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/SimpleDirectedGraph.java rename to tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java index 396eb8b428..d72a2dd08b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/SimpleDirectedGraph.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleDirectedGraph.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.graph; +package org.apache.tajo.util.graph; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/SimpleUndirectedGraph.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleUndirectedGraph.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/SimpleUndirectedGraph.java rename to tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleUndirectedGraph.java index 3822ad7772..4ac12c1557 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/SimpleUndirectedGraph.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/SimpleUndirectedGraph.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.graph; +package org.apache.tajo.util.graph; import com.google.common.collect.Lists; import org.apache.tajo.annotation.Nullable; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/UndirectedGraph.java b/tajo-common/src/main/java/org/apache/tajo/util/graph/UndirectedGraph.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/UndirectedGraph.java rename to tajo-common/src/main/java/org/apache/tajo/util/graph/UndirectedGraph.java index 7f74cd1838..454e9e9d70 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/graph/UndirectedGraph.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/graph/UndirectedGraph.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.graph; +package org.apache.tajo.util.graph; import org.apache.tajo.annotation.NotNull; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestSimpleDirectedGraph.java b/tajo-common/src/test/java/org/apache/tajo/util/graph/TestSimpleDirectedGraph.java similarity index 92% rename from tajo-core/src/test/java/org/apache/tajo/engine/planner/TestSimpleDirectedGraph.java rename to tajo-common/src/test/java/org/apache/tajo/util/graph/TestSimpleDirectedGraph.java index 607a6ae29a..a3b10a47a7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestSimpleDirectedGraph.java +++ b/tajo-common/src/test/java/org/apache/tajo/util/graph/TestSimpleDirectedGraph.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.util.graph; -import org.apache.tajo.engine.planner.graph.DirectedGraphVisitor; -import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph; +import org.apache.tajo.util.graph.DirectedGraphVisitor; +import org.apache.tajo.util.graph.SimpleDirectedGraph; import org.junit.Test; import java.util.Stack; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestSimpleUndirectedGraph.java b/tajo-common/src/test/java/org/apache/tajo/util/graph/TestSimpleUndirectedGraph.java similarity index 94% rename from tajo-core/src/test/java/org/apache/tajo/engine/planner/TestSimpleUndirectedGraph.java rename to tajo-common/src/test/java/org/apache/tajo/util/graph/TestSimpleUndirectedGraph.java index 1cd2389121..6fdb138fb6 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestSimpleUndirectedGraph.java +++ b/tajo-common/src/test/java/org/apache/tajo/util/graph/TestSimpleUndirectedGraph.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.util.graph; import com.google.common.collect.Sets; -import org.apache.tajo.engine.planner.graph.SimpleUndirectedGraph; -import org.apache.tajo.engine.planner.graph.UndirectedGraph; +import org.apache.tajo.util.graph.SimpleUndirectedGraph; +import org.apache.tajo.util.graph.UndirectedGraph; import org.junit.Test; import java.util.Set; diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index ca302664ae..fce96e4b60 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -160,13 +160,13 @@ --proto_path=../tajo-common/src/main/proto --proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto --proto_path=../tajo-client/src/main/proto + --proto_path=../tajo-plan/src/main/proto --java_out=target/generated-sources/proto src/main/proto/ResourceTrackerProtocol.proto src/main/proto/QueryMasterProtocol.proto src/main/proto/TajoMasterProtocol.proto src/main/proto/TajoWorkerProtocol.proto src/main/proto/InternalTypes.proto - src/main/proto/Plan.proto @@ -231,6 +231,10 @@ org.apache.tajo tajo-catalog-common + + org.apache.tajo + tajo-plan + org.apache.tajo tajo-catalog-client diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenEmitter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenEmitter.java index 9e45e7d988..611d815d9a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenEmitter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenEmitter.java @@ -19,12 +19,12 @@ package org.apache.tajo.engine.codegen; import com.google.common.base.Preconditions; -import org.apache.tajo.engine.eval.BinaryEval; -import org.apache.tajo.engine.eval.CaseWhenEval; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.EvalType; import org.apache.tajo.org.objectweb.asm.Label; import org.apache.tajo.org.objectweb.asm.Opcodes; +import org.apache.tajo.plan.expr.BinaryEval; +import org.apache.tajo.plan.expr.CaseWhenEval; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.EvalType; import java.util.List; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenSwitchGenerator.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenSwitchGenerator.java index 6f928ac168..c63ccdeecf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenSwitchGenerator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CaseWhenSwitchGenerator.java @@ -19,8 +19,8 @@ package org.apache.tajo.engine.codegen; import com.google.common.collect.Maps; -import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.org.objectweb.asm.Label; +import org.apache.tajo.plan.expr.EvalNode; import java.util.NavigableMap; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CompilationError.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CompilationError.java index 27054deb1a..db6f51fb9d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CompilationError.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/CompilationError.java @@ -18,7 +18,7 @@ package org.apache.tajo.engine.codegen; -import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.plan.expr.EvalNode; public class CompilationError extends RuntimeException { public CompilationError(String message) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeEmitter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeEmitter.java index d94c9071ba..8067519220 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeEmitter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeEmitter.java @@ -18,7 +18,7 @@ package org.apache.tajo.engine.codegen; -import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.plan.expr.EvalNode; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java index 03c04023d8..8384de71ec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java @@ -23,11 +23,11 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.IntervalDatum; -import org.apache.tajo.engine.eval.*; import org.apache.tajo.org.objectweb.asm.ClassWriter; import org.apache.tajo.org.objectweb.asm.MethodVisitor; import org.apache.tajo.org.objectweb.asm.Opcodes; import org.apache.tajo.org.objectweb.asm.commons.GeneratorAdapter; +import org.apache.tajo.plan.expr.*; import org.apache.tajo.storage.Tuple; import java.util.Map; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java index 147b1492cf..69252c4fba 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java @@ -24,12 +24,12 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.IntervalDatum; import org.apache.tajo.datum.ProtobufDatum; -import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.org.objectweb.asm.ClassWriter; import org.apache.tajo.org.objectweb.asm.Label; import org.apache.tajo.org.objectweb.asm.Opcodes; import org.apache.tajo.org.objectweb.asm.Type; +import org.apache.tajo.plan.expr.*; import org.apache.tajo.storage.Tuple; import java.io.PrintStream; @@ -38,7 +38,7 @@ import static org.apache.tajo.common.TajoDataTypes.DataType; import static org.apache.tajo.engine.codegen.TajoGeneratorAdapter.getDescription; -import static org.apache.tajo.engine.eval.FunctionEval.ParamType; +import static org.apache.tajo.plan.expr.FunctionEval.ParamType; public class EvalCodeGenerator extends SimpleEvalNodeVisitor { @@ -701,7 +701,7 @@ public EvalNode visitConst(EvalCodeGenContext context, ConstEval constEval, Stac return constEval; } - public static ParamType [] getParamTypes(EvalNode [] arguments) { + public static ParamType[] getParamTypes(EvalNode [] arguments) { ParamType[] paramTypes = new ParamType[arguments.length]; for (int i = 0; i < arguments.length; i++) { if (arguments[i].getType() == EvalType.CONST) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java index 54d857b8f4..d588e7f59f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ExecutorPreCompiler.java @@ -22,9 +22,13 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.*; -import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.util.Pair; import java.util.Collections; @@ -87,7 +91,7 @@ private static void compileIfAbsent(CompilationContext context, Schema schema, E } private static void compileProjectableNode(CompilationContext context, Schema schema, Projectable node) { - Target [] targets; + Target[] targets; if (node.hasTargets()) { targets = node.getTargets(); } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/LegacyFunctionBindingEmitter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/LegacyFunctionBindingEmitter.java index 944a01c849..36dfe4fb13 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/LegacyFunctionBindingEmitter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/LegacyFunctionBindingEmitter.java @@ -20,9 +20,9 @@ import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.FunctionEval; import org.apache.tajo.org.objectweb.asm.Opcodes; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.FunctionEval; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ScalarFunctionBindingEmitter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ScalarFunctionBindingEmitter.java index 350aee18b4..b33a7c4700 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ScalarFunctionBindingEmitter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/ScalarFunctionBindingEmitter.java @@ -20,12 +20,12 @@ import com.google.common.base.Preconditions; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.FunctionEval; import org.apache.tajo.function.FunctionUtil; import org.apache.tajo.function.StaticMethodInvocationDesc; import org.apache.tajo.org.objectweb.asm.Label; import org.apache.tajo.org.objectweb.asm.Opcodes; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.FunctionEval; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java index 57dc9046c2..9cc1dc3278 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java @@ -22,18 +22,18 @@ import com.google.common.collect.Maps; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.EvalType; import org.apache.tajo.exception.InvalidCastException; import org.apache.tajo.exception.UnsupportedException; -import org.apache.tajo.util.TUtil; -import org.apache.tajo.util.datetime.DateTimeUtil; import org.apache.tajo.org.objectweb.asm.Label; import org.apache.tajo.org.objectweb.asm.MethodVisitor; import org.apache.tajo.org.objectweb.asm.Opcodes; import org.apache.tajo.org.objectweb.asm.Type; import org.apache.tajo.org.objectweb.asm.commons.GeneratorAdapter; import org.apache.tajo.org.objectweb.asm.commons.TableSwitchGenerator; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.EvalType; +import org.apache.tajo.util.TUtil; +import org.apache.tajo.util.datetime.DateTimeUtil; import java.util.HashMap; import java.util.Map; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java index c81f242d38..a055b049a4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java @@ -20,8 +20,8 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.IntervalDatum; -import org.apache.tajo.engine.eval.*; import org.apache.tajo.org.objectweb.asm.Opcodes; +import org.apache.tajo.plan.expr.*; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java index 493c098cb1..5961076a8b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgDouble.java @@ -26,8 +26,8 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.ProtobufDatum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgFloat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgFloat.java index 4a300363c2..2370421fe2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgFloat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgFloat.java @@ -19,7 +19,7 @@ package org.apache.tajo.engine.function.builtin; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java index 192b1a6b9e..07cf37382e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgInt.java @@ -19,7 +19,7 @@ package org.apache.tajo.engine.function.builtin; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java index 4794f4dac6..f48de6c4a6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/AvgLong.java @@ -23,8 +23,8 @@ import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.*; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Coalesce.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Coalesce.java index 98abb024df..474f771938 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Coalesce.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Coalesce.java @@ -21,7 +21,7 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; public abstract class Coalesce extends GeneralFunction { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountRows.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountRows.java index aa1d941dae..e301c1b61d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountRows.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountRows.java @@ -25,8 +25,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValue.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValue.java index 12d8892f05..1abfcb748e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValue.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValue.java @@ -21,7 +21,7 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValueDistinct.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValueDistinct.java index 7a7f3d8acf..1ffb51bcf1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValueDistinct.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/CountValueDistinct.java @@ -24,7 +24,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Date.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Date.java index 7666e89dcd..aee7c60fcc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Date.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Date.java @@ -24,7 +24,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxDouble.java index 5c9ceb15b0..a6840c4e20 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxDouble.java @@ -25,8 +25,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Float8Datum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxFloat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxFloat.java index 067026d188..6b725aff6e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxFloat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxFloat.java @@ -24,8 +24,8 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxInt.java index a329390ca7..2d9cf481ec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxInt.java @@ -24,8 +24,8 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxLong.java index 71bab1a389..b1cc30b4d8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxLong.java @@ -25,8 +25,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxString.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxString.java index 439462fd51..d3c67ce5ff 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxString.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MaxString.java @@ -24,8 +24,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinDouble.java index 7b9e0abbeb..38eba48159 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinDouble.java @@ -24,8 +24,8 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinFloat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinFloat.java index 95a97beffe..17e63e244c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinFloat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinFloat.java @@ -25,8 +25,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Float4Datum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinInt.java index 24b379a7e7..d2a5a2d74d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinInt.java @@ -24,8 +24,8 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinLong.java index ad9ebfa309..e346ac7b15 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinLong.java @@ -25,8 +25,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinString.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinString.java index cbf214ef0e..7e87a49044 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinString.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/MinString.java @@ -25,8 +25,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/RandomInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/RandomInt.java index bdcf485f79..6901d7b1ad 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/RandomInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/RandomInt.java @@ -22,7 +22,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Sleep.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Sleep.java index 0ae8386773..708978aaa9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Sleep.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/Sleep.java @@ -21,7 +21,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java index ece7487b97..a2da84d668 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDouble.java @@ -25,8 +25,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Float8Datum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDoubleDistinct.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDoubleDistinct.java index 9b5b19022f..e2a2e0dd65 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDoubleDistinct.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumDoubleDistinct.java @@ -27,8 +27,8 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Float8Datum; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloat.java index 72c4a6bea0..73257e2e2f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloat.java @@ -24,8 +24,8 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloatDistinct.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloatDistinct.java index f23062237a..a32315dbb9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloatDistinct.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumFloatDistinct.java @@ -27,8 +27,8 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Float8Datum; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumInt.java index ce9812865d..da8bf16aab 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumInt.java @@ -24,8 +24,8 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumIntDistinct.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumIntDistinct.java index e1f21768f6..eb3237b50f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumIntDistinct.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumIntDistinct.java @@ -27,8 +27,8 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java index 894348dbd4..7a6e707239 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLong.java @@ -25,8 +25,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLongDistinct.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLongDistinct.java index d899c3764d..368ede07d5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLongDistinct.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/builtin/SumLongDistinct.java @@ -27,8 +27,8 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddDays.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddDays.java index 2f7a8d7810..643e2c0fe6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddDays.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddDays.java @@ -23,7 +23,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.IntervalDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddMonths.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddMonths.java index 4af4ff9ddb..aef258afe6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddMonths.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/AddMonths.java @@ -23,7 +23,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.IntervalDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java index 2b3fcdf735..f43a41f430 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentDate.java @@ -22,7 +22,7 @@ import org.apache.tajo.datum.DateDatum; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentTime.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentTime.java index 1efdfa9f50..0ba5ca0c50 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentTime.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/CurrentTime.java @@ -22,7 +22,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.TimeDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromDate.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromDate.java index 288fbe15c6..31ecfd9bf7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromDate.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromDate.java @@ -21,7 +21,7 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java index b3184e369b..66a54bf043 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTime.java @@ -24,7 +24,7 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TimeDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java index 98900ef30b..843c593b87 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DatePartFromTimestamp.java @@ -21,7 +21,7 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DateTimePartFromUnixTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DateTimePartFromUnixTimestamp.java index 8705b069e6..97b5b88aa9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DateTimePartFromUnixTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/DateTimePartFromUnixTimestamp.java @@ -21,7 +21,7 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/NowTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/NowTimestamp.java index adc093ca4c..dd0d195561 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/NowTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/NowTimestamp.java @@ -22,7 +22,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.TimestampDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java index 4ad76c4644..02b52e37c4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToCharTimestamp.java @@ -24,10 +24,10 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TimestampDatum; -import org.apache.tajo.engine.eval.FunctionEval; -import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.expr.FunctionEval; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.datetime.DateTimeFormat; import org.apache.tajo.util.datetime.DateTimeUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToDate.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToDate.java index 09a43952e6..727b78a172 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToDate.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToDate.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.DateDatum; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampInt.java index d14cfd69ca..3821d76af3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampInt.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java index f42a171827..e50bacc85a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/datetime/ToTimestampText.java @@ -21,7 +21,7 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryInet4.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryInet4.java index d922e762ce..3789709976 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryInet4.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryInet4.java @@ -24,7 +24,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryText.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryText.java index 17a43be336..232e41419c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryText.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryText.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryInet4.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryInet4.java index 19ce8ba204..46257a4766 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryInet4.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryInet4.java @@ -24,7 +24,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryText.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryText.java index 168f86c4fd..ebd4fad6fe 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryText.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryText.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsDouble.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsDouble.java index 1ff2f29c68..c5357219fb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsDouble.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsDouble.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsFloat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsFloat.java index 4e3f54c533..002b4452ab 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsFloat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsFloat.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsInt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsInt.java index b1fb79e370..00523ec3da 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsInt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsInt.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsLong.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsLong.java index cdb5a29c90..82a6ba80a3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsLong.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/AbsLong.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Acos.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Acos.java index f1f280443d..f5e44150ef 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Acos.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Acos.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Asin.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Asin.java index 6f6c905ab0..898b73c674 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Asin.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Asin.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Atan.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Atan.java index 9e70ae3fb2..15b9fb7ddc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Atan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Atan.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Atan2.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Atan2.java index bc2764c5f9..e8401af0e7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Atan2.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Atan2.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Cbrt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Cbrt.java index b85fe5819b..0f83ea016e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Cbrt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Cbrt.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Ceil.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Ceil.java index cca7f60d96..4bfe501ceb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Ceil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Ceil.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Cos.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Cos.java index 997784a442..3a71ae7410 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Cos.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Cos.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Degrees.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Degrees.java index cd5fd2f345..61d87fbd4f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Degrees.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Degrees.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Div.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Div.java index d904a2c151..4c82a79122 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Div.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Div.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Exp.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Exp.java index 3c3b07cd39..2657d80730 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Exp.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Exp.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Floor.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Floor.java index 7e4cee0d9c..aca0821c38 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Floor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Floor.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Mod.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Mod.java index c748a2d459..bfb4d54c78 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Mod.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Mod.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pi.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pi.java index abc06b51a5..ab2ff774c7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pi.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pi.java @@ -21,7 +21,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pow.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pow.java index 269fec8acc..8a28c84749 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pow.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Pow.java @@ -19,11 +19,10 @@ package org.apache.tajo.engine.function.math; import org.apache.tajo.catalog.Column; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Radians.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Radians.java index a8bf3637fa..26e767c7e1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Radians.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Radians.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Round.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Round.java index cdcb70a62b..2a13378550 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Round.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Round.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/RoundFloat8.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/RoundFloat8.java index d104431e61..f6a92f9d6c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/RoundFloat8.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/RoundFloat8.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.exception.InvalidOperationException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sign.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sign.java index f9c49bfcf8..71e4d01306 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sign.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sign.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sin.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sin.java index 31d56b46ce..e4f7877409 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sin.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sin.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sqrt.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sqrt.java index aba33f48a7..4e11908794 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sqrt.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Sqrt.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Tan.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Tan.java index d551d8b120..ff3099027d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Tan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/math/Tan.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Ascii.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Ascii.java index ecb32a42c2..0a43947f47 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Ascii.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Ascii.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BTrim.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BTrim.java index b7c5611865..3214a5eef0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BTrim.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BTrim.java @@ -25,10 +25,10 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.FunctionEval; -import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.expr.FunctionEval; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; /** diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BitLength.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BitLength.java index 3a5c216624..c099c77d89 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BitLength.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/BitLength.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/CharLength.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/CharLength.java index 76d3b556fc..b287c93bc5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/CharLength.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/CharLength.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Chr.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Chr.java index fac4b273e3..4d0a4a9c74 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Chr.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Chr.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat.java index 50b14401d4..3268817bd4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat_ws.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat_ws.java index d8e2113c28..4a723cb04a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat_ws.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Concat_ws.java @@ -25,7 +25,7 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Decode.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Decode.java index 13f0749242..72d350110b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Decode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Decode.java @@ -25,7 +25,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Digest.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Digest.java index 163c6fefbf..3de914a849 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Digest.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Digest.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Encode.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Encode.java index 0023ae65af..4cdbc24a29 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Encode.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Encode.java @@ -25,7 +25,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/FindInSet.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/FindInSet.java index 0c3e22120e..ced7d4e71c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/FindInSet.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/FindInSet.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/InitCap.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/InitCap.java index 4347dbb932..35bb1640b0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/InitCap.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/InitCap.java @@ -24,7 +24,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/LTrim.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/LTrim.java index db6b714bef..f3452939fc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/LTrim.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/LTrim.java @@ -25,10 +25,10 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.FunctionEval; -import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.expr.FunctionEval; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; /** diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Left.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Left.java index 31469f5398..bf3fbfb6ee 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Left.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Left.java @@ -24,7 +24,7 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Length.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Length.java index 389f3586f8..39bdd2fd8c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Length.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Length.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Locate.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Locate.java index 67ee389ef5..7b703992bd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Locate.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Locate.java @@ -24,7 +24,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lower.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lower.java index 80c8192755..7c1e33e4cb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lower.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lower.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lpad.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lpad.java index bc0031e90b..58c0951867 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lpad.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Lpad.java @@ -25,10 +25,10 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.FunctionEval; -import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.expr.FunctionEval; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; /** diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Md5.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Md5.java index fb5f73c4b3..f4cb7cb230 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Md5.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Md5.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/OctetLength.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/OctetLength.java index dc719077de..cb5af15e13 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/OctetLength.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/OctetLength.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/QuoteIdent.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/QuoteIdent.java index d5237d70d0..aaef10d802 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/QuoteIdent.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/QuoteIdent.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RTrim.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RTrim.java index 9e3ff8d8a9..8e295fa7fa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RTrim.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RTrim.java @@ -25,10 +25,10 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.FunctionEval; -import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.expr.FunctionEval; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; /** diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RegexpReplace.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RegexpReplace.java index 2b59d344c3..da06f732a3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RegexpReplace.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/RegexpReplace.java @@ -20,17 +20,20 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.*; +import org.apache.tajo.datum.BooleanDatum; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.datum.NullDatum; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; import java.util.regex.Matcher; import java.util.regex.Pattern; -import static org.apache.tajo.engine.eval.FunctionEval.ParamType; +import static org.apache.tajo.plan.expr.FunctionEval.ParamType; /** * This function is defined as: @@ -67,7 +70,7 @@ public RegexpReplace() { }); } - public void init(ParamType [] paramTypes) { + public void init(ParamType[] paramTypes) { if (paramTypes[0] == ParamType.NULL || paramTypes[1] == ParamType.NULL || paramTypes[2] == ParamType.NULL) { isAlwaysNull = true; } else if (paramTypes[1] == ParamType.CONSTANT) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Repeat.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Repeat.java index 4c52a6e9e3..c02ef661d9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Repeat.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Repeat.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Reverse.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Reverse.java index a77ba696d9..bf672940f1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Reverse.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Reverse.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Right.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Right.java index aa0dad0649..68af4236bb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Right.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Right.java @@ -24,7 +24,7 @@ import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Rpad.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Rpad.java index 225777b0e7..9d42848fa9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Rpad.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Rpad.java @@ -25,10 +25,10 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.FunctionEval; -import org.apache.tajo.engine.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.plan.expr.FunctionEval; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/SplitPart.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/SplitPart.java index fd1de55175..1c3be00299 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/SplitPart.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/SplitPart.java @@ -20,7 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/StrPos.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/StrPos.java index d6f88de3b2..9c3e4f1e4c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/StrPos.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/StrPos.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/StrPosb.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/StrPosb.java index 56fb3a1fc5..c33200693c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/StrPosb.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/StrPosb.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Substr.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Substr.java index 74492b1f71..7e4ddd6d9e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Substr.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Substr.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToBin.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToBin.java index 3048aa8bb8..a9f85f37fa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToBin.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToBin.java @@ -22,7 +22,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToHex.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToHex.java index a8624ab508..2e20008237 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToHex.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/ToHex.java @@ -24,7 +24,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Upper.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Upper.java index 47d38ba25e..fc6ff3a483 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Upper.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/string/Upper.java @@ -23,7 +23,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/window/Rank.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/window/Rank.java index e461f9d795..0df3f23426 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/window/Rank.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/window/Rank.java @@ -24,8 +24,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.function.WindowAggFunc; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.plan.function.WindowAggFunc; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/window/RowNumber.java b/tajo-core/src/main/java/org/apache/tajo/engine/function/window/RowNumber.java index 8b0943f572..bd838f8cd5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/window/RowNumber.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/function/window/RowNumber.java @@ -23,8 +23,8 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.Int8Datum; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.function.WindowAggFunc; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.plan.function.WindowAggFunc; import org.apache.tajo.engine.function.annotation.Description; import org.apache.tajo.engine.function.annotation.ParamTypes; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java b/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java index 0340a3233e..57f2536058 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/json/CoreGsonHelper.java @@ -22,16 +22,18 @@ import com.google.gson.GsonBuilder; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.function.Function; import org.apache.tajo.catalog.json.FunctionAdapter; import org.apache.tajo.catalog.json.TableMetaAdapter; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.GeneralFunction; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.GeneralFunction; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.function.Function; import org.apache.tajo.json.*; +import org.apache.tajo.plan.serder.EvalNodeAdapter; +import org.apache.tajo.plan.serder.LogicalNodeAdapter; import org.apache.tajo.util.TUtil; import java.lang.reflect.Type; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLSyntaxError.java b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLSyntaxError.java index d56550953c..e4bfc7c392 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLSyntaxError.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/parser/SQLSyntaxError.java @@ -19,7 +19,7 @@ package org.apache.tajo.engine.parser; -import org.apache.tajo.engine.exception.InvalidQueryException; +import org.apache.tajo.plan.InvalidQueryException; public class SQLSyntaxError extends InvalidQueryException { private static final long serialVersionUID = 5388279335175632066L; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BroadcastJoinMarkCandidateVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BroadcastJoinMarkCandidateVisitor.java index 28ef000b93..6b3dc487d2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BroadcastJoinMarkCandidateVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BroadcastJoinMarkCandidateVisitor.java @@ -19,10 +19,13 @@ package org.apache.tajo.engine.planner; import org.apache.tajo.engine.planner.global.GlobalPlanner; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.ScanNode; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BroadcastJoinPlanVisitor.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BroadcastJoinPlanVisitor.java index 0fb8f36002..bd8efbee6a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BroadcastJoinPlanVisitor.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/BroadcastJoinPlanVisitor.java @@ -19,10 +19,13 @@ package org.apache.tajo.engine.planner; import org.apache.tajo.engine.planner.global.GlobalPlanner; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.ScanNode; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlanner.java index ebe47b4bd7..d4c57db220 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlanner.java @@ -22,7 +22,7 @@ package org.apache.tajo.engine.planner; import org.apache.tajo.worker.TaskAttemptContext; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.exception.InternalException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java index 485677a41d..f7f071f40d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PhysicalPlannerImpl.java @@ -36,7 +36,6 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; -import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.physical.*; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InternalException; @@ -45,6 +44,9 @@ import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.TupleComparator; @@ -56,7 +58,6 @@ import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.List; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/Projector.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/Projector.java index d8499d0696..7c2e81fb09 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/Projector.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/Projector.java @@ -20,14 +20,16 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.worker.TaskAttemptContext; public class Projector { private final TaskAttemptContext context; private final Schema inSchema; - private final Target [] targets; + private final Target[] targets; // for projection private final int targetNum; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java index b3b5bb02ce..4a2cd19396 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/DataChannel.java @@ -22,11 +22,13 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.plan.util.SchemaUtil; import org.apache.tajo.util.TUtil; import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import static org.apache.tajo.ipc.TajoWorkerProtocol.*; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; +import static org.apache.tajo.plan.serder.PlanProto.TransmitType; public class DataChannel { private ExecutionBlockId srcId; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java index 1d14996af8..77eb32d29e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/ExecutionBlock.java @@ -16,7 +16,7 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.plan.logical.*; import java.util.*; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java index 01e02d7286..b33e4a09fd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/GlobalPlanner.java @@ -33,13 +33,19 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.planner.*; +import org.apache.tajo.engine.planner.BroadcastJoinMarkCandidateVisitor; +import org.apache.tajo.engine.planner.BroadcastJoinPlanVisitor; import org.apache.tajo.engine.planner.global.builder.DistinctGroupbyBuilder; -import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule; import org.apache.tajo.exception.InternalException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TajoWorker; @@ -48,7 +54,7 @@ import java.util.*; import static org.apache.tajo.conf.TajoConf.ConfVars; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.*; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.*; /** * Build DAG @@ -520,7 +526,7 @@ private void buildJoinPlanWithUnionChannel(GlobalPlanContext context, JoinNode j } } - private AggregationFunctionCallEval createSumFunction(EvalNode [] args) throws InternalException { + private AggregationFunctionCallEval createSumFunction(EvalNode[] args) throws InternalException { FunctionDesc functionDesc = getCatalog().getFunction("sum", CatalogProtos.FunctionType.AGGREGATION, args[0].getValueType()); return new AggregationFunctionCallEval(functionDesc, (AggFunction) functionDesc.newInstance(), args); @@ -556,7 +562,7 @@ private AggregationFunctionCallEval createMinFunction(EvalNode [] args) throws I */ private static class RewrittenFunctions { AggregationFunctionCallEval [] firstStageEvals; - Target [] firstStageTargets; + Target[] firstStageTargets; AggregationFunctionCallEval secondStageEvals; public RewrittenFunctions(int firstStageEvalNum) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java index 9b99c5f4ac..3cca4f2943 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/MasterPlan.java @@ -23,10 +23,10 @@ import org.apache.tajo.ExecutionBlockId; import org.apache.tajo.QueryId; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlannerUtil; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph; +import org.apache.tajo.util.graph.SimpleDirectedGraph; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.TajoWorkerProtocol; @@ -36,6 +36,8 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; + public class MasterPlan { private final QueryId queryId; private final QueryContext context; @@ -110,11 +112,11 @@ public void addConnect(DataChannel dataChannel) { execBlockGraph.addEdge(dataChannel.getSrcId(), dataChannel.getTargetId(), dataChannel); } - public void addConnect(ExecutionBlock src, ExecutionBlock target, TajoWorkerProtocol.ShuffleType type) { + public void addConnect(ExecutionBlock src, ExecutionBlock target, ShuffleType type) { addConnect(src.getId(), target.getId(), type); } - public void addConnect(ExecutionBlockId src, ExecutionBlockId target, TajoWorkerProtocol.ShuffleType type) { + public void addConnect(ExecutionBlockId src, ExecutionBlockId target, ShuffleType type) { addConnect(new DataChannel(src, target, type)); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java index cbe2d7ea45..671bb1920f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/global/builder/DistinctGroupbyBuilder.java @@ -25,32 +25,31 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.engine.eval.AggregationFunctionCallEval; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.EvalTreeUtil; -import org.apache.tajo.engine.eval.FieldEval; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.Target; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.GlobalPlanner.GlobalPlanContext; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.planner.logical.DistinctGroupbyNode; -import org.apache.tajo.engine.planner.logical.GroupbyNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.ScanNode; -import org.apache.tajo.engine.planner.rewrite.ProjectionPushDownRule; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.DistinctAggregationAlgorithm; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.SortSpecArray; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.AggregationFunctionCallEval; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.EvalTreeUtil; +import org.apache.tajo.plan.expr.FieldEval; +import org.apache.tajo.plan.logical.DistinctGroupbyNode; +import org.apache.tajo.plan.logical.GroupbyNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.util.TUtil; import java.util.*; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.HASH_SHUFFLE; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.HASH_SHUFFLE; public class DistinctGroupbyBuilder { private static Log LOG = LogFactory.getLog(DistinctGroupbyBuilder.class); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/AggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/AggregationExec.java index 2a671e6306..8c484a02e0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/AggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/AggregationExec.java @@ -20,8 +20,8 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.eval.AggregationFunctionCallEval; -import org.apache.tajo.engine.planner.logical.GroupbyNode; +import org.apache.tajo.plan.expr.AggregationFunctionCallEval; +import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BNLJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BNLJoinExec.java index 91cefa15e3..117b04c4b8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BNLJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BNLJoinExec.java @@ -18,10 +18,10 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.JoinNode; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.FrameTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java index 51f70a0b7e..f699607998 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/BSTIndexScanExec.java @@ -22,9 +22,9 @@ import org.apache.hadoop.io.IOUtils; import org.apache.tajo.catalog.Schema; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.index.bst.BSTIndex; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java index a07290401c..ec239de648 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java @@ -30,10 +30,10 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.engine.planner.logical.CreateTableNode; -import org.apache.tajo.engine.planner.logical.InsertNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.StoreTableNode; +import org.apache.tajo.plan.logical.CreateTableNode; +import org.apache.tajo.plan.logical.InsertNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.StoreTableNode; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.StorageUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java index 7201ed411d..bd24fa352c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyFirstAggregationExec.java @@ -24,10 +24,10 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.datum.Int2Datum; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.AggregationFunctionCallEval; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.planner.logical.DistinctGroupbyNode; -import org.apache.tajo.engine.planner.logical.GroupbyNode; +import org.apache.tajo.plan.expr.AggregationFunctionCallEval; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.plan.logical.DistinctGroupbyNode; +import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java index 3fac509aba..eac5c70b68 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyHashAggregationExec.java @@ -23,10 +23,10 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.AggregationFunctionCallEval; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.planner.logical.DistinctGroupbyNode; -import org.apache.tajo.engine.planner.logical.GroupbyNode; +import org.apache.tajo.plan.expr.AggregationFunctionCallEval; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.plan.logical.DistinctGroupbyNode; +import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java index bc8885f964..383ccd3b4c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySecondAggregationExec.java @@ -18,22 +18,22 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.eval.AggregationFunctionCallEval; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.planner.logical.DistinctGroupbyNode; -import org.apache.tajo.engine.planner.logical.GroupbyNode; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.storage.VTuple; -import org.apache.tajo.worker.TaskAttemptContext; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; + import org.apache.commons.logging.Log; + import org.apache.commons.logging.LogFactory; + import org.apache.tajo.catalog.Column; + import org.apache.tajo.plan.expr.AggregationFunctionCallEval; + import org.apache.tajo.plan.function.FunctionContext; + import org.apache.tajo.plan.logical.DistinctGroupbyNode; + import org.apache.tajo.plan.logical.GroupbyNode; + import org.apache.tajo.storage.Tuple; + import org.apache.tajo.storage.VTuple; + import org.apache.tajo.worker.TaskAttemptContext; + + import java.io.IOException; + import java.util.ArrayList; + import java.util.HashSet; + import java.util.List; + import java.util.Set; /** * This class adjusts shuffle columns between DistinctGroupbyFirstAggregationExec and diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java index b786672dfa..06b241c734 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbySortAggregationExec.java @@ -22,8 +22,8 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.planner.logical.DistinctGroupbyNode; -import org.apache.tajo.engine.planner.logical.GroupbyNode; +import org.apache.tajo.plan.logical.DistinctGroupbyNode; +import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java index 239dabf5d9..ff6fc4a02f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/DistinctGroupbyThirdAggregationExec.java @@ -21,12 +21,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.Column; -import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.AggregationFunctionCallEval; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.planner.Target; -import org.apache.tajo.engine.planner.logical.DistinctGroupbyNode; -import org.apache.tajo.engine.planner.logical.GroupbyNode; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.AggregationFunctionCallEval; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.plan.logical.DistinctGroupbyNode; +import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/EvalExprExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/EvalExprExec.java index b1ab7c4c7a..4e02e67e5b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/EvalExprExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/EvalExprExec.java @@ -18,8 +18,8 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.tajo.engine.planner.Target; -import org.apache.tajo.engine.planner.logical.EvalExprNode; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.logical.EvalExprNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java index cfd7fb784d..4c0caea69a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ExternalSortExec.java @@ -36,7 +36,7 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.planner.PhysicalPlanningException; -import org.apache.tajo.engine.planner.logical.SortNode; +import org.apache.tajo.plan.logical.SortNode; import org.apache.tajo.storage.*; import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java index 3323d1fb87..80bba2b6b8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashAggregateExec.java @@ -18,8 +18,8 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.planner.logical.GroupbyNode; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java index e27a43dea9..c28a5cd676 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashBasedColPartitionStoreExec.java @@ -21,7 +21,7 @@ import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.planner.logical.StoreTableNode; +import org.apache.tajo.plan.logical.StoreTableNode; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.StringUtils; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java index 9dabbb3edf..cf333b0c53 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashFullOuterJoinExec.java @@ -20,12 +20,12 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.engine.codegen.CompilationError; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.engine.utils.TupleUtil; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.FrameTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java index 426a7a19e2..18be6b9c0c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashJoinExec.java @@ -20,11 +20,11 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.FrameTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftAntiJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftAntiJoinExec.java index 50a1438dab..236f5e3644 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftAntiJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftAntiJoinExec.java @@ -20,7 +20,7 @@ import org.apache.tajo.worker.TaskAttemptContext; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.planner.logical.JoinNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java index b752db5c23..c38be88936 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftOuterJoinExec.java @@ -22,14 +22,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.eval.AlgebraicUtil; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.EvalTreeUtil; -import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.engine.utils.TupleUtil; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.expr.AlgebraicUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.EvalTreeUtil; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.FrameTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftSemiJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftSemiJoinExec.java index 4fdd03ae62..5196a63c0d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftSemiJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashLeftSemiJoinExec.java @@ -20,7 +20,7 @@ import org.apache.tajo.worker.TaskAttemptContext; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.planner.logical.JoinNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java index 47fcb8d71f..5bf80fd4d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HashShuffleFileWriteExec.java @@ -26,7 +26,7 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.engine.planner.logical.ShuffleFileWriteNode; +import org.apache.tajo.plan.logical.ShuffleFileWriteNode; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.HashShuffleAppender; import org.apache.tajo.storage.HashShuffleAppenderManager; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HavingExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HavingExec.java index 0418f65168..f9f4351004 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HavingExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/HavingExec.java @@ -18,8 +18,8 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.logical.HavingNode; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.HavingNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/LimitExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/LimitExec.java index d736c25d88..14e236637b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/LimitExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/LimitExec.java @@ -20,7 +20,7 @@ import org.apache.tajo.worker.TaskAttemptContext; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.planner.logical.LimitNode; +import org.apache.tajo.plan.logical.LimitNode; import org.apache.tajo.storage.Tuple; import java.io.IOException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java index 9f4f20a211..13fec7b0a2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MemSortExec.java @@ -19,7 +19,7 @@ package org.apache.tajo.engine.planner.physical; import org.apache.tajo.worker.TaskAttemptContext; -import org.apache.tajo.engine.planner.logical.SortNode; +import org.apache.tajo.plan.logical.SortNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java index e1cc6a8359..cb2552b6b2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeFullOuterJoinExec.java @@ -20,11 +20,10 @@ import com.google.common.base.Preconditions; import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.utils.TupleUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.FrameTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; @@ -84,7 +83,7 @@ public MergeFullOuterJoinExec(TaskAttemptContext context, JoinNode plan, Physica this.joincomparator = new JoinTupleComparator(leftChild.getSchema(), rightChild.getSchema(), sortSpecs); - this.tupleComparator = PlannerUtil.getComparatorsFromJoinQual( + this.tupleComparator = PhysicalPlanUtil.getComparatorsFromJoinQual( plan.getJoinQual(), leftChild.getSchema(), rightChild.getSchema()); // for projection diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java index bbfe973860..20df210b86 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/MergeJoinExec.java @@ -20,10 +20,9 @@ import com.google.common.base.Preconditions; import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.JoinNode; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.FrameTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; @@ -78,7 +77,7 @@ public MergeJoinExec(TaskAttemptContext context, JoinNode plan, PhysicalExec out this.joincomparator = new JoinTupleComparator(outer.getSchema(), inner.getSchema(), sortSpecs); - this.tupleComparator = PlannerUtil.getComparatorsFromJoinQual( + this.tupleComparator = PhysicalPlanUtil.getComparatorsFromJoinQual( plan.getJoinQual(), outer.getSchema(), inner.getSchema()); this.outerIterator = outerTupleSlots.iterator(); this.innerIterator = innerTupleSlots.iterator(); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/NLJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/NLJoinExec.java index dc061ed935..b5c6244f23 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/NLJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/NLJoinExec.java @@ -18,9 +18,9 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.JoinNode; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.FrameTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/NLLeftOuterJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/NLLeftOuterJoinExec.java index 37ef7dfed8..8ff757064c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/NLLeftOuterJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/NLLeftOuterJoinExec.java @@ -18,10 +18,10 @@ package org.apache.tajo.engine.planner.physical; -import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.utils.TupleUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.FrameTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java index 4c72075c79..0569c1b2b5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PartitionMergeScanExec.java @@ -21,8 +21,8 @@ import com.google.common.collect.Lists; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java index 2f55cf7901..1b824b5266 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/PhysicalPlanUtil.java @@ -18,15 +18,34 @@ package org.apache.tajo.engine.planner.physical; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.tajo.SessionVars; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.PhysicalPlanningException; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.PersistentStoreNode; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.PersistentStoreNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.storage.TupleComparator; +import org.apache.tajo.storage.fragment.FileFragment; +import org.apache.tajo.storage.fragment.FragmentConvertor; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.Stack; +import java.util.concurrent.atomic.AtomicInteger; public class PhysicalPlanUtil { public static T findExecutor(PhysicalExec plan, Class clazz) @@ -34,6 +53,106 @@ public static T findExecutor(PhysicalExec plan, Class(), clazz); } + public static TupleComparator[] getComparatorsFromJoinQual(EvalNode joinQual, Schema leftSchema, Schema rightSchema) { + SortSpec[][] sortSpecs = PlannerUtil.getSortKeysFromJoinQual(joinQual, leftSchema, rightSchema); + TupleComparator[] comparators = new TupleComparator[2]; + comparators[0] = new TupleComparator(leftSchema, sortSpecs[0]); + comparators[1] = new TupleComparator(rightSchema, sortSpecs[1]); + return comparators; + } + + /** + * Listing table data file which is not empty. + * If the table is a partitioned table, return file list which has same partition key. + * @param tajoConf + * @param tableDesc + * @param fileIndex + * @param numResultFiles + * @return + * @throws java.io.IOException + */ + public static CatalogProtos.FragmentProto[] getNonZeroLengthDataFiles(TajoConf tajoConf,TableDesc tableDesc, + int fileIndex, int numResultFiles) throws IOException { + FileSystem fs = tableDesc.getPath().getFileSystem(tajoConf); + + List nonZeroLengthFiles = new ArrayList(); + if (fs.exists(tableDesc.getPath())) { + getNonZeroLengthDataFiles(fs, tableDesc.getPath(), nonZeroLengthFiles, fileIndex, numResultFiles, + new AtomicInteger(0)); + } + + List fragments = new ArrayList(); + + //In the case of partitioned table, return same partition key data files. + int numPartitionColumns = 0; + if (tableDesc.hasPartition()) { + numPartitionColumns = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size(); + } + String[] previousPartitionPathNames = null; + for (FileStatus eachFile: nonZeroLengthFiles) { + FileFragment fileFragment = new FileFragment(tableDesc.getName(), eachFile.getPath(), 0, eachFile.getLen(), null); + + if (numPartitionColumns > 0) { + // finding partition key; + Path filePath = fileFragment.getPath(); + Path parentPath = filePath; + String[] parentPathNames = new String[numPartitionColumns]; + for (int i = 0; i < numPartitionColumns; i++) { + parentPath = parentPath.getParent(); + parentPathNames[numPartitionColumns - i - 1] = parentPath.getName(); + } + + // If current partitionKey == previousPartitionKey, add to result. + if (previousPartitionPathNames == null) { + fragments.add(fileFragment); + } else if (previousPartitionPathNames != null && Arrays.equals(previousPartitionPathNames, parentPathNames)) { + fragments.add(fileFragment); + } else { + break; + } + previousPartitionPathNames = parentPathNames; + } else { + fragments.add(fileFragment); + } + } + return FragmentConvertor.toFragmentProtoArray(fragments.toArray(new FileFragment[]{})); + } + + private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List result, + int startFileIndex, int numResultFiles, + AtomicInteger currentFileIndex) throws IOException { + if (fs.isDirectory(path)) { + FileStatus[] files = fs.listStatus(path, StorageManager.hiddenFileFilter); + if (files != null && files.length > 0) { + for (FileStatus eachFile : files) { + if (result.size() >= numResultFiles) { + return; + } + if (eachFile.isDirectory()) { + getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles, + currentFileIndex); + } else if (eachFile.isFile() && eachFile.getLen() > 0) { + if (currentFileIndex.get() >= startFileIndex) { + result.add(eachFile); + } + currentFileIndex.incrementAndGet(); + } + } + } + } else { + FileStatus fileStatus = fs.getFileStatus(path); + if (fileStatus != null && fileStatus.getLen() > 0) { + if (currentFileIndex.get() >= startFileIndex) { + result.add(fileStatus); + } + currentFileIndex.incrementAndGet(); + if (result.size() >= numResultFiles) { + return; + } + } + } + } + private static class FindVisitor extends BasicPhysicalExecutorVisitor, PhysicalExec> { public PhysicalExec visit(PhysicalExec exec, Stack stack, Class target) throws PhysicalPlanningException { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java index 89cd75af94..72a667dbba 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ProjectionExec.java @@ -22,7 +22,7 @@ package org.apache.tajo.engine.planner.physical; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.Projectable; +import org.apache.tajo.plan.logical.Projectable; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java index 4e5d1cfac2..dd729109da 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RangeShuffleFileWriteExec.java @@ -27,7 +27,7 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.planner.PlannerUtil; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.index.bst.BSTIndex; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java index 5d4dad5b55..9d8122fa7e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/RightOuterMergeJoinExec.java @@ -21,11 +21,10 @@ import com.google.common.base.Preconditions; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.logical.JoinNode; import org.apache.tajo.engine.utils.TupleUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.JoinNode; import org.apache.tajo.storage.FrameTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; @@ -83,7 +82,7 @@ public RightOuterMergeJoinExec(TaskAttemptContext context, JoinNode plan, Physic sortSpecs[1] = innerSortKey; this.joinComparator = new JoinTupleComparator(outer.getSchema(), inner.getSchema(), sortSpecs); - this.tupleComparator = PlannerUtil.getComparatorsFromJoinQual( + this.tupleComparator = PhysicalPlanUtil.getComparatorsFromJoinQual( plan.getJoinQual(), outer.getSchema(), inner.getSchema()); // for projection diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SelectionExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SelectionExec.java index 5ae9a8f289..9e84462bf5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SelectionExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SelectionExec.java @@ -19,8 +19,8 @@ package org.apache.tajo.engine.planner.physical; import org.apache.tajo.engine.codegen.CompilationError; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.logical.SelectionNode; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.SelectionNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java index c7f8e2dd9d..c6a4f55a19 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SeqScanExec.java @@ -27,17 +27,17 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.datum.Datum; import org.apache.tajo.engine.codegen.CompilationError; -import org.apache.tajo.engine.eval.ConstEval; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.EvalTreeUtil; -import org.apache.tajo.engine.eval.FieldEval; import org.apache.tajo.engine.planner.Projector; -import org.apache.tajo.engine.planner.Target; -import org.apache.tajo.engine.planner.logical.ScanNode; -import org.apache.tajo.engine.utils.SchemaUtil; import org.apache.tajo.engine.utils.TupleCache; import org.apache.tajo.engine.utils.TupleCacheKey; -import org.apache.tajo.engine.utils.TupleUtil; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.ConstEval; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.EvalTreeUtil; +import org.apache.tajo.plan.expr.FieldEval; +import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.FragmentConvertor; @@ -118,7 +118,8 @@ private void rewriteColumnPartitionedTableSchema() throws IOException { // Get a partition key value from a given path Tuple partitionRow = - TupleUtil.buildTupleFromPartitionPath(columnPartitionSchema, fileFragments.get(0).getPath(), false); + PartitionedTableRewriter.buildTupleFromPartitionPath(columnPartitionSchema, fileFragments.get(0).getPath(), + false); // Targets or search conditions may contain column references. // However, actual values absent in tuples. So, Replace all column references by constant datum. diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java index c4d43a38cb..425eb86282 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortAggregateExec.java @@ -19,8 +19,8 @@ package org.apache.tajo.engine.planner.physical; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.planner.logical.GroupbyNode; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.worker.TaskAttemptContext; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java index 6084f0e86d..f7c20fc870 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/SortBasedColPartitionStoreExec.java @@ -23,7 +23,7 @@ import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.planner.logical.StoreTableNode; +import org.apache.tajo.plan.logical.StoreTableNode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.StringUtils; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java index b496e42c9d..0dc172cbca 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/StoreTableExec.java @@ -26,8 +26,8 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.engine.planner.logical.InsertNode; -import org.apache.tajo.engine.planner.logical.PersistentStoreNode; +import org.apache.tajo.plan.logical.InsertNode; +import org.apache.tajo.plan.logical.PersistentStoreNode; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/UnionExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/UnionExec.java index 497c6d3205..0383ae730c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/UnionExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/UnionExec.java @@ -22,7 +22,7 @@ package org.apache.tajo.engine.planner.physical; import org.apache.tajo.worker.TaskAttemptContext; -import org.apache.tajo.engine.exception.InvalidQueryException; +import org.apache.tajo.plan.InvalidQueryException; import org.apache.tajo.storage.Tuple; import java.io.IOException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/WindowAggExec.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/WindowAggExec.java index 7aeed1310b..3f4b22b86c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/WindowAggExec.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/WindowAggExec.java @@ -23,10 +23,10 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.eval.WindowFunctionEval; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.planner.logical.WindowAggNode; -import org.apache.tajo.engine.planner.logical.WindowSpec; +import org.apache.tajo.plan.expr.WindowFunctionEval; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.plan.logical.WindowAggNode; +import org.apache.tajo.plan.logical.WindowSpec; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index f4160e495c..86fd355d72 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -24,8 +24,8 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.master.session.Session; +import org.apache.tajo.plan.logical.NodeType; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; @@ -132,9 +132,9 @@ public void setPartitionMethod(PartitionMethodDesc partitionMethodDesc) { put(QueryVars.OUTPUT_PARTITIONS, partitionMethodDesc != null ? partitionMethodDesc.toJson() : null); } - public PartitionMethodDesc getPartitionMethod() { - return PartitionMethodDesc.fromJson(get(QueryVars.OUTPUT_PARTITIONS)); - } +// public PartitionMethodDesc getPartitionMethod() { +// return PartitionMethodDesc.fromJson(get(QueryVars.OUTPUT_PARTITIONS)); +// } public void setOutputOverwrite() { setBool(QueryVars.OUTPUT_OVERWRITE, true); @@ -160,8 +160,8 @@ public boolean equalKey(ConfigKey key, String another) { } } - public boolean isCommandType(NodeType commandType) { - return equalKey(QueryVars.COMMAND_TYPE, commandType.name()); + public boolean isCommandType(String commandType) { + return equalKey(QueryVars.COMMAND_TYPE, commandType); } public void setCommandType(NodeType nodeType) { @@ -178,7 +178,7 @@ public void setCreateTable() { } public boolean isCreateTable() { - return isCommandType(NodeType.CREATE_TABLE); + return isCommandType(NodeType.CREATE_TABLE.name()); } public void setInsert() { @@ -186,6 +186,6 @@ public void setInsert() { } public boolean isInsert() { - return isCommandType(NodeType.INSERT); + return isCommandType(NodeType.INSERT.name()); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java index 0752e11a2a..aeb4e05110 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/TupleUtil.java @@ -31,7 +31,7 @@ import org.apache.tajo.catalog.statistics.ColumnStats; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; import org.apache.tajo.storage.Tuple; @@ -215,68 +215,4 @@ public List nextBlock() { return results; } } - - /** - * Take a look at a column partition path. A partition path consists - * of a table path part and column values part. This method transforms - * a partition path into a tuple with a given partition column schema. - * - * hdfs://192.168.0.1/tajo/warehouse/table1/col1=abc/col2=def/col3=ghi - * ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^ - * table path part column values part - * - * When a file path is given, it can perform two ways depending on beNullIfFile flag. - * If it is true, it returns NULL when a given path is a file. - * Otherwise, it returns a built tuple regardless of file or directory. - * - * @param partitionColumnSchema The partition column schema - * @param partitionPath The partition path - * @param beNullIfFile If true, this method returns NULL when a given path is a file. - * @return The tuple transformed from a column values part. - */ - public static Tuple buildTupleFromPartitionPath(Schema partitionColumnSchema, Path partitionPath, - boolean beNullIfFile) { - int startIdx = partitionPath.toString().indexOf(getColumnPartitionPathPrefix(partitionColumnSchema)); - - if (startIdx == -1) { // if there is no partition column in the patch - return null; - } - String columnValuesPart = partitionPath.toString().substring(startIdx); - - String [] columnValues = columnValuesPart.split("/"); - - // true means this is a file. - if (beNullIfFile && partitionColumnSchema.size() < columnValues.length) { - return null; - } - - Tuple tuple = new VTuple(partitionColumnSchema.size()); - int i = 0; - for (; i < columnValues.length && i < partitionColumnSchema.size(); i++) { - String [] parts = columnValues[i].split("="); - if (parts.length != 2) { - return null; - } - int columnId = partitionColumnSchema.getColumnIdByName(parts[0]); - Column keyColumn = partitionColumnSchema.getColumn(columnId); - tuple.put(columnId, DatumFactory.createFromString(keyColumn.getDataType(), StringUtils.unescapePathName(parts[1]))); - } - for (; i < partitionColumnSchema.size(); i++) { - tuple.put(i, NullDatum.get()); - } - return tuple; - } - - /** - * Get a prefix of column partition path. For example, consider a column partition (col1, col2). - * Then, you will get a string 'col1='. - * - * @param partitionColumn the schema of column partition - * @return The first part string of column partition path. - */ - private static String getColumnPartitionPathPrefix(Schema partitionColumn) { - StringBuilder sb = new StringBuilder(); - sb.append(partitionColumn.getColumn(0).getSimpleName()).append("="); - return sb.toString(); - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java b/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java index 333df11a54..97872766c0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/utils/test/ErrorInjectionRewriter.java @@ -18,9 +18,9 @@ package org.apache.tajo.engine.utils.test; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.rewrite.RewriteRule; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.rewrite.RewriteRule; public class ErrorInjectionRewriter implements RewriteRule { @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 382b789207..cd3be9841e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -28,7 +28,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.service.AbstractService; import org.apache.hadoop.util.StringUtils; -import org.apache.tajo.*; +import org.apache.tajo.QueryId; +import org.apache.tajo.QueryIdFactory; +import org.apache.tajo.SessionVars; +import org.apache.tajo.TajoConstants; import org.apache.tajo.algebra.AlterTablespaceSetType; import org.apache.tajo.algebra.Expr; import org.apache.tajo.algebra.JsonHelper; @@ -37,20 +40,13 @@ import org.apache.tajo.catalog.exception.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.client.TajoClient; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.exception.IllegalQueryStatusException; -import org.apache.tajo.engine.exception.VerifyException; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.*; -import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.planner.physical.EvalExprExec; -import org.apache.tajo.engine.planner.physical.SeqScanExec; import org.apache.tajo.engine.planner.physical.StoreTableExec; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.ClientProtos; @@ -60,6 +56,14 @@ import org.apache.tajo.master.querymaster.QueryJobManager; import org.apache.tajo.master.querymaster.QueryMasterTask; import org.apache.tajo.master.session.Session; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.verifier.LogicalPlanVerifier; +import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; +import org.apache.tajo.plan.verifier.VerificationState; +import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.storage.*; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.worker.TaskAttemptContext; @@ -71,8 +75,8 @@ import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto; -import static org.apache.tajo.ipc.ClientProtos.SubmitQueryResponse; import static org.apache.tajo.ipc.ClientProtos.SerializedResultSet; +import static org.apache.tajo.ipc.ClientProtos.SubmitQueryResponse; public class GlobalEngine extends AbstractService { /** Class Logger */ @@ -267,7 +271,7 @@ private SubmitQueryResponse executeQueryInternal(QueryContext queryContext, // NonFromQuery indicates a form of 'select a, x+y;' } else if (PlannerUtil.checkIfNonFromQuery(plan)) { - Target [] targets = plan.getRootBlock().getRawTargets(); + Target[] targets = plan.getRootBlock().getRawTargets(); if (targets == null) { throw new PlanningException("No targets"); } diff --git a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java index c0bd8420ba..768528d568 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/NonForwardQueryResultScanner.java @@ -26,11 +26,10 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.engine.planner.physical.PhysicalPlanUtil; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.engine.planner.physical.SeqScanExec; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.master.session.Session; import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; import org.apache.tajo.storage.StorageManager; @@ -77,7 +76,7 @@ public void init() throws IOException { } private void initSeqScanExec() throws IOException { - FragmentProto[] fragments = PlannerUtil.getNonZeroLengthDataFiles(tajoConf, tableDesc, + FragmentProto[] fragments = PhysicalPlanUtil.getNonZeroLengthDataFiles(tajoConf, tableDesc, currentFileIndex, MAX_FILE_NUM_PER_SCAN); if (fragments != null && fragments.length > 0) { this.taskContext = new TaskAttemptContext( diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java index b8240b832c..1f445ecb97 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Query.java @@ -43,9 +43,9 @@ import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.ExecutionBlockCursor; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.planner.logical.CreateTableNode; -import org.apache.tajo.engine.planner.logical.InsertNode; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.logical.CreateTableNode; +import org.apache.tajo.plan.logical.InsertNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.event.*; import org.apache.tajo.storage.StorageManager; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java index 877a20a94c..536778aab7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryInProgress.java @@ -28,7 +28,7 @@ import org.apache.tajo.QueryId; import org.apache.tajo.TajoProtos; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.planner.logical.LogicalRootNode; +import org.apache.tajo.plan.logical.LogicalRootNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.QueryMasterProtocol; import org.apache.tajo.ipc.QueryMasterProtocol.QueryMasterProtocolService; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java index e4f47cdb6d..bcca039a85 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryJobManager.java @@ -27,7 +27,7 @@ import org.apache.tajo.QueryId; import org.apache.tajo.QueryIdFactory; import org.apache.tajo.TajoProtos; -import org.apache.tajo.engine.planner.logical.LogicalRootNode; +import org.apache.tajo.plan.logical.LogicalRootNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.ipc.TajoMasterProtocol; import org.apache.tajo.master.TajoMaster; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java index cb06df940b..8ba960010a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java @@ -36,14 +36,14 @@ import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.planner.LogicalOptimizer; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.LogicalPlanner; -import org.apache.tajo.engine.planner.PlannerUtil; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.UnimplementedException; import org.apache.tajo.ipc.TajoMasterProtocol; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java index 8a3ef7487f..fe2752f32e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryUnit.java @@ -33,13 +33,13 @@ import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.QueryUnitId; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.ipc.TajoWorkerProtocol.FailureIntermediateProto; import org.apache.tajo.ipc.TajoWorkerProtocol.IntermediateEntryProto; import org.apache.tajo.master.FragmentPair; import org.apache.tajo.master.TaskState; import org.apache.tajo.master.event.*; import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.storage.DataLocation; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.Pair; diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java index 615ebcfcb5..447c16402d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/Repartitioner.java @@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import com.google.common.primitives.Ints; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; @@ -31,28 +30,30 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.StatisticsUtil; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.engine.planner.*; +import org.apache.tajo.engine.planner.PhysicalPlannerImpl; +import org.apache.tajo.engine.planner.RangePartitionAlgorithm; +import org.apache.tajo.engine.planner.UniformRangePartition; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.utils.TupleUtil; import org.apache.tajo.exception.InternalException; -import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage; import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty; import org.apache.tajo.master.TaskSchedulerContext; import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry; -import org.apache.tajo.storage.StorageManager; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.storage.RowStoreUtil; +import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.TupleRange; import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.util.Pair; import org.apache.tajo.unit.StorageUnit; +import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.TajoIdUtils; import org.apache.tajo.worker.FetchImpl; @@ -64,7 +65,8 @@ import java.util.*; import java.util.Map.Entry; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.*; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.*; /** * Repartitioner creates non-leaf tasks and shuffles intermediate data. @@ -576,7 +578,7 @@ private static void addJoinShuffle(SubQuery subQuery, int partitionId, * @return key: pullserver's address, value: a list of requests */ private static Collection mergeShuffleRequest(int partitionId, - TajoWorkerProtocol.ShuffleType type, + ShuffleType type, List partitions) { // ebId + pullhost -> FetchImmpl Map mergedPartitions = new HashMap(); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java index a126144c50..91fd22d23c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/SubQuery.java @@ -43,14 +43,11 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; -import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.ipc.TajoMasterProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol.DistinctGroupbyEnforcer.MultipleAggregationStage; @@ -61,6 +58,8 @@ import org.apache.tajo.master.event.*; import org.apache.tajo.master.event.QueryUnitAttemptScheduleEvent.QueryUnitAttemptScheduleContext; import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.unit.StorageUnit; @@ -76,7 +75,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.apache.tajo.conf.TajoConf.ConfVars; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; /** diff --git a/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java b/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java index ad8f120861..0de1b2ba95 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/IndexUtil.java @@ -23,11 +23,11 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.logical.IndexScanNode; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.IndexScanNode; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.fragment.FileFragment; import java.util.Iterator; diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockSharedResource.java b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockSharedResource.java index e77e265c05..4b5a203061 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockSharedResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/ExecutionBlockSharedResource.java @@ -25,11 +25,11 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.engine.codegen.ExecutorPreCompiler; import org.apache.tajo.engine.codegen.TajoClassLoader; -import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.util.Pair; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java index f4117936c8..f9c752be97 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/FetchImpl.java @@ -26,18 +26,21 @@ import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.master.querymaster.QueryUnit; import org.apache.tajo.master.querymaster.Repartitioner; +import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.util.TUtil; import java.net.URI; import java.util.ArrayList; import java.util.List; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; + /** * FetchImpl information to indicate the locations of intermediate data. */ public class FetchImpl implements ProtoObject, Cloneable { private QueryUnit.PullHost host; // The pull server host information - private TajoWorkerProtocol.ShuffleType type; // hash or range partition method. + private ShuffleType type; // hash or range partition method. private ExecutionBlockId executionBlockId; // The executionBlock id private int partitionId; // The hash partition id private String name; // The intermediate source name @@ -74,13 +77,13 @@ public FetchImpl(TajoWorkerProtocol.FetchProto proto) { } } - public FetchImpl(QueryUnit.PullHost host, TajoWorkerProtocol.ShuffleType type, ExecutionBlockId executionBlockId, + public FetchImpl(QueryUnit.PullHost host, ShuffleType type, ExecutionBlockId executionBlockId, int partitionId) { this(host, type, executionBlockId, partitionId, null, false, null, new ArrayList(), new ArrayList()); } - public FetchImpl(QueryUnit.PullHost host, TajoWorkerProtocol.ShuffleType type, ExecutionBlockId executionBlockId, + public FetchImpl(QueryUnit.PullHost host, ShuffleType type, ExecutionBlockId executionBlockId, int partitionId, List intermediateEntryList) { this(host, type, executionBlockId, partitionId, null, false, null, new ArrayList(), new ArrayList()); @@ -89,7 +92,7 @@ public FetchImpl(QueryUnit.PullHost host, TajoWorkerProtocol.ShuffleType type, E } } - public FetchImpl(QueryUnit.PullHost host, TajoWorkerProtocol.ShuffleType type, ExecutionBlockId executionBlockId, + public FetchImpl(QueryUnit.PullHost host, ShuffleType type, ExecutionBlockId executionBlockId, int partitionId, String rangeParams, boolean hasNext, String name, List taskIds, List attemptIds) { this.host = host; @@ -175,11 +178,11 @@ public void setHasNext(boolean hasNext) { this.hasNext = hasNext; } - public TajoWorkerProtocol.ShuffleType getType() { + public ShuffleType getType() { return type; } - public void setType(TajoWorkerProtocol.ShuffleType type) { + public void setType(ShuffleType type) { this.type = type; } diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java index 5cf6c4619c..c11db6f9d5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoQueryEngine.java @@ -21,7 +21,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.exception.InternalException; import org.apache.tajo.storage.StorageManager; diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java index a7eaaf852d..263241527d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/Task.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/Task.java @@ -41,14 +41,14 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.engine.query.QueryUnitRequest; import org.apache.tajo.ipc.QueryMasterProtocol; import org.apache.tajo.ipc.TajoWorkerProtocol.*; import org.apache.tajo.ipc.TajoWorkerProtocol.EnforceProperty.EnforceType; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.pullserver.TajoPullServerService; import org.apache.tajo.pullserver.retriever.FileChunk; import org.apache.tajo.rpc.NullCallback; @@ -71,6 +71,7 @@ import java.util.concurrent.ExecutorService; import static org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; public class Task { private static final Log LOG = LogFactory.getLog(Task.class); diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java index 96208acdf9..99976d8e44 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskAttemptContext.java @@ -29,10 +29,10 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.eval.EvalNode; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.storage.HashShuffleAppenderManager; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; diff --git a/tajo-core/src/main/proto/TajoWorkerProtocol.proto b/tajo-core/src/main/proto/TajoWorkerProtocol.proto index 2760301448..e515438a10 100644 --- a/tajo-core/src/main/proto/TajoWorkerProtocol.proto +++ b/tajo-core/src/main/proto/TajoWorkerProtocol.proto @@ -27,6 +27,7 @@ import "tajo_protos.proto"; import "TajoIdProtos.proto"; import "CatalogProtos.proto"; import "PrimitiveProtos.proto"; +import "Plan.proto"; message SessionProto { required string session_id = 1; @@ -173,19 +174,6 @@ message GetTaskRequestProto { required ExecutionBlockIdProto executionBlockId = 3; } -enum ShuffleType { - NONE_SHUFFLE = 0; - HASH_SHUFFLE = 1; - RANGE_SHUFFLE = 2; - SCATTERED_HASH_SHUFFLE = 3; -} - -enum TransmitType { - PUSH_TRANSMIT = 0; - PULL_TRANSMIT = 1; - FILE_WRITE = 2; -} - message DataChannelProto { required ExecutionBlockIdProto srcId = 1; required ExecutionBlockIdProto targetId = 2; diff --git a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp index 6e74b99213..23d3c467e6 100644 --- a/tajo-core/src/main/resources/webapps/worker/querytasks.jsp +++ b/tajo-core/src/main/resources/webapps/worker/querytasks.jsp @@ -23,7 +23,7 @@ <%@ page import="org.apache.tajo.QueryId" %> <%@ page import="org.apache.tajo.QueryUnitAttemptId" %> <%@ page import="org.apache.tajo.catalog.statistics.TableStats" %> -<%@ page import="org.apache.tajo.engine.planner.PlannerUtil" %> +<%@ page import="org.apache.tajo.plan.util.PlannerUtil" %> <%@ page import="org.apache.tajo.ipc.TajoMasterProtocol" %> <%@ page import="org.apache.tajo.master.querymaster.*" %> <%@ page import="org.apache.tajo.util.FileUtil" %> diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 7b89587a02..a272b15ca5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -34,9 +34,12 @@ import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.master.GlobalEngine; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.verifier.LogicalPlanVerifier; +import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; +import org.apache.tajo.plan.verifier.VerificationState; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.FileUtil; import org.junit.*; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 21f4380397..b00ab32ec2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -36,12 +36,16 @@ import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.plan.EvalTreeProtoDeserializer; -import org.apache.tajo.engine.plan.EvalTreeProtoSerializer; -import org.apache.tajo.engine.plan.proto.PlanProto; -import org.apache.tajo.engine.planner.*; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.serder.EvalTreeProtoDeserializer; +import org.apache.tajo.plan.serder.EvalTreeProtoSerializer; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.serder.PlanProto; +import org.apache.tajo.plan.verifier.LogicalPlanVerifier; +import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; +import org.apache.tajo.plan.verifier.VerificationState; import org.apache.tajo.storage.LazyTuple; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java index beeb1c67e4..4c1efd6407 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java @@ -24,6 +24,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.json.CoreGsonHelper; +import org.apache.tajo.plan.expr.*; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.junit.Test; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index 222b8e1c62..dcacff28c1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -32,19 +32,19 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.function.FunctionLoader; -import org.apache.tajo.engine.function.GeneralFunction; -import org.apache.tajo.engine.optimizer.eval.EvalTreeOptimizer; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.LogicalPlanner; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.Target; -import org.apache.tajo.engine.planner.logical.GroupbyNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.InternalException; -import org.apache.tajo.master.TajoMaster; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.exprrewrite.EvalTreeOptimizer; +import org.apache.tajo.plan.function.GeneralFunction; +import org.apache.tajo.plan.logical.GroupbyNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.nameresolver.NameResolvingMode; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.CommonTestingUtil; import org.junit.AfterClass; @@ -58,9 +58,7 @@ import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; import static org.apache.tajo.common.TajoDataTypes.Type.INT4; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class TestEvalTreeUtil { static TajoTestingCluster util; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestAggFunction.java b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestAggFunction.java index f635468f23..aa0d360330 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/function/TestAggFunction.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/function/TestAggFunction.java @@ -18,6 +18,7 @@ package org.apache.tajo.engine.function; +import org.apache.tajo.plan.function.FunctionContext; import org.junit.Test; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java index 0e4a9cefc6..87bc7579da 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java @@ -29,9 +29,12 @@ import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.function.builtin.SumInt; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.master.TajoMaster; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; import org.junit.AfterClass; @@ -128,7 +131,7 @@ public final void testProjectionPushWithNaturalJoin() throws PlanningException, LogicalNode plan = newPlan.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; - TestLogicalNode.testCloneLogicalNode(root); + TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.PROJECTION, root.getChild().getType()); ProjectionNode projNode = root.getChild(); assertEquals(NodeType.JOIN, projNode.getChild().getType()); @@ -140,7 +143,7 @@ public final void testProjectionPushWithNaturalJoin() throws PlanningException, assertEquals(NodeType.ROOT, optimized.getType()); root = (LogicalRootNode) optimized; - TestLogicalNode.testCloneLogicalNode(root); + TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.JOIN, root.getChild().getType()); joinNode = root.getChild(); assertEquals(NodeType.SCAN, joinNode.getLeftChild().getType()); @@ -164,7 +167,7 @@ public final void testProjectionPush() throws CloneNotSupportedException, Planni assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; - TestLogicalNode.testCloneLogicalNode(root); + TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.PROJECTION, root.getChild().getType()); ProjectionNode projNode = root.getChild(); assertEquals(NodeType.SELECTION, projNode.getChild().getType()); @@ -174,7 +177,7 @@ public final void testProjectionPush() throws CloneNotSupportedException, Planni LogicalNode optimized = optimizer.optimize(newPlan); assertEquals(NodeType.ROOT, optimized.getType()); root = (LogicalRootNode) optimized; - TestLogicalNode.testCloneLogicalNode(root); + TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.SCAN, root.getChild().getType()); } @@ -186,7 +189,7 @@ public final void testOptimizeWithGroupBy() throws CloneNotSupportedException, P assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; - TestLogicalNode.testCloneLogicalNode(root); + TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.PROJECTION, root.getChild().getType()); ProjectionNode projNode = root.getChild(); assertEquals(NodeType.GROUP_BY, projNode.getChild().getType()); @@ -198,7 +201,7 @@ public final void testOptimizeWithGroupBy() throws CloneNotSupportedException, P LogicalNode optimized = optimizer.optimize(newPlan); assertEquals(NodeType.ROOT, optimized.getType()); root = (LogicalRootNode) optimized; - TestLogicalNode.testCloneLogicalNode(root); + TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.GROUP_BY, root.getChild().getType()); groupbyNode = root.getChild(); assertEquals(NodeType.SCAN, groupbyNode.getChild().getType()); @@ -213,7 +216,7 @@ public final void testPushable() throws CloneNotSupportedException, PlanningExce assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; - TestLogicalNode.testCloneLogicalNode(root); + TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.PROJECTION, root.getChild().getType()); ProjectionNode projNode = root.getChild(); @@ -244,7 +247,7 @@ public final void testPushable() throws CloneNotSupportedException, PlanningExce assertEquals(NodeType.ROOT, plan.getType()); root = (LogicalRootNode) plan; - TestLogicalNode.testCloneLogicalNode(root); + TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.PROJECTION, root.getChild().getType()); projNode = root.getChild(); @@ -264,4 +267,5 @@ public final void testInsertInto() throws CloneNotSupportedException, PlanningEx LogicalPlan newPlan = planner.createPlan(defaultContext, expr); optimizer.optimize(newPlan); } + } diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java index 1f8fc352c6..cee15931aa 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlan.java @@ -19,23 +19,16 @@ package org.apache.tajo.engine.planner; import org.apache.tajo.TajoTestingCluster; -import org.apache.tajo.benchmark.TPCH; -import org.apache.tajo.catalog.*; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph; -import org.apache.tajo.master.TajoMaster; -import org.apache.tajo.util.CommonTestingUtil; +import org.apache.tajo.util.graph.SimpleDirectedGraph; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; import org.junit.BeforeClass; import org.junit.Test; import java.util.HashSet; import java.util.Set; -import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; -import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; -import static org.apache.tajo.engine.planner.LogicalPlan.BlockType; +import static org.apache.tajo.plan.LogicalPlan.BlockType; import static org.junit.Assert.*; public class TestLogicalPlan { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 57e0e500d0..e4e7916195 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -32,15 +32,16 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.function.builtin.SumInt; import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.KeyValueSet; @@ -154,6 +155,11 @@ public static void tearDown() throws Exception { "select length(name), length(deptname), *, empid+10 from employee where empId > 500", // 13 }; + public static final void testCloneLogicalNode(LogicalNode n1) throws CloneNotSupportedException { + LogicalNode copy = (LogicalNode) n1.clone(); + assertTrue(n1.deepEquals(copy)); + } + @Test public final void testSingleRelation() throws CloneNotSupportedException, PlanningException { QueryContext qc = new QueryContext(util.getConfiguration(), session); @@ -162,7 +168,7 @@ public final void testSingleRelation() throws CloneNotSupportedException, Planni LogicalPlan planNode = planner.createPlan(qc, expr); LogicalNode plan = planNode.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); - TestLogicalNode.testCloneLogicalNode(plan); + testCloneLogicalNode(plan); LogicalRootNode root = (LogicalRootNode) plan; testJsonSerDerObject(root); @@ -200,7 +206,7 @@ public final void testImplicityJoinPlan() throws CloneNotSupportedException, Pla assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; testJsonSerDerObject(root); - TestLogicalNode.testCloneLogicalNode(root); + testCloneLogicalNode(root); Schema expectedSchema = new Schema(); expectedSchema.addColumn("name", Type.TEXT); @@ -229,7 +235,7 @@ public final void testImplicityJoinPlan() throws CloneNotSupportedException, Pla expr = sqlAnalyzer.parse(QUERIES[2]); plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); - TestLogicalNode.testCloneLogicalNode(plan); + testCloneLogicalNode(plan); expectedSchema.addColumn("score", Type.INT4); assertSchema(expectedSchema, plan.getOutSchema()); @@ -393,7 +399,7 @@ public final void testGroupby() throws CloneNotSupportedException, PlanningExcep // with having clause context = sqlAnalyzer.parse(QUERIES[3]); plan = planner.createPlan(qc, context).getRootBlock().getRoot(); - TestLogicalNode.testCloneLogicalNode(plan); + testCloneLogicalNode(plan); assertEquals(NodeType.ROOT, plan.getType()); root = (LogicalRootNode) plan; @@ -705,7 +711,7 @@ public final void testStoreTable() throws CloneNotSupportedException, PlanningEx Expr context = sqlAnalyzer.parse(QUERIES[8]); LogicalNode plan = planner.createPlan(qc, context).getRootBlock().getRoot(); - TestLogicalNode.testCloneLogicalNode(plan); + testCloneLogicalNode(plan); testJsonSerDerObject(plan); assertEquals(NodeType.ROOT, plan.getType()); @@ -724,7 +730,7 @@ public final void testOrderBy() throws CloneNotSupportedException, PlanningExcep LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); - TestLogicalNode.testCloneLogicalNode(plan); + testCloneLogicalNode(plan); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; @@ -754,7 +760,7 @@ public final void testLimit() throws CloneNotSupportedException, PlanningExcepti LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); - TestLogicalNode.testCloneLogicalNode(plan); + testCloneLogicalNode(plan); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; @@ -775,7 +781,7 @@ public final void testSPJPush() throws CloneNotSupportedException, PlanningExcep Expr expr = sqlAnalyzer.parse(QUERIES[5]); LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); - TestLogicalNode.testCloneLogicalNode(plan); + testCloneLogicalNode(plan); assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; @@ -797,7 +803,7 @@ public final void testSPJ() throws CloneNotSupportedException, PlanningException Expr expr = sqlAnalyzer.parse(QUERIES[6]); LogicalNode plan = planner.createPlan(qc, expr).getRootBlock().getRoot(); testJsonSerDerObject(plan); - TestLogicalNode.testCloneLogicalNode(plan); + testCloneLogicalNode(plan); } @Test @@ -879,7 +885,7 @@ public final void testAsterisk() throws CloneNotSupportedException, PlanningExce LogicalPlan planNode = planner.createPlan(qc, expr); LogicalNode plan = planNode.getRootBlock().getRoot(); assertEquals(NodeType.ROOT, plan.getType()); - TestLogicalNode.testCloneLogicalNode(plan); + testCloneLogicalNode(plan); LogicalRootNode root = (LogicalRootNode) plan; testJsonSerDerObject(root); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index 756dadc41f..90ec9b1788 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -18,8 +18,10 @@ package org.apache.tajo.engine.planner; -import org.apache.commons.configuration.Configuration; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.tajo.LocalTajoTestingUtility; import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; @@ -30,10 +32,14 @@ import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.eval.*; import org.apache.tajo.engine.function.builtin.SumInt; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.engine.planner.physical.PhysicalPlanUtil; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.VTuple; @@ -46,9 +52,7 @@ import org.junit.Test; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; @@ -122,7 +126,7 @@ public final void testFindTopNode() throws CloneNotSupportedException, PlanningE assertEquals(NodeType.ROOT, plan.getType()); LogicalRootNode root = (LogicalRootNode) plan; - TestLogicalNode.testCloneLogicalNode(root); + TestLogicalPlanner.testCloneLogicalNode(root); assertEquals(NodeType.PROJECTION, root.getChild().getType()); ProjectionNode projNode = root.getChild(); @@ -295,7 +299,7 @@ public final void testComparatorsFromJoinQual() { FieldEval f4 = new FieldEval("people.fid2", CatalogUtil.newSimpleDataType(Type.INT4)); EvalNode joinQual = new BinaryEval(EvalType.EQUAL, f1, f2); - TupleComparator [] comparators = PlannerUtil.getComparatorsFromJoinQual(joinQual, outerSchema, innerSchema); + TupleComparator [] comparators = PhysicalPlanUtil.getComparatorsFromJoinQual(joinQual, outerSchema, innerSchema); Tuple t1 = new VTuple(2); t1.put(0, DatumFactory.createInt4(1)); @@ -316,7 +320,7 @@ public final void testComparatorsFromJoinQual() { // tests for composited join key EvalNode joinQual2 = new BinaryEval(EvalType.EQUAL, f3, f4); EvalNode compositedJoinQual = new BinaryEval(EvalType.AND, joinQual, joinQual2); - comparators = PlannerUtil.getComparatorsFromJoinQual(compositedJoinQual, outerSchema, innerSchema); + comparators = PhysicalPlanUtil.getComparatorsFromJoinQual(compositedJoinQual, outerSchema, innerSchema); outerComparator = comparators[0]; assertTrue(outerComparator.compare(t1, t2) < 0); @@ -353,7 +357,7 @@ public void testGetNonZeroLengthDataFiles() throws Exception { int start = i * fileNum; FragmentProto[] fragments = - PlannerUtil.getNonZeroLengthDataFiles(util.getConfiguration(), tableDesc, start, fileNum); + PhysicalPlanUtil.getNonZeroLengthDataFiles(util.getConfiguration(), tableDesc, start, fileNum); assertNotNull(fragments); numResultFiles += fragments.length; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java index e17b325aae..71f3f8df3e 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestQueryValidation.java @@ -19,6 +19,7 @@ package org.apache.tajo.engine.planner; import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.plan.PlanningException; import org.junit.Test; import java.io.IOException; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java index 22944247bc..b04438461a 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestUniformRangePartition.java @@ -22,6 +22,7 @@ import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; import org.apache.tajo.storage.TupleRange; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java index 8810692be1..0e7f3e69cd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestBroadcastJoinPlan.java @@ -36,13 +36,12 @@ import org.apache.tajo.datum.TextDatum; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.LogicalOptimizer; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.LogicalPlanner; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.PlanningException; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.master.TajoMaster; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.StorageManager; import org.apache.tajo.storage.Tuple; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestMasterPlan.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestMasterPlan.java index ab56bea114..817d27ae51 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestMasterPlan.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/global/TestMasterPlan.java @@ -19,9 +19,9 @@ package org.apache.tajo.engine.planner.global; import org.apache.tajo.LocalTajoTestingUtility; -import org.apache.tajo.ipc.TajoWorkerProtocol; import org.junit.Test; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -35,11 +35,11 @@ public void testConnect() { ExecutionBlock eb2 = masterPlan.newExecutionBlock(); ExecutionBlock eb3 = masterPlan.newExecutionBlock(); - masterPlan.addConnect(eb1, eb2, TajoWorkerProtocol.ShuffleType.RANGE_SHUFFLE); + masterPlan.addConnect(eb1, eb2, ShuffleType.RANGE_SHUFFLE); assertTrue(masterPlan.isConnected(eb1.getId(), eb2.getId())); assertTrue(masterPlan.isReverseConnected(eb2.getId(), eb1.getId())); - masterPlan.addConnect(eb3, eb2, TajoWorkerProtocol.ShuffleType.RANGE_SHUFFLE); + masterPlan.addConnect(eb3, eb2, ShuffleType.RANGE_SHUFFLE); assertTrue(masterPlan.isConnected(eb1.getId(), eb2.getId())); assertTrue(masterPlan.isConnected(eb3.getId(), eb2.getId())); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java index 197ff653e1..df7f897fa3 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBNLJoinExec.java @@ -31,9 +31,12 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java index 00a44165cb..868a82eda7 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestBSTIndexExec.java @@ -32,12 +32,12 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.LogicalOptimizer; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java index 6572506b25..46f55c6864 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestExternalSortExec.java @@ -32,7 +32,10 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index aba5169585..104ca52be4 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -31,9 +31,12 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 7998c61bbf..23c0fb4ce6 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -32,9 +32,12 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java index 77f524d30d..e94176724c 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashAntiJoinExec.java @@ -31,7 +31,11 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java index eff647d9f1..e25ca8f1c1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashJoinExec.java @@ -33,9 +33,12 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java index 148592dea2..4998012dce 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestHashSemiJoinExec.java @@ -31,7 +31,11 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index 3e404859db..f20984a8a8 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -31,9 +31,12 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java index 9ee51c3d5b..4b453fe0fe 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterNLJoinExec.java @@ -29,12 +29,12 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.PlanningException; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java index ddad0a21bf..7a8426f9d1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestMergeJoinExec.java @@ -33,9 +33,13 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java index 20053cbf30..fc67a38db1 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestNLJoinExec.java @@ -29,13 +29,13 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.PlanningException; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 30c2cd5e38..341ce9e1cf 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -36,18 +36,19 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.eval.AggregationFunctionCallEval; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.*; +import org.apache.tajo.engine.planner.PhysicalPlanner; +import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.planner.logical.*; import org.apache.tajo.engine.query.QueryContext; -import org.apache.tajo.ipc.TajoWorkerProtocol; -import org.apache.tajo.master.TajoMaster; import org.apache.tajo.master.session.Session; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.expr.AggregationFunctionCallEval; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; import org.apache.tajo.storage.fragment.FileFragment; @@ -73,8 +74,8 @@ import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; import static org.apache.tajo.ipc.TajoWorkerProtocol.ColumnPartitionEnforcer.ColumnPartitionAlgorithm; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType; import static org.apache.tajo.ipc.TajoWorkerProtocol.SortEnforce.SortAlgorithm; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; import static org.junit.Assert.*; public class TestPhysicalPlanner { @@ -1056,7 +1057,7 @@ public final void testIndexedStoreExec() throws IOException, PlanningException { SortNode sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT); DataChannel channel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(), - TajoWorkerProtocol.ShuffleType.RANGE_SHUFFLE); + ShuffleType.RANGE_SHUFFLE); channel.setShuffleKeys(PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()).toArray()); ctx.setDataChannel(channel); diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index c8b9e4ab04..a02f2ce6d2 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -32,12 +32,12 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 09f5ab8dec..df1732c412 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -29,12 +29,12 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlanner; import org.apache.tajo.engine.planner.PhysicalPlannerImpl; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.PlanningException; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index f8e2f41b03..77e4e4ecdd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -31,9 +31,12 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 63e189987a..3ed5997a8f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -33,8 +33,10 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java index b95ae41ee4..222b508897 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestJoinBroadcast.java @@ -30,7 +30,7 @@ import org.apache.tajo.datum.TextDatum; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.jdbc.TajoResultSet; import org.apache.tajo.master.querymaster.QueryMasterTask; import org.apache.tajo.storage.*; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index da09129313..828d2a3a58 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -25,7 +25,10 @@ import org.apache.hadoop.io.compress.CompressionCodec; import org.apache.hadoop.io.compress.CompressionCodecFactory; import org.apache.hadoop.io.compress.DeflateCodec; -import org.apache.tajo.*; +import org.apache.tajo.QueryId; +import org.apache.tajo.QueryTestCaseBase; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; @@ -35,16 +38,12 @@ import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.MasterPlan; -import org.apache.tajo.engine.planner.logical.NodeType; import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.jdbc.TajoResultSet; -import org.apache.tajo.master.querymaster.Query; import org.apache.tajo.master.querymaster.QueryMasterTask; -import org.apache.tajo.master.querymaster.QueryUnit; -import org.apache.tajo.master.querymaster.SubQuery; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.TajoWorker; import org.junit.Test; @@ -56,9 +55,8 @@ import java.util.Random; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.SCATTERED_HASH_SHUFFLE; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.SCATTERED_HASH_SHUFFLE; import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; public class TestTablePartitions extends QueryTestCaseBase { diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/util/TestTupleUtil.java b/tajo-core/src/test/java/org/apache/tajo/engine/util/TestTupleUtil.java index cecb2818ee..62c959c00d 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/util/TestTupleUtil.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/util/TestTupleUtil.java @@ -24,10 +24,10 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.planner.PlannerUtil; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.engine.planner.RangePartitionAlgorithm; import org.apache.tajo.engine.planner.UniformRangePartition; -import org.apache.tajo.engine.utils.TupleUtil; +import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; import org.apache.tajo.storage.*; import org.apache.tajo.storage.RowStoreUtil.RowStoreDecoder; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; @@ -129,41 +129,41 @@ public void testBuildTupleFromPartitionPath() { schema.addColumn("key2", Type.TEXT); Path path = new Path("hdfs://tajo/warehouse/partition_test/"); - Tuple tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, true); + Tuple tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, true); assertNull(tuple); - tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, false); + tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, false); assertNull(tuple); path = new Path("hdfs://tajo/warehouse/partition_test/key1=123"); - tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, true); + tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, true); assertNotNull(tuple); assertEquals(DatumFactory.createInt8(123), tuple.get(0)); - tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, false); + tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, false); assertNotNull(tuple); assertEquals(DatumFactory.createInt8(123), tuple.get(0)); path = new Path("hdfs://tajo/warehouse/partition_test/key1=123/part-0000"); // wrong cases; - tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, true); + tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, true); assertNull(tuple); - tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, false); + tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, false); assertNull(tuple); path = new Path("hdfs://tajo/warehouse/partition_test/key1=123/key2=abc"); - tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, true); + tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, true); assertNotNull(tuple); assertEquals(DatumFactory.createInt8(123), tuple.get(0)); assertEquals(DatumFactory.createText("abc"), tuple.get(1)); - tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, false); + tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, false); assertNotNull(tuple); assertEquals(DatumFactory.createInt8(123), tuple.get(0)); assertEquals(DatumFactory.createText("abc"), tuple.get(1)); path = new Path("hdfs://tajo/warehouse/partition_test/key1=123/key2=abc/part-0001"); - tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, true); + tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, true); assertNull(tuple); - tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, false); + tuple = PartitionedTableRewriter.buildTupleFromPartitionPath(schema, path, false); assertNotNull(tuple); assertEquals(DatumFactory.createInt8(123), tuple.get(0)); assertEquals(DatumFactory.createText("abc"), tuple.get(1)); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 234c58e572..067c6c8e86 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -27,9 +27,9 @@ import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.LogicalOptimizer; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.engine.planner.global.ExecutionBlockCursor; import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java index 167c1f6e1c..c908737945 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestGlobalPlanner.java @@ -29,22 +29,22 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.engine.eval.BinaryEval; -import org.apache.tajo.engine.eval.EvalType; -import org.apache.tajo.engine.eval.FieldEval; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.global.DataChannel; import org.apache.tajo.engine.planner.global.ExecutionBlock; import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.expr.BinaryEval; +import org.apache.tajo.plan.expr.EvalType; +import org.apache.tajo.plan.expr.FieldEval; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.TUtil; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; diff --git a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java index afa330e9a7..3c967703a5 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/TestRepartitioner.java @@ -28,6 +28,7 @@ import org.apache.tajo.master.querymaster.QueryUnit; import org.apache.tajo.master.querymaster.QueryUnit.IntermediateEntry; import org.apache.tajo.master.querymaster.Repartitioner; +import org.apache.tajo.plan.serder.PlanProto; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import org.apache.tajo.worker.FetchImpl; @@ -38,8 +39,10 @@ import java.util.*; import static junit.framework.Assert.assertEquals; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType; import static org.apache.tajo.master.querymaster.Repartitioner.FetchGroupMeta; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.HASH_SHUFFLE; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType.SCATTERED_HASH_SHUFFLE; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; @@ -68,7 +71,7 @@ public void testCreateHashFetchURL() throws Exception { new HashMap>>(); for (Map.Entry> eachEntry: intermediateEntries.entrySet()) { - FetchImpl fetch = new FetchImpl(new QueryUnit.PullHost(hostName, port), TajoWorkerProtocol.ShuffleType.HASH_SHUFFLE, + FetchImpl fetch = new FetchImpl(new QueryUnit.PullHost(hostName, port), ShuffleType.HASH_SHUFFLE, sid, eachEntry.getKey(), eachEntry.getValue()); fetch.setName(sid.toString()); @@ -117,7 +120,7 @@ public void testScheduleFetchesByEvenDistributedVolumes() { ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); FetchImpl [] fetches = new FetchImpl[12]; for (int i = 0; i < 12; i++) { - fetches[i] = new FetchImpl(new QueryUnit.PullHost("localhost", 10000 + i), ShuffleType.HASH_SHUFFLE, ebId, i / 2); + fetches[i] = new FetchImpl(new QueryUnit.PullHost("localhost", 10000 + i), HASH_SHUFFLE, ebId, i / 2); } int [] VOLUMES = {100, 80, 70, 30, 10, 5}; @@ -480,8 +483,8 @@ public void testFetchImpl() { ExecutionBlockId ebId = new ExecutionBlockId(LocalTajoTestingUtility.newQueryId(), 0); QueryUnit.PullHost pullHost = new QueryUnit.PullHost("localhost", 0); - FetchImpl expected = new FetchImpl(pullHost, ShuffleType.SCATTERED_HASH_SHUFFLE, ebId, 1); - FetchImpl fetch2 = new FetchImpl(pullHost, ShuffleType.SCATTERED_HASH_SHUFFLE, ebId, 1); + FetchImpl expected = new FetchImpl(pullHost, SCATTERED_HASH_SHUFFLE, ebId, 1); + FetchImpl fetch2 = new FetchImpl(pullHost, SCATTERED_HASH_SHUFFLE, ebId, 1); assertEquals(expected, fetch2); fetch2.setOffset(5); fetch2.setLength(10); diff --git a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java index 9eebfcdf68..37ee402b79 100644 --- a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestKillQuery.java @@ -23,9 +23,9 @@ import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.parser.SQLAnalyzer; -import org.apache.tajo.engine.planner.LogicalOptimizer; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.engine.planner.global.GlobalPlanner; import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java index aff5d73fad..124976a944 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestRangeRetrieverHandler.java @@ -36,7 +36,11 @@ import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.planner.*; import org.apache.tajo.engine.planner.enforce.Enforcer; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.engine.planner.physical.ExternalSortExec; import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.engine.planner.physical.ProjectionExec; diff --git a/tajo-plan/pom.xml b/tajo-plan/pom.xml new file mode 100644 index 0000000000..6a52e9bc9b --- /dev/null +++ b/tajo-plan/pom.xml @@ -0,0 +1,288 @@ + + + + + 4.0.0 + + tajo-project + org.apache.tajo + 0.9.1-SNAPSHOT + ../tajo-project + + tajo-plan + jar + Tajo Plan + This module contains logical plan and query optimization parts. + + UTF-8 + UTF-8 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + ${project.build.sourceEncoding} + + + + org.apache.rat + apache-rat-plugin + + + verify + + check + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + create-protobuf-generated-sources-directory + initialize + + + + + + + run + + + + + + org.codehaus.mojo + exec-maven-plugin + 1.2 + + + generate-sources + generate-sources + + protoc + + -Isrc/main/proto/ + --proto_path=../tajo-common/src/main/proto + --proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto + --proto_path=../tajo-client/src/main/proto + --java_out=target/generated-sources/proto + src/main/proto/Plan.proto + + + + exec + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.5 + + + add-source + generate-sources + + add-source + + + + target/generated-sources/proto + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.2 + + + package + + test-jar + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-dependencies + package + + copy-dependencies + + + runtime + ${project.build.directory}/lib + false + false + true + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + + + + + + + + org.apache.tajo + tajo-common + + + org.apache.tajo + tajo-algebra + + + org.apache.tajo + tajo-catalog-common + + + org.apache.hadoop + hadoop-common + + + junit + junit + test + + + org.mockito + mockito-core + test + + + + + + docs + + false + + + + + org.apache.maven.plugins + maven-javadoc-plugin + + + + module-javadocs + package + + jar + + + ${project.build.directory} + + + + + + + + + dist + + false + + tar|rpm|deb + + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + dist + package + + run + + + + + run() { + echo "\$ ${@}" + "${@}" + res=$? + if [ $res != 0 ]; then + echo + echo "Failed!" + echo + exit $res + fi + } + + ROOT=`cd ${basedir}/..;pwd` + echo + echo "Current directory `pwd`" + echo + run rm -rf ${project.artifactId}-${project.version} + run mkdir ${project.artifactId}-${project.version} + run cd ${project.artifactId}-${project.version} + run cp -r ${basedir}/target/${project.artifactId}-${project.version}*.jar . + echo + echo "Tajo Algebra dist layout available at: ${project.build.directory}/${project.artifactId}-${project.version}" + echo + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-surefire-report-plugin + 2.15 + + + + + diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java index 3115104a9b..406cdfc535 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprAnnotator.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; import com.google.common.collect.Sets; import org.apache.tajo.algebra.*; @@ -27,14 +27,15 @@ import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.GeneralFunction; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; -import org.apache.tajo.engine.planner.nameresolver.NameResolver; import org.apache.tajo.exception.InternalException; import org.apache.tajo.exception.InvalidOperationException; +import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.GeneralFunction; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.nameresolver.NameResolver; +import org.apache.tajo.plan.nameresolver.NameResolvingMode; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.datetime.DateTimeUtil; @@ -48,9 +49,7 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.FunctionType; import static org.apache.tajo.common.TajoDataTypes.DataType; import static org.apache.tajo.common.TajoDataTypes.Type; -import static org.apache.tajo.engine.planner.logical.WindowSpec.WindowEndBound; -import static org.apache.tajo.engine.planner.logical.WindowSpec.WindowFrame; -import static org.apache.tajo.engine.planner.logical.WindowSpec.WindowStartBound; +import static org.apache.tajo.plan.logical.WindowSpec.*; /** * ExprAnnotator makes an annotated expression called EvalNode from an diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java index 37180560c1..2a6cb0bbe8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprNormalizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java @@ -16,14 +16,15 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; import com.google.common.collect.Sets; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.engine.exception.NoSuchColumnException; -import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; -import org.apache.tajo.engine.planner.nameresolver.NameResolver; +import org.apache.tajo.catalog.exception.NoSuchColumnException; +import org.apache.tajo.plan.nameresolver.NameResolver; +import org.apache.tajo.plan.nameresolver.NameResolvingMode; +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import java.util.ArrayList; import java.util.List; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/exception/IllegalQueryStatusException.java b/tajo-plan/src/main/java/org/apache/tajo/plan/IllegalQueryStatusException.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/exception/IllegalQueryStatusException.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/IllegalQueryStatusException.java index 66e9ee7745..31f9186132 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/exception/IllegalQueryStatusException.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/IllegalQueryStatusException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.exception; +package org.apache.tajo.plan; public class IllegalQueryStatusException extends Exception { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/exception/InvalidQueryException.java b/tajo-plan/src/main/java/org/apache/tajo/plan/InvalidQueryException.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/exception/InvalidQueryException.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/InvalidQueryException.java index fba691d891..db986e1dd4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/exception/InvalidQueryException.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/InvalidQueryException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.exception; +package org.apache.tajo.plan; public class InvalidQueryException extends RuntimeException { private static final long serialVersionUID = -7085849718839416246L; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java similarity index 91% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index a43cc1acd0..bcc3bce39c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -16,34 +16,39 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.tajo.OverridableConf; import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.engine.eval.AlgebraicUtil; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.graph.DirectedGraphCursor; -import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.planner.logical.join.FoundJoinOrder; -import org.apache.tajo.engine.planner.logical.join.GreedyHeuristicJoinOrderAlgorithm; -import org.apache.tajo.engine.planner.logical.join.JoinGraph; -import org.apache.tajo.engine.planner.logical.join.JoinOrderAlgorithm; -import org.apache.tajo.engine.planner.rewrite.*; -import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.util.graph.DirectedGraphCursor; +import org.apache.tajo.plan.expr.AlgebraicUtil; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.joinorder.FoundJoinOrder; +import org.apache.tajo.plan.joinorder.GreedyHeuristicJoinOrderAlgorithm; +import org.apache.tajo.plan.joinorder.JoinGraph; +import org.apache.tajo.plan.joinorder.JoinOrderAlgorithm; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.*; +import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule; +import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; +import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import java.util.LinkedHashSet; import java.util.Set; import java.util.Stack; -import static org.apache.tajo.engine.planner.LogicalPlan.BlockEdge; -import static org.apache.tajo.engine.planner.logical.join.GreedyHeuristicJoinOrderAlgorithm.getCost; +import static org.apache.tajo.plan.LogicalPlan.BlockEdge; +import static org.apache.tajo.plan.joinorder.GreedyHeuristicJoinOrderAlgorithm.getCost; /** * This class optimizes a logical plan. @@ -85,7 +90,7 @@ public LogicalNode optimize(LogicalPlan plan) throws PlanningException { return optimize(null, plan); } - public LogicalNode optimize(QueryContext context, LogicalPlan plan) throws PlanningException { + public LogicalNode optimize(OverridableConf context, LogicalPlan plan) throws PlanningException { rulesBeforeJoinOpt.rewrite(plan); DirectedGraphCursor blockCursor = diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java index 1cbe87acd7..3baf61db81 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlan.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlan.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -25,16 +25,17 @@ import org.apache.tajo.annotation.NotThreadSafe; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.eval.ConstEval; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.graph.DirectedGraphCursor; -import org.apache.tajo.engine.planner.graph.SimpleDirectedGraph; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.LogicalRootNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.RelationNode; -import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; -import org.apache.tajo.engine.planner.nameresolver.NameResolver; +import org.apache.tajo.util.graph.DirectedGraphCursor; +import org.apache.tajo.util.graph.SimpleDirectedGraph; +import org.apache.tajo.plan.expr.ConstEval; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalRootNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.RelationNode; +import org.apache.tajo.plan.nameresolver.NameResolver; +import org.apache.tajo.plan.nameresolver.NameResolvingMode; +import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor; import org.apache.tajo.util.TUtil; import java.lang.reflect.Constructor; @@ -430,7 +431,7 @@ public class QueryBlock { /** It contains a planning log for this block */ private final List planingHistory = Lists.newArrayList(); /** It is for debugging or unit tests */ - private Target [] rawTargets; + private Target[] rawTargets; public QueryBlock(String blockName) { this.blockName = blockName; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java similarity index 94% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java index 93389798ff..6bdfb9dc6e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanPreprocessor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanPreprocessor.java @@ -16,20 +16,24 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; +import org.apache.tajo.SessionVars; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.exception.NoSuchColumnException; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.engine.eval.ConstEval; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.FieldEval; -import org.apache.tajo.engine.exception.NoSuchColumnException; -import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; -import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.planner.nameresolver.NameResolver; -import org.apache.tajo.engine.planner.nameresolver.NameResolvingMode; -import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.plan.LogicalPlan.QueryBlock; +import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; +import org.apache.tajo.plan.expr.ConstEval; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.FieldEval; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.nameresolver.NameResolver; +import org.apache.tajo.plan.nameresolver.NameResolvingMode; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import org.apache.tajo.util.TUtil; import java.util.*; @@ -87,7 +91,8 @@ public static Column[] getColumns(LogicalPlanner.PlanContext ctx, QualifiedAster if (CatalogUtil.isFQTableName(asteriskExpr.getQualifier())) { qualifier = asteriskExpr.getQualifier(); } else { - qualifier = CatalogUtil.buildFQName(ctx.queryContext.getCurrentDatabase(), asteriskExpr.getQualifier()); + qualifier = CatalogUtil.buildFQName( + ctx.queryContext.get(SessionVars.CURRENT_DATABASE), asteriskExpr.getQualifier()); } relationOp = block.getRelation(qualifier); @@ -207,7 +212,7 @@ public LogicalNode visitProjection(LogicalPlanner.PlanContext ctx, Stack s } } - Target [] targets = buildTargets(ctx, expr.getNamedExprs()); + Target[] targets = buildTargets(ctx, expr.getNamedExprs()); stack.pop(); // <--- Pop @@ -366,7 +371,8 @@ public LogicalNode visitRelation(LogicalPlanner.PlanContext ctx, Stack sta if (CatalogUtil.isFQTableName(expr.getName())) { actualRelationName = relation.getName(); } else { - actualRelationName = CatalogUtil.buildFQName(ctx.queryContext.getCurrentDatabase(), relation.getName()); + actualRelationName = + CatalogUtil.buildFQName(ctx.queryContext.get(SessionVars.CURRENT_DATABASE), relation.getName()); } TableDesc desc = catalog.getTableDesc(actualRelationName); @@ -395,7 +401,7 @@ public LogicalNode visitTableSubQuery(LogicalPlanner.PlanContext ctx, Stack stack, Proje //////////////////////////////////////////////////////// ProjectionNode projectionNode; - Target [] targets; + Target[] targets; targets = buildTargets(context, referenceNames); // Set ProjectionNode @@ -301,7 +304,7 @@ private void insertDistinctOperator(PlanContext context, ProjectionNode projecti int finalTargetNum = projection.size(); String [] referenceNames = new String[finalTargetNum]; - ExprNormalizedResult [] normalizedExprList = new ExprNormalizedResult[finalTargetNum]; + ExprNormalizedResult[] normalizedExprList = new ExprNormalizedResult[finalTargetNum]; List windowSpecReferencesList = TUtil.newList(); @@ -1459,7 +1462,7 @@ private InsertNode buildInsertIntoTablePlan(PlanContext context, InsertNode inse databaseName = CatalogUtil.extractQualifier(expr.getTableName()); tableName = CatalogUtil.extractSimpleName(expr.getTableName()); } else { - databaseName = context.queryContext.getCurrentDatabase(); + databaseName = context.queryContext.get(SessionVars.CURRENT_DATABASE); tableName = expr.getTableName(); } TableDesc desc = catalog.getTableDesc(databaseName, tableName); @@ -1658,9 +1661,8 @@ public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, String parentTableName = expr.getLikeParentTableName(); if (CatalogUtil.isFQTableName(parentTableName) == false) { - parentTableName = - CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), - parentTableName); + parentTableName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), + parentTableName); } TableDesc parentTableDesc = catalog.getTableDesc(parentTableName); if(parentTableDesc == null) @@ -1679,7 +1681,6 @@ public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, return createTableNode; } - @Override public LogicalNode visitCreateTable(PlanContext context, Stack stack, CreateTable expr) throws PlanningException { @@ -1692,7 +1693,7 @@ public LogicalNode visitCreateTable(PlanContext context, Stack stack, Crea createTableNode.setTableName(expr.getTableName()); } else { createTableNode.setTableName( - CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), expr.getTableName())); + CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), expr.getTableName())); } // This is CREATE TABLE LIKE if(expr.getLikeParentTableName() != null) @@ -1705,7 +1706,7 @@ public LogicalNode visitCreateTable(PlanContext context, Stack stack, Crea } // Set default storage properties to be created. - KeyValueSet keyValueSet = StorageUtil.newPhysicalProperties(createTableNode.getStorageType()); + KeyValueSet keyValueSet = CatalogUtil.newPhysicalProperties(createTableNode.getStorageType()); if (expr.hasParams()) { keyValueSet.putAll(expr.getParams()); } @@ -1788,7 +1789,7 @@ private PartitionMethodDesc getPartitionMethod(PlanContext context, CreateTable.ColumnPartition partition = (CreateTable.ColumnPartition) expr; String partitionExpression = Joiner.on(',').join(partition.getColumns()); - partitionMethodDesc = new PartitionMethodDesc(context.queryContext.getCurrentDatabase(), tableName, + partitionMethodDesc = new PartitionMethodDesc(context.queryContext.get(SessionVars.CURRENT_DATABASE), tableName, CatalogProtos.PartitionType.COLUMN, partitionExpression, convertColumnsToSchema(partition.getColumns())); } else { throw new PlanningException(String.format("Not supported PartitonType: %s", expr.getPartitionType())); @@ -1851,7 +1852,8 @@ public LogicalNode visitDropTable(PlanContext context, Stack stack, DropTa if (CatalogUtil.isFQTableName(dropTable.getTableName())) { qualified = dropTable.getTableName(); } else { - qualified = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), dropTable.getTableName()); + qualified = CatalogUtil.buildFQName( + context.queryContext.get(SessionVars.CURRENT_DATABASE), dropTable.getTableName()); } dropTableNode.init(qualified, dropTable.isIfExists(), dropTable.isPurge()); return dropTableNode; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/NamedExprsManager.java b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/NamedExprsManager.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java index 8666a5c545..991860ff33 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/NamedExprsManager.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/NamedExprsManager.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; @@ -27,9 +27,9 @@ import org.apache.tajo.algebra.NamedExpr; import org.apache.tajo.algebra.OpType; import org.apache.tajo.annotation.Nullable; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.EvalType; -import org.apache.tajo.engine.eval.FieldEval; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.EvalType; +import org.apache.tajo.plan.expr.FieldEval; import org.apache.tajo.util.TUtil; import java.util.*; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlanString.java b/tajo-plan/src/main/java/org/apache/tajo/plan/PlanString.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/PlanString.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/PlanString.java index 98f921ba25..b06c68f7f1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlanString.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/PlanString.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalNode; import java.util.ArrayList; import java.util.List; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlanningException.java b/tajo-plan/src/main/java/org/apache/tajo/plan/PlanningException.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/PlanningException.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/PlanningException.java index 4fa88ee089..2a52401aa7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlanningException.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/PlanningException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; public class PlanningException extends Exception { public PlanningException(String message) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/Target.java b/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java similarity index 91% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/Target.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/Target.java index 6a16d3c0a6..f49a93d9a9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/Target.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/Target.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; import org.apache.tajo.common.TajoDataTypes.DataType; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.FieldEval; -import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.FieldEval; +import org.apache.tajo.plan.serder.PlanGsonHelper; import org.apache.tajo.util.TUtil; /** @@ -46,7 +46,8 @@ public Target(final EvalNode eval, final String alias) { String normalized = alias; // If an expr is a column reference and its alias is equivalent to column name, ignore a given alias. - if (eval instanceof FieldEval && eval.getName().equals(normalized)) { + if (eval instanceof FieldEval + && eval.getName().equals(normalized)) { column = ((FieldEval) eval).getColumnRef(); } else { column = new Column(normalized, eval.getValueType()); @@ -124,6 +125,6 @@ public Object clone() throws CloneNotSupportedException { } public String toJson() { - return CoreGsonHelper.toJson(this, Target.class); + return PlanGsonHelper.toJson(this, Target.class); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/TypeDeterminant.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/TypeDeterminant.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java index 511814cf9e..8605b3d506 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/TypeDeterminant.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java @@ -16,9 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; import com.google.common.base.Preconditions; +import org.apache.tajo.DataTypeUtil; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.CatalogUtil; @@ -27,7 +28,7 @@ import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.DataTypeUtil; +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java index 8c3e6067a0..df1e341baa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AlgebraVisitor.java @@ -16,9 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.algebra; import org.apache.tajo.algebra.*; +import org.apache.tajo.plan.PlanningException; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/exception/AmbiguousFieldException.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AmbiguousFieldException.java similarity index 92% rename from tajo-core/src/main/java/org/apache/tajo/engine/exception/AmbiguousFieldException.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AmbiguousFieldException.java index 3125e30b40..ea3081c1a7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/exception/AmbiguousFieldException.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/AmbiguousFieldException.java @@ -16,7 +16,9 @@ * limitations under the License. */ -package org.apache.tajo.engine.exception; +package org.apache.tajo.plan.algebra; + +import org.apache.tajo.plan.InvalidQueryException; public class AmbiguousFieldException extends InvalidQueryException { private static final long serialVersionUID = 3102675985226352347L; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java index 24ff2e47bb..57ddfbd6d0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BaseAlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/algebra/BaseAlgebraVisitor.java @@ -16,9 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.algebra; import org.apache.tajo.algebra.*; +import org.apache.tajo.plan.PlanningException; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/Prioritized.java b/tajo-plan/src/main/java/org/apache/tajo/plan/annotator/Prioritized.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/Prioritized.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/annotator/Prioritized.java index 865d895071..8b202fca06 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/Prioritized.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/annotator/Prioritized.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.optimizer.eval; +package org.apache.tajo.plan.annotator; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AggregationFunctionCallEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AggregationFunctionCallEval.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/AggregationFunctionCallEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/AggregationFunctionCallEval.java index 32165199c5..542eae854b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AggregationFunctionCallEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AggregationFunctionCallEval.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicException.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicException.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicException.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicException.java index 9dba6df217..8cd9a8be1e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicException.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicException.java @@ -19,7 +19,7 @@ /** * */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; public class AlgebraicException extends RuntimeException { private static final long serialVersionUID = -1813125460274622006L; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java index 0b4ba19540..84352f0a65 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/AlgebraicUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/AlgebraicUtil.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import org.apache.tajo.catalog.Column; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java index 3b94cc9399..81b0f8e9dc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BasicEvalNodeVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import org.apache.tajo.exception.UnsupportedException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BetweenPredicateEval.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/BetweenPredicateEval.java index 07f9a6f424..84197e8d99 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BetweenPredicateEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BetweenPredicateEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.CatalogUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BinaryEval.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/BinaryEval.java index 85a0d7cc22..faeefcd3fc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/BinaryEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BinaryEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java index 3eb3a69ce0..4321d02184 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CaseWhenEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CaseWhenEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.collect.Lists; import com.google.gson.annotations.Expose; @@ -27,8 +27,8 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; -import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.plan.serder.PlanGsonHelper; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.TUtil; @@ -256,7 +256,7 @@ public String toString() { @Override public String toJson() { - return CoreGsonHelper.toJson(IfThenEval.this, IfThenEval.class); + return PlanGsonHelper.toJson(IfThenEval.this, IfThenEval.class); } @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CastEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/CastEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java index d996385e37..d625a11ba9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/CastEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/CastEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Schema; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/ConstEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/ConstEval.java index 323a4e98b8..b9f48a9dc6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/ConstEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/ConstEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java index 754f8885ce..638383adc3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.json.CoreGsonHelper; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.plan.serder.PlanGsonHelper; import org.apache.tajo.storage.Tuple; /** @@ -54,7 +54,7 @@ public EvalType getType() { @Override public String toJson() { - return CoreGsonHelper.toJson(this, EvalNode.class); + return PlanGsonHelper.toJson(this, EvalNode.class); } public abstract T eval(Schema schema, Tuple tuple); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor.java index 1680b3112c..372786b0a4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; @Deprecated public interface EvalNodeVisitor { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor2.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor2.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java index e85984e137..bae193abde 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalNodeVisitor2.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeFactory.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeFactory.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeFactory.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeFactory.java index aa72f25704..c708f4cda1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeFactory.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeFactory.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import org.apache.tajo.datum.Datum; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java index 457f651783..1f3f2ab800 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalTreeUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -29,10 +29,10 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.planner.ExprFinder; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.Target; import org.apache.tajo.exception.InternalException; +import org.apache.tajo.plan.util.ExprFinder; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.Target; import org.apache.tajo.util.TUtil; import java.util.*; @@ -144,7 +144,7 @@ public static List findAllColumnRefs(EvalNode node) { return finder.getColumnRefs(); } - public static Schema getSchemaByTargets(Schema inputSchema, Target [] targets) + public static Schema getSchemaByTargets(Schema inputSchema, Target[] targets) throws InternalException { Schema schema = new Schema(); for (Target target : targets) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java index d5335102e1..c1df658a1b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/EvalType.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalType.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; public enum EvalType { // Unary expression diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java index 5cedbca38f..c42768d287 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FieldEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FunctionEval.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/FunctionEval.java index b449040195..ec8d7da852 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/FunctionEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FunctionEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.base.Objects; import com.google.common.base.Preconditions; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/GeneralFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/GeneralFunctionEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java index 9446d7087f..e28e5f3f12 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/GeneralFunctionEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/GeneralFunctionEval.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.Schema; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.plan.function.GeneralFunction; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.TUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/InEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/InEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/InEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/InEval.java index 51097e631c..4dcc7bf4a3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/InEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/InEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.collect.Sets; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/InvalidEvalException.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/InvalidEvalException.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/InvalidEvalException.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/InvalidEvalException.java index 96eda91615..1649c81366 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/InvalidEvalException.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/InvalidEvalException.java @@ -19,7 +19,7 @@ /** * */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; public class InvalidEvalException extends RuntimeException { private static final long serialVersionUID = -2897003028483298256L; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/IsNullEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/IsNullEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/IsNullEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/IsNullEval.java index 25567fd0bc..6a17bf8995 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/IsNullEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/IsNullEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.CatalogUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/LikePredicateEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/LikePredicateEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/LikePredicateEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/LikePredicateEval.java index 535677f771..73f4f3ab2c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/LikePredicateEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/LikePredicateEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import org.apache.tajo.util.StringUtils; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/NotEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/NotEval.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/NotEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/NotEval.java index 72c85d2eab..916c1f7f50 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/NotEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/NotEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/PartialBinaryExpr.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/PartialBinaryExpr.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/PartialBinaryExpr.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/PartialBinaryExpr.java index 40966e502a..3aded7bbb7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/PartialBinaryExpr.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/PartialBinaryExpr.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.TajoDataTypes.DataType; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/PatternMatchPredicateEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/PatternMatchPredicateEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/PatternMatchPredicateEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/PatternMatchPredicateEval.java index 0a8e800c5c..1d83d602f6 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/PatternMatchPredicateEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/PatternMatchPredicateEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.CatalogUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RegexPredicateEval.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/RegexPredicateEval.java index f1e0241e5b..7519e2d8ab 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RegexPredicateEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RegexPredicateEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java index 5b21be046f..4a97e67b25 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/RowConstantEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.CatalogUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SignedEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SignedEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/SignedEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/SignedEval.java index 80edf66f4c..36f7e2fbec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SignedEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SignedEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimilarToPredicateEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimilarToPredicateEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/SimilarToPredicateEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimilarToPredicateEval.java index a690759647..ed5e123123 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimilarToPredicateEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimilarToPredicateEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java similarity index 99% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java index 7e7594a7a9..e706391c38 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/SimpleEvalNodeVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/SimpleEvalNodeVisitor.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.base.Preconditions; import org.apache.tajo.exception.UnsupportedException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/UnaryEval.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/UnaryEval.java index e7ae112893..7d457760b2 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/UnaryEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/UnaryEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.common.base.Objects; import com.google.common.base.Preconditions; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/eval/WindowFunctionEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java similarity index 94% rename from tajo-core/src/main/java/org/apache/tajo/engine/eval/WindowFunctionEval.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java index 4057e70c19..84b4a45d2f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/eval/WindowFunctionEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/WindowFunctionEval.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.eval; +package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.FunctionDesc; @@ -24,9 +24,9 @@ import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.FunctionContext; -import org.apache.tajo.engine.planner.logical.WindowSpec; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.FunctionContext; +import org.apache.tajo.plan.logical.WindowSpec; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.TUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/EvalTreeOptimizationRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/EvalTreeOptimizationRule.java similarity index 84% rename from tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/EvalTreeOptimizationRule.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/EvalTreeOptimizationRule.java index 2f17663809..7e0322475a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/EvalTreeOptimizationRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/EvalTreeOptimizationRule.java @@ -16,10 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.optimizer.eval; +package org.apache.tajo.plan.exprrewrite; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.annotator.Prioritized; +import org.apache.tajo.plan.expr.EvalNode; @Prioritized public interface EvalTreeOptimizationRule { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/EvalTreeOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/EvalTreeOptimizer.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/EvalTreeOptimizer.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/EvalTreeOptimizer.java index 3ed272e269..3bea6a4e05 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/EvalTreeOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/EvalTreeOptimizer.java @@ -16,14 +16,15 @@ * limitations under the License. */ -package org.apache.tajo.engine.optimizer.eval; +package org.apache.tajo.plan.exprrewrite; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.annotator.Prioritized; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.util.ClassUtil; import java.util.Collections; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantFolding.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java similarity index 91% rename from tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantFolding.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java index 8bcb0cd837..0e115e290a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantFolding.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantFolding.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.engine.optimizer.eval.rules; +package org.apache.tajo.plan.exprrewrite.rules; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.optimizer.eval.EvalTreeOptimizationRule; -import org.apache.tajo.engine.optimizer.eval.Prioritized; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.exprrewrite.EvalTreeOptimizationRule; +import org.apache.tajo.plan.annotator.Prioritized; +import org.apache.tajo.plan.LogicalPlanner; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantPropagation.java b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantPropagation.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantPropagation.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantPropagation.java index c7c6bd0ae2..6fe3b3ea4f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/optimizer/eval/rules/ConstantPropagation.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/exprrewrite/rules/ConstantPropagation.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.engine.optimizer.eval.rules; +package org.apache.tajo.plan.exprrewrite.rules; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.optimizer.eval.EvalTreeOptimizationRule; -import org.apache.tajo.engine.optimizer.eval.Prioritized; -import org.apache.tajo.engine.planner.LogicalPlanner; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.exprrewrite.EvalTreeOptimizationRule; +import org.apache.tajo.plan.annotator.Prioritized; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/AggFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/AggFunction.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/function/AggFunction.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/function/AggFunction.java index 10f4c3f1c6..08ea6a7f73 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/AggFunction.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/AggFunction.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.function; +package org.apache.tajo.plan.function; import org.apache.tajo.catalog.Column; import org.apache.tajo.function.Function; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionContext.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/FunctionContext.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionContext.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/function/FunctionContext.java index 076c8ec65a..9844bbcdee 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/FunctionContext.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/FunctionContext.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.function; +package org.apache.tajo.plan.function; public interface FunctionContext { } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/GeneralFunction.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/function/GeneralFunction.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java index a97baf0ddf..006449ffee 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/GeneralFunction.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/GeneralFunction.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package org.apache.tajo.engine.function; +package org.apache.tajo.plan.function; import org.apache.tajo.catalog.Column; -import org.apache.tajo.function.Function; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.eval.FunctionEval; +import org.apache.tajo.function.Function; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.plan.expr.FunctionEval; import org.apache.tajo.storage.Tuple; @Deprecated diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/function/WindowAggFunc.java b/tajo-plan/src/main/java/org/apache/tajo/plan/function/WindowAggFunc.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/function/WindowAggFunc.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/function/WindowAggFunc.java index 164738a9f2..82f725765b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/function/WindowAggFunc.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/function/WindowAggFunc.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.function; +package org.apache.tajo.plan.function; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.json.CatalogGsonHelper; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/Edge.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/Edge.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/Edge.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/Edge.java index c8bca039a9..0ff305ff32 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/Edge.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/Edge.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical.join; +package org.apache.tajo.plan.joinorder; -import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.plan.expr.EvalNode; public class Edge { private String src; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/FoundJoinOrder.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/FoundJoinOrder.java similarity index 92% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/FoundJoinOrder.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/FoundJoinOrder.java index 5ae34f75bc..57fdb7b332 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/FoundJoinOrder.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/FoundJoinOrder.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical.join; +package org.apache.tajo.plan.joinorder; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.tajo.engine.planner.logical.JoinNode; +import org.apache.tajo.plan.logical.JoinNode; /** * It contains the result of join enumeration. diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java index e445de7f47..a916d5cd9e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/GreedyHeuristicJoinOrderAlgorithm.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/GreedyHeuristicJoinOrderAlgorithm.java @@ -16,19 +16,22 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical.join; +package org.apache.tajo.plan.joinorder; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.eval.AlgebraicUtil; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.expr.AlgebraicUtil; +import org.apache.tajo.plan.logical.*; import org.apache.tajo.util.TUtil; -import java.util.*; +import java.util.LinkedHashSet; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; /** * This is a greedy heuristic algorithm to find a bushy join tree. This algorithm finds diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinEdge.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinEdge.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java index eabdf139c5..ce06748cc3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinEdge.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinEdge.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical.join; +package org.apache.tajo.plan.joinorder; import com.google.common.collect.Sets; import org.apache.tajo.algebra.JoinType; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.util.TUtil; import java.util.Collections; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java similarity index 90% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java index 5a861f9e9f..f7fc30e54b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinGraph.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinGraph.java @@ -16,23 +16,23 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical.join; +package org.apache.tajo.plan.joinorder; import com.google.common.collect.Sets; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.eval.AlgebraicUtil; -import org.apache.tajo.engine.eval.BinaryEval; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.EvalTreeUtil; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.NamedExprsManager; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.graph.SimpleUndirectedGraph; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.RelationNode; +import org.apache.tajo.util.graph.SimpleUndirectedGraph; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.NamedExprsManager; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.expr.AlgebraicUtil; +import org.apache.tajo.plan.expr.BinaryEval; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.EvalTreeUtil; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.RelationNode; import org.apache.tajo.util.TUtil; import java.util.*; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinOrderAlgorithm.java b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderAlgorithm.java similarity index 85% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinOrderAlgorithm.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderAlgorithm.java index eafa671f29..2dd0670579 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/join/JoinOrderAlgorithm.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/joinorder/JoinOrderAlgorithm.java @@ -16,11 +16,13 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical.join; +package org.apache.tajo.plan.joinorder; import org.apache.hadoop.classification.InterfaceStability; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.joinorder.FoundJoinOrder; +import org.apache.tajo.plan.joinorder.JoinGraph; import java.util.Set; @@ -39,7 +41,7 @@ public interface JoinOrderAlgorithm { * A join graph does not contain relations that do not have any corresponding join condition. * @param relationsWithoutQual The names of relations that do not have any corresponding join condition. * @return - * @throws PlanningException + * @throws org.apache.tajo.plan.PlanningException */ FoundJoinOrder findBestOrder(LogicalPlan plan, LogicalPlan.QueryBlock block, JoinGraph joinGraph, Set relationsWithoutQual) throws PlanningException; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/AlterTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/AlterTableNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java index 76a47d0b3b..e9e24677d0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/AlterTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTableNode.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.tajo.algebra.AlterTableOpType; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; public class AlterTableNode extends LogicalNode { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlterTablespaceNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTablespaceNode.java similarity index 91% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/AlterTablespaceNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTablespaceNode.java index d64a89d3ab..7b79cc1177 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/AlterTablespaceNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/AlterTablespaceNode.java @@ -16,15 +16,16 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.algebra.AlterTablespaceSetType; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.LogicalNodeVisitor; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.LogicalNodeVisitor; +import org.apache.tajo.plan.logical.NodeType; public class AlterTablespaceNode extends LogicalNode implements Cloneable { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/BinaryNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/BinaryNode.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/BinaryNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/BinaryNode.java index 8aaeb58405..709ef34166 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/BinaryNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/BinaryNode.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.tajo.json.GsonObject; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateDatabaseNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateDatabaseNode.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateDatabaseNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateDatabaseNode.java index 9dc73e2c9b..e3f73feb77 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateDatabaseNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateDatabaseNode.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; public class CreateDatabaseNode extends LogicalNode implements Cloneable { private String databaseName; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateTableNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java index a39b391fce..d03da6a725 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/CreateTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateTableNode.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.hadoop.fs.Path; -import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; public class CreateTableNode extends StoreTableNode implements Cloneable { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DistinctGroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DistinctGroupbyNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java index 47e8933976..e31e488785 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DistinctGroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DistinctGroupbyNode.java @@ -16,20 +16,18 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.eval.AggregationFunctionCallEval; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.AggregationFunctionCallEval; import org.apache.tajo.util.TUtil; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; -import java.util.Set; public class DistinctGroupbyNode extends UnaryNode implements Projectable, Cloneable { @Expose @@ -48,7 +46,7 @@ public class DistinctGroupbyNode extends UnaryNode implements Projectable, Clone private int[] resultColumnIds; /** Aggregation Functions */ - @Expose private AggregationFunctionCallEval [] aggrFunctions; + @Expose private AggregationFunctionCallEval[] aggrFunctions; public DistinctGroupbyNode(int pid) { super(pid, NodeType.DISTINCT_GROUP_BY); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropDatabaseNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropDatabaseNode.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropDatabaseNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropDatabaseNode.java index 1578759eb1..b88c384ae7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropDatabaseNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropDatabaseNode.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; public class DropDatabaseNode extends LogicalNode implements Cloneable { @Expose private String databaseName; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropTableNode.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropTableNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropTableNode.java index ac68a9c06c..1a6185241a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/DropTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/DropTableNode.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; public class DropTableNode extends LogicalNode implements Cloneable { private String tableName; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java similarity index 91% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java index 45ab611f5b..2519165a63 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/EvalExprNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/EvalExprNode.java @@ -19,12 +19,12 @@ /** * */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.Target; import org.apache.tajo.util.TUtil; public class EvalExprNode extends LogicalNode implements Projectable { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ExceptNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ExceptNode.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ExceptNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/ExceptNode.java index 1540e1c5b5..221b62c92c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ExceptNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ExceptNode.java @@ -19,9 +19,9 @@ /** * */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; public class ExceptNode extends BinaryNode { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/GroupElement.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupElement.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/GroupElement.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupElement.java index 3fb05c2d33..a969c30b88 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/GroupElement.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupElement.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.logical; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.Expose; import org.apache.tajo.algebra.Aggregation; -import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.plan.expr.EvalNode; public class GroupElement implements Cloneable { @Expose private Aggregation.GroupType type; @@ -42,7 +42,7 @@ public Aggregation.GroupType getType() { return this.type; } - public EvalNode [] getGroupingSets() { + public EvalNode[] getGroupingSets() { return this.groupingSets; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java index 75aa52c12b..2c74ce3874 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/GroupbyNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/GroupbyNode.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.eval.AggregationFunctionCallEval; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.AggregationFunctionCallEval; import org.apache.tajo.util.TUtil; public class GroupbyNode extends UnaryNode implements Projectable, Cloneable { @@ -66,7 +66,7 @@ public boolean hasAggFunctions() { return this.aggrFunctions != null; } - public AggregationFunctionCallEval [] getAggFunctions() { + public AggregationFunctionCallEval[] getAggFunctions() { return this.aggrFunctions; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/HavingNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/HavingNode.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/HavingNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/HavingNode.java index aa6d597abc..a3371c727f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/HavingNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/HavingNode.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.expr.EvalNode; public class HavingNode extends UnaryNode implements SelectableNode, Cloneable { @Expose private EvalNode qual; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/IndexScanNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/IndexScanNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java index bff0b31c46..0d59733c77 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/IndexScanNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IndexScanNode.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.Gson; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; import org.apache.tajo.datum.Datum; -import org.apache.tajo.engine.json.CoreGsonHelper; +import org.apache.tajo.plan.serder.PlanGsonHelper; public class IndexScanNode extends ScanNode { @Expose private SortSpec [] sortKeys; @@ -65,7 +65,7 @@ public void setKeySchema( Schema keySchema ) { @Override public String toString() { - Gson gson = CoreGsonHelper.getInstance(); + Gson gson = PlanGsonHelper.getInstance(); StringBuilder builder = new StringBuilder(); builder.append("IndexScanNode : {\n"); builder.append(" \"keySchema\" : \"" + gson.toJson(this.keySchema) + "\"\n"); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/InsertNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/InsertNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java index f5e87ef1cd..f860f088f0 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/InsertNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; import org.apache.tajo.util.TUtil; public class InsertNode extends StoreTableNode implements Cloneable { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/IntersectNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IntersectNode.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/IntersectNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/IntersectNode.java index 4bcfd24d37..3f1bf0dfc5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/IntersectNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/IntersectNode.java @@ -19,9 +19,9 @@ /** * */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; public class IntersectNode extends BinaryNode { public IntersectNode(int pid) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/JoinNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/JoinNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java index 558a4884ae..58dfac24fb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/JoinNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinNode.java @@ -19,15 +19,15 @@ /** * */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.tajo.algebra.JoinType; -import org.apache.tajo.engine.eval.BinaryEval; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.BinaryEval; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.util.TUtil; import java.util.ArrayList; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LimitNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LimitNode.java similarity index 94% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LimitNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/LimitNode.java index 7f5b258d66..7a3431efbd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LimitNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LimitNode.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; public final class LimitNode extends UnaryNode implements Cloneable { @Expose private long fetchFirstNum; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java similarity index 92% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java index f62caa45f4..c42a05e99f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNode.java @@ -19,14 +19,14 @@ /** * */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.json.CoreGsonHelper; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.PlannerUtil; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.serder.PlanGsonHelper; import org.apache.tajo.util.TUtil; public abstract class LogicalNode implements Cloneable, GsonObject { @@ -114,7 +114,7 @@ public Object clone() throws CloneNotSupportedException { @Override public String toJson() { - return CoreGsonHelper.toJson(this, LogicalNode.class); + return PlanGsonHelper.toJson(this, LogicalNode.class); } public abstract void preOrder(LogicalNodeVisitor visitor); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNodeVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNodeVisitor.java similarity index 94% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNodeVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNodeVisitor.java index 5b0c1c2129..b84ec654a1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalNodeVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalNodeVisitor.java @@ -19,7 +19,7 @@ /** * */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; public interface LogicalNodeVisitor { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalRootNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalRootNode.java similarity index 92% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalRootNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalRootNode.java index bacf14f65c..011a9b7fde 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/LogicalRootNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/LogicalRootNode.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; public class LogicalRootNode extends UnaryNode implements Cloneable { public LogicalRootNode(int pid) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/exception/NoSuchColumnException.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NoSuchColumnException.java similarity index 67% rename from tajo-core/src/main/java/org/apache/tajo/engine/exception/NoSuchColumnException.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/NoSuchColumnException.java index 70191a9cb1..94cef1efaf 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/exception/NoSuchColumnException.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NoSuchColumnException.java @@ -1,4 +1,4 @@ -/* +/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -16,10 +16,19 @@ * limitations under the License. */ -package org.apache.tajo.engine.exception; +package org.apache.tajo.plan.logical; + + +import org.apache.tajo.plan.verifier.VerifyException; public class NoSuchColumnException extends VerifyException { + private static final long serialVersionUID = 277182608283894937L; + + public NoSuchColumnException(String databaseName, String relName, String columnName) { + super(String.format("ERROR: column \" %s.%s \" in %s does not exist", relName, columnName, databaseName)); + } + public NoSuchColumnException(String columnName) { - super("ERROR: no such a column '" + columnName + "'"); + super("ERROR: column \"" + columnName + "\" does not exist"); } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java index fa1199bc69..5d1874f711 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/NodeType.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/NodeType.java @@ -19,11 +19,9 @@ /** * */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; -import org.apache.tajo.engine.planner.AlterTablespaceNode; - /** * This indicates a logical node type. */ diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/PartitionedTableScanNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PartitionedTableScanNode.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/PartitionedTableScanNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/PartitionedTableScanNode.java index c33bdf518d..6fd969a183 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/PartitionedTableScanNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PartitionedTableScanNode.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.util.TUtil; public class PartitionedTableScanNode extends ScanNode { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/PersistentStoreNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/PersistentStoreNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java index 61507c808a..420ed7ccd7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/PersistentStoreNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/PersistentStoreNode.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; +import org.apache.tajo.plan.PlanString; import org.apache.tajo.util.KeyValueSet; -import org.apache.tajo.engine.planner.PlanString; import org.apache.tajo.util.TUtil; import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/Projectable.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/Projectable.java similarity index 81% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/Projectable.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/Projectable.java index 1e4bdc5aa0..68d18610e5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/Projectable.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/Projectable.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.plan.Target; /** * Projectable is an interface for a LogicalNode which has a list of targets. * What a logical node has a list of targets means that the node evaluated a list of expressions. - * For example, {@link org.apache.tajo.engine.planner.logical.ScanNode}, - * {@link org.apache.tajo.engine.planner.logical.JoinNode}, - * {@link org.apache.tajo.engine.planner.logical.GroupbyNode}, and - * {@link org.apache.tajo.engine.planner.logical.ProjectionNode} are all Projectable nodes. + * For example, {@link ScanNode}, + * {@link JoinNode}, + * {@link GroupbyNode}, and + * {@link ProjectionNode} are all Projectable nodes. * The expression evaluation occurs only at those projectable nodes. */ public interface Projectable { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ProjectionNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ProjectionNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java index e9fd803c1a..4ef7e2d617 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ProjectionNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ProjectionNode.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.Target; import org.apache.tajo.util.TUtil; public class ProjectionNode extends UnaryNode implements Projectable { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/RelationNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/RelationNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java index 83c16cdd62..fd8e93735e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/RelationNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/RelationNode.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import org.apache.tajo.catalog.Schema; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java index fc5cefca95..a7ef5431ca 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ScanNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.Target; -import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.util.TUtil; public class ScanNode extends RelationNode implements Projectable, SelectableNode, Cloneable { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/SelectableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SelectableNode.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/SelectableNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/SelectableNode.java index 7082f4bca4..1a66d9424b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/SelectableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SelectableNode.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; -import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.plan.expr.EvalNode; /** * An interface for logical node which is able to filter tuples. diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/SelectionNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SelectionNode.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/SelectionNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/SelectionNode.java index 3bbbd82691..4c4d5ee285 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/SelectionNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SelectionNode.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.expr.EvalNode; public class SelectionNode extends UnaryNode implements SelectableNode, Cloneable { @Expose private EvalNode qual; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ShuffleFileWriteNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java similarity index 88% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ShuffleFileWriteNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java index c7ea454a65..29aefc9956 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/ShuffleFileWriteNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ShuffleFileWriteNode.java @@ -16,21 +16,20 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; -import org.apache.tajo.ipc.TajoWorkerProtocol; import org.apache.tajo.util.TUtil; -import static org.apache.tajo.ipc.TajoWorkerProtocol.ShuffleType.NONE_SHUFFLE; +import static org.apache.tajo.plan.serder.PlanProto.ShuffleType; /** * ShuffeFileWriteNode is an expression for an intermediate data materialization step. */ public class ShuffleFileWriteNode extends PersistentStoreNode implements Cloneable { - @Expose private TajoWorkerProtocol.ShuffleType shuffleType = NONE_SHUFFLE; + @Expose private ShuffleType shuffleType = ShuffleType.NONE_SHUFFLE; @Expose private int numOutputs; @Expose private Column [] shuffleKeys; @@ -50,7 +49,7 @@ public final boolean hasShuffleKeys() { return shuffleKeys; } - public final void setShuffle(TajoWorkerProtocol.ShuffleType type, Column[] keys, int numPartitions) { + public final void setShuffle(ShuffleType type, Column[] keys, int numPartitions) { Preconditions.checkArgument(keys.length >= 0, "At least one partition key must be specified."); // In outer join, zero can be passed into this value because of empty tables. @@ -63,7 +62,7 @@ public final void setShuffle(TajoWorkerProtocol.ShuffleType type, Column[] keys, this.numOutputs = numPartitions; } - public TajoWorkerProtocol.ShuffleType getShuffleType() { + public ShuffleType getShuffleType() { return this.shuffleType; } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/SortNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/SortNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java index a73271014d..a697f9f176 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/SortNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; import org.apache.tajo.util.TUtil; public final class SortNode extends UnaryNode implements Cloneable { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/StoreTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/StoreTableNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java index b0f7b7a6ca..0623d212ec 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/StoreTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/StoreTableNode.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.partition.PartitionMethodDesc; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; import org.apache.tajo.util.TUtil; public class StoreTableNode extends PersistentStoreNode implements Cloneable { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TableSubQueryNode.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/TableSubQueryNode.java index 3c808fcd78..c8fbecdc24 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TableSubQueryNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TableSubQueryNode.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.Target; -import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.Target; public class TableSubQueryNode extends RelationNode implements Projectable { @Expose private String tableName; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java similarity index 94% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java index 3ffbd7bc4e..10c65b6d5f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/TruncateTableNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/TruncateTableNode.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; import org.apache.tajo.util.TUtil; import java.util.List; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnaryNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/UnaryNode.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnaryNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/UnaryNode.java index 0b06e9ebf3..0fc5c373cc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnaryNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/UnaryNode.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.gson.annotations.Expose; import org.apache.tajo.util.TUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnionNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/UnionNode.java similarity index 91% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnionNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/UnionNode.java index 49183d016e..2839ab5328 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/UnionNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/UnionNode.java @@ -19,9 +19,9 @@ /** * */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; -import org.apache.tajo.engine.planner.PlanString; +import org.apache.tajo.plan.PlanString; public class UnionNode extends BinaryNode { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/WindowAggNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/WindowAggNode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java index 20b195f75b..3f624f611e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/WindowAggNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowAggNode.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.engine.eval.WindowFunctionEval; -import org.apache.tajo.engine.planner.PlanString; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.Target; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.expr.WindowFunctionEval; import org.apache.tajo.util.TUtil; public class WindowAggNode extends UnaryNode implements Projectable, Cloneable { @@ -83,7 +83,7 @@ public boolean hasAggFunctions() { return this.windowFuncs != null; } - public WindowFunctionEval [] getWindowFunctions() { + public WindowFunctionEval[] getWindowFunctions() { return this.windowFuncs; } @@ -199,6 +199,7 @@ public PlanString getPlanString() { if( i < sortSpecs.length - 1) { sb.append(","); } + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/WindowSpec.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowSpec.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/WindowSpec.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowSpec.java index afec0ac97b..73f4e13ca5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/logical/WindowSpec.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/WindowSpec.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.logical; +package org.apache.tajo.plan.logical; import com.google.common.base.Objects; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.eval.EvalNode; +import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.util.TUtil; import static org.apache.tajo.algebra.WindowSpec.WindowFrameEndBoundType; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java similarity index 96% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index aee5d43b6f..51a016f4d8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.nameresolver; +package org.apache.tajo.plan.nameresolver; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -24,12 +24,12 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.exception.AmbiguousFieldException; -import org.apache.tajo.engine.exception.NoSuchColumnException; -import org.apache.tajo.engine.exception.VerifyException; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.logical.RelationNode; +import org.apache.tajo.catalog.exception.NoSuchColumnException; +import org.apache.tajo.plan.algebra.AmbiguousFieldException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.verifier.VerifyException; +import org.apache.tajo.plan.logical.RelationNode; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolvingMode.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolvingMode.java index dcbb5f1b96..4d9f9a5a0d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/NameResolvingMode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolvingMode.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.nameresolver; +package org.apache.tajo.plan.nameresolver; /** * diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java similarity index 91% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java index 396bc1b184..a1d9dbdd51 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByLegacy.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.nameresolver; +package org.apache.tajo.plan.nameresolver; import org.apache.tajo.algebra.ColumnReferenceExpr; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; -import org.apache.tajo.engine.exception.NoSuchColumnException; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; -import org.apache.tajo.engine.planner.logical.RelationNode; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NoSuchColumnException; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.RelationNode; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRels.java similarity index 85% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRels.java index 9713e5251a..a67a1caecd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRels.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRels.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.nameresolver; +package org.apache.tajo.plan.nameresolver; import org.apache.tajo.algebra.ColumnReferenceExpr; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.exception.NoSuchColumnException; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.NoSuchColumnException; public class ResolverByRels extends NameResolver { @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRelsAndSubExprs.java similarity index 86% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRelsAndSubExprs.java index 7ca3c535d3..f48611745c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverByRelsAndSubExprs.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByRelsAndSubExprs.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.nameresolver; +package org.apache.tajo.plan.nameresolver; import org.apache.tajo.algebra.ColumnReferenceExpr; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.exception.NoSuchColumnException; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.catalog.exception.NoSuchColumnException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; public class ResolverByRelsAndSubExprs extends NameResolver { @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverBySubExprsAndRels.java similarity index 86% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverBySubExprsAndRels.java index 7337ecee5e..3064fdee47 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/nameresolver/ResolverBySubExprsAndRels.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverBySubExprsAndRels.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.nameresolver; +package org.apache.tajo.plan.nameresolver; import org.apache.tajo.algebra.ColumnReferenceExpr; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.exception.NoSuchColumnException; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.NoSuchColumnException; public class ResolverBySubExprsAndRels extends NameResolver { @Override diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/BasicQueryRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BasicQueryRewriteEngine.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/BasicQueryRewriteEngine.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BasicQueryRewriteEngine.java index 6b3ed1e80f..491dda1be1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/BasicQueryRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BasicQueryRewriteEngine.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.rewrite; +package org.apache.tajo.plan.rewrite; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; import java.util.LinkedHashMap; import java.util.Map; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/QueryRewriteEngine.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/QueryRewriteEngine.java similarity index 87% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/QueryRewriteEngine.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/QueryRewriteEngine.java index cb66582973..b7f5637d84 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/QueryRewriteEngine.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/QueryRewriteEngine.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.rewrite; +package org.apache.tajo.plan.rewrite; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; public interface QueryRewriteEngine { /** diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/RewriteRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/RewriteRule.java similarity index 92% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/RewriteRule.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/RewriteRule.java index 89854df5a8..0ba746027e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/RewriteRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/RewriteRule.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.rewrite; +package org.apache.tajo.plan.rewrite; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; /** * An interface for a rewrite rule. diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index b5dafcb034..31063bfa9b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.rewrite; +package org.apache.tajo.plan.rewrite.rules; import com.google.common.collect.*; import org.apache.commons.logging.Log; @@ -26,11 +26,13 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.exception.InvalidQueryException; -import org.apache.tajo.engine.planner.*; -import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.planner.rewrite.FilterPushDownRule.FilterPushDownContext; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule.FilterPushDownContext; +import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.util.TUtil; import java.util.*; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java similarity index 79% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java index 666c5fc1ef..14b24a79d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/PartitionedTableRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.rewrite; +package org.apache.tajo.plan.rewrite.rules; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -28,14 +28,18 @@ import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.planner.BasicLogicalPlanVisitor; -import org.apache.tajo.engine.planner.LogicalPlan; -import org.apache.tajo.engine.planner.PlannerUtil; -import org.apache.tajo.engine.planner.PlanningException; -import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.utils.TupleUtil; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.storage.Tuple; +import org.apache.tajo.storage.VTuple; +import org.apache.tajo.util.StringUtils; import java.io.IOException; import java.util.List; @@ -82,10 +86,9 @@ public LogicalPlan rewrite(LogicalPlan plan) throws PlanningException { } private static class PartitionPathFilter implements PathFilter { + private Schema schema; private EvalNode partitionFilter; - - public PartitionPathFilter(Schema schema, EvalNode partitionFilter) { this.schema = schema; this.partitionFilter = partitionFilter; @@ -93,7 +96,7 @@ public PartitionPathFilter(Schema schema, EvalNode partitionFilter) { @Override public boolean accept(Path path) { - Tuple tuple = TupleUtil.buildTupleFromPartitionPath(schema, path, true); + Tuple tuple = buildTupleFromPartitionPath(schema, path, true); if (tuple == null) { // if it is a file or not acceptable file return false; } @@ -329,6 +332,70 @@ private void updateTableStat(PartitionedTableScanNode scanNode) throws PlanningE } } + /** + * Take a look at a column partition path. A partition path consists + * of a table path part and column values part. This method transforms + * a partition path into a tuple with a given partition column schema. + * + * hdfs://192.168.0.1/tajo/warehouse/table1/col1=abc/col2=def/col3=ghi + * ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^ + * table path part column values part + * + * When a file path is given, it can perform two ways depending on beNullIfFile flag. + * If it is true, it returns NULL when a given path is a file. + * Otherwise, it returns a built tuple regardless of file or directory. + * + * @param partitionColumnSchema The partition column schema + * @param partitionPath The partition path + * @param beNullIfFile If true, this method returns NULL when a given path is a file. + * @return The tuple transformed from a column values part. + */ + public static Tuple buildTupleFromPartitionPath(Schema partitionColumnSchema, Path partitionPath, + boolean beNullIfFile) { + int startIdx = partitionPath.toString().indexOf(getColumnPartitionPathPrefix(partitionColumnSchema)); + + if (startIdx == -1) { // if there is no partition column in the patch + return null; + } + String columnValuesPart = partitionPath.toString().substring(startIdx); + + String [] columnValues = columnValuesPart.split("/"); + + // true means this is a file. + if (beNullIfFile && partitionColumnSchema.size() < columnValues.length) { + return null; + } + + Tuple tuple = new VTuple(partitionColumnSchema.size()); + int i = 0; + for (; i < columnValues.length && i < partitionColumnSchema.size(); i++) { + String [] parts = columnValues[i].split("="); + if (parts.length != 2) { + return null; + } + int columnId = partitionColumnSchema.getColumnIdByName(parts[0]); + Column keyColumn = partitionColumnSchema.getColumn(columnId); + tuple.put(columnId, DatumFactory.createFromString(keyColumn.getDataType(), StringUtils.unescapePathName(parts[1]))); + } + for (; i < partitionColumnSchema.size(); i++) { + tuple.put(i, NullDatum.get()); + } + return tuple; + } + + /** + * Get a prefix of column partition path. For example, consider a column partition (col1, col2). + * Then, you will get a string 'col1='. + * + * @param partitionColumn the schema of column partition + * @return The first part string of column partition path. + */ + private static String getColumnPartitionPathPrefix(Schema partitionColumn) { + StringBuilder sb = new StringBuilder(); + sb.append(partitionColumn.getColumn(0).getSimpleName()).append("="); + return sb.toString(); + } + private final class Rewriter extends BasicLogicalPlanVisitor { @Override public Object visitScan(Object object, LogicalPlan plan, LogicalPlan.QueryBlock block, ScanNode scanNode, diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java index ec5df04fa7..167da5b69a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/rewrite/ProjectionPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/ProjectionPushDownRule.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner.rewrite; +package org.apache.tajo.plan.rewrite.rules; import com.google.common.collect.*; import org.apache.commons.logging.Log; @@ -25,18 +25,21 @@ import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.planner.*; -import org.apache.tajo.engine.planner.LogicalPlan.QueryBlock; -import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.utils.SchemaUtil; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.LogicalPlan.QueryBlock; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.RewriteRule; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.util.SchemaUtil; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.util.TUtil; import java.util.*; /** * ProjectionPushDownRule deploys expressions in a selection list to proper - * {@link org.apache.tajo.engine.planner.logical.Projectable} + * {@link org.apache.tajo.plan.logical.Projectable} * nodes. In this process, the expressions are usually pushed down into as lower as possible. * It also enables scanners to read only necessary columns. */ diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/json/EvalNodeAdapter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeAdapter.java similarity index 93% rename from tajo-core/src/main/java/org/apache/tajo/engine/json/EvalNodeAdapter.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeAdapter.java index 56600a681c..7a3238c6e5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/json/EvalNodeAdapter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalNodeAdapter.java @@ -19,12 +19,12 @@ /** * */ -package org.apache.tajo.engine.json; +package org.apache.tajo.plan.serder; import com.google.gson.*; -import org.apache.tajo.engine.eval.EvalNode; -import org.apache.tajo.engine.eval.EvalType; import org.apache.tajo.json.GsonSerDerAdapter; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.expr.EvalType; import java.lang.reflect.Type; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java similarity index 95% rename from tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java index b62e23676a..e6d54b1fbc 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoDeserializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoDeserializer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.plan; +package org.apache.tajo.plan.serder; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -25,22 +25,21 @@ import org.apache.tajo.catalog.exception.NoSuchFunctionException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.datum.*; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.function.AggFunction; -import org.apache.tajo.engine.function.GeneralFunction; -import org.apache.tajo.engine.plan.proto.PlanProto; import org.apache.tajo.exception.InternalException; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.GeneralFunction; import java.util.*; /** * It deserializes a serialized eval tree consisting of a number of EvalNodes. * - * {@link org.apache.tajo.engine.plan.EvalTreeProtoSerializer} serializes an eval tree in a postfix traverse order. + * {@link EvalTreeProtoSerializer} serializes an eval tree in a postfix traverse order. * So, this class firstly sorts all serialized eval nodes in ascending order of their sequence IDs. Then, * it sequentially restores each serialized node to EvalNode instance. * - * @see org.apache.tajo.engine.plan.EvalTreeProtoSerializer + * @see EvalTreeProtoSerializer */ public class EvalTreeProtoDeserializer { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java index d2b6fd741f..9f22c20bc8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/plan/EvalTreeProtoSerializer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/EvalTreeProtoSerializer.java @@ -16,21 +16,20 @@ * limitations under the License. */ -package org.apache.tajo.engine.plan; +package org.apache.tajo.plan.serder; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.protobuf.ByteString; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.IntervalDatum; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.plan.proto.PlanProto; +import org.apache.tajo.plan.expr.*; import java.util.Map; import java.util.Stack; /** - * It traverses an eval tree consisting of a number of {@link org.apache.tajo.engine.eval.EvalNode} + * It traverses an eval tree consisting of a number of {@link org.apache.tajo.plan.expr.EvalNode} * in a postfix traverse order. The postfix traverse order guarantees that all child nodes of some node N * were already visited when the node N is visited. This manner makes tree serialization possible in a simple logic. */ @@ -64,7 +63,7 @@ public static PlanProto.EvalTree serialize(EvalNode evalNode) { * @param context Context * @param evalNode EvalNode * @return The array of IDs which points to stored EvalNode. - * @see org.apache.tajo.engine.eval.EvalNode + * @see org.apache.tajo.plan.expr.EvalNode */ private int [] registerGetChildIds(EvalTreeProtoBuilderContext context, EvalNode evalNode) { int [] childIds = new int[evalNode.childNum()]; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/json/LogicalNodeAdapter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeAdapter.java similarity index 91% rename from tajo-core/src/main/java/org/apache/tajo/engine/json/LogicalNodeAdapter.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeAdapter.java index e2becef27a..30ff053359 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/json/LogicalNodeAdapter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/LogicalNodeAdapter.java @@ -19,11 +19,11 @@ /** * */ -package org.apache.tajo.engine.json; +package org.apache.tajo.plan.serder; import com.google.gson.*; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.NodeType; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.json.GsonSerDerAdapter; import java.lang.reflect.Type; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/serder/PlanGsonHelper.java b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/PlanGsonHelper.java new file mode 100644 index 0000000000..8cafbd075d --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/serder/PlanGsonHelper.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.serder; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.catalog.json.FunctionAdapter; +import org.apache.tajo.catalog.json.TableMetaAdapter; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.function.Function; +import org.apache.tajo.json.*; +import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.plan.function.AggFunction; +import org.apache.tajo.plan.function.GeneralFunction; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.util.TUtil; + +import java.lang.reflect.Type; +import java.util.Map; + +public class PlanGsonHelper { + private static Gson gson; + private static Gson gsonPretty; + + private PlanGsonHelper() { + } + + private static Map registerAdapters() { + Map adapters = TUtil.newHashMap(); + adapters.put(Path.class, new PathSerializer()); + adapters.put(Class.class, new ClassNameSerializer()); + adapters.put(LogicalNode.class, new LogicalNodeAdapter()); + adapters.put(EvalNode.class, new EvalNodeAdapter()); + adapters.put(TableMeta.class, new TableMetaAdapter()); + adapters.put(Function.class, new FunctionAdapter()); + adapters.put(GeneralFunction.class, new FunctionAdapter()); + adapters.put(AggFunction.class, new FunctionAdapter()); + adapters.put(Datum.class, new DatumAdapter()); + adapters.put(DataType.class, new DataTypeAdapter()); + + return adapters; + } + + public static Gson getInstance() { + if (gson == null ) { + GsonHelper helper = new GsonHelper(registerAdapters()); + gson = helper.getGson(); + } + return gson; + } + + public static Gson getPrettyInstance() { + if (gsonPretty == null) { + GsonBuilder prettyBuilder = new GsonBuilder() + .setPrettyPrinting() + .excludeFieldsWithoutExposeAnnotation(); + GsonHelper.registerAdapters(prettyBuilder, registerAdapters()); + gsonPretty = prettyBuilder.create(); + } + + return gsonPretty; + } + + public static String toJson(GsonObject object, Class clazz) { + return getInstance().toJson(object, clazz); + } + + public static T fromJson(String json, Class clazz) { + return getInstance().fromJson(json, clazz); + } +} diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprFinder.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/ExprFinder.java similarity index 94% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprFinder.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/util/ExprFinder.java index dbc844dc62..68005825a5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprFinder.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/ExprFinder.java @@ -16,12 +16,14 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.util; import org.apache.tajo.algebra.BinaryOperator; import org.apache.tajo.algebra.Expr; import org.apache.tajo.algebra.OpType; import org.apache.tajo.algebra.UnaryOperator; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import java.util.HashSet; import java.util.Set; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java similarity index 84% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 827be83d1d..a7175a1868 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -16,37 +16,27 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.util; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; import org.apache.tajo.common.TajoDataTypes.DataType; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.exception.InvalidQueryException; -import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.utils.SchemaUtil; -import org.apache.tajo.storage.StorageManager; -import org.apache.tajo.storage.TupleComparator; -import org.apache.tajo.storage.fragment.FileFragment; -import org.apache.tajo.storage.fragment.FragmentConvertor; +import org.apache.tajo.plan.*; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor; +import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import org.apache.tajo.util.TUtil; -import java.io.IOException; import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; public class PlannerUtil { @@ -571,14 +561,6 @@ public static SortSpec[][] getSortKeysFromJoinQual(EvalNode joinQual, Schema out return new SortSpec[][]{outerSortSpec, innerSortSpec}; } - public static TupleComparator[] getComparatorsFromJoinQual(EvalNode joinQual, Schema leftSchema, Schema rightSchema) { - SortSpec[][] sortSpecs = getSortKeysFromJoinQual(joinQual, leftSchema, rightSchema); - TupleComparator[] comparators = new TupleComparator[2]; - comparators[0] = new TupleComparator(leftSchema, sortSpecs[0]); - comparators[1] = new TupleComparator(rightSchema, sortSpecs[1]); - return comparators; - } - /** * @return the first array contains left table's columns, and the second array contains right table's columns. */ @@ -793,96 +775,4 @@ public static String buildExplainString(LogicalNode node) { return explains.toString(); } - - /** - * Listing table data file which is not empty. - * If the table is a partitioned table, return file list which has same partition key. - * @param tajoConf - * @param tableDesc - * @param fileIndex - * @param numResultFiles - * @return - * @throws IOException - */ - public static FragmentProto[] getNonZeroLengthDataFiles(TajoConf tajoConf,TableDesc tableDesc, - int fileIndex, int numResultFiles) throws IOException { - FileSystem fs = tableDesc.getPath().getFileSystem(tajoConf); - - List nonZeroLengthFiles = new ArrayList(); - if (fs.exists(tableDesc.getPath())) { - getNonZeroLengthDataFiles(fs, tableDesc.getPath(), nonZeroLengthFiles, fileIndex, numResultFiles, - new AtomicInteger(0)); - } - - List fragments = new ArrayList(); - - //In the case of partitioned table, return same partition key data files. - int numPartitionColumns = 0; - if (tableDesc.hasPartition()) { - numPartitionColumns = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size(); - } - String[] previousPartitionPathNames = null; - for (FileStatus eachFile: nonZeroLengthFiles) { - FileFragment fileFragment = new FileFragment(tableDesc.getName(), eachFile.getPath(), 0, eachFile.getLen(), null); - - if (numPartitionColumns > 0) { - // finding partition key; - Path filePath = fileFragment.getPath(); - Path parentPath = filePath; - String[] parentPathNames = new String[numPartitionColumns]; - for (int i = 0; i < numPartitionColumns; i++) { - parentPath = parentPath.getParent(); - parentPathNames[numPartitionColumns - i - 1] = parentPath.getName(); - } - - // If current partitionKey == previousPartitionKey, add to result. - if (previousPartitionPathNames == null) { - fragments.add(fileFragment); - } else if (previousPartitionPathNames != null && Arrays.equals(previousPartitionPathNames, parentPathNames)) { - fragments.add(fileFragment); - } else { - break; - } - previousPartitionPathNames = parentPathNames; - } else { - fragments.add(fileFragment); - } - } - return FragmentConvertor.toFragmentProtoArray(fragments.toArray(new FileFragment[]{})); - } - - private static void getNonZeroLengthDataFiles(FileSystem fs, Path path, List result, - int startFileIndex, int numResultFiles, - AtomicInteger currentFileIndex) throws IOException { - if (fs.isDirectory(path)) { - FileStatus[] files = fs.listStatus(path, StorageManager.hiddenFileFilter); - if (files != null && files.length > 0) { - for (FileStatus eachFile : files) { - if (result.size() >= numResultFiles) { - return; - } - if (eachFile.isDirectory()) { - getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles, - currentFileIndex); - } else if (eachFile.isFile() && eachFile.getLen() > 0) { - if (currentFileIndex.get() >= startFileIndex) { - result.add(eachFile); - } - currentFileIndex.incrementAndGet(); - } - } - } - } else { - FileStatus fileStatus = fs.getFileStatus(path); - if (fileStatus != null && fileStatus.getLen() > 0) { - if (currentFileIndex.get() >= startFileIndex) { - result.add(fileStatus); - } - currentFileIndex.incrementAndGet(); - if (result.size() >= numResultFiles) { - return; - } - } - } - } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/utils/SchemaUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/SchemaUtil.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/utils/SchemaUtil.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/util/SchemaUtil.java index 981b572c46..78135abefb 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/utils/SchemaUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/SchemaUtil.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.utils; +package org.apache.tajo.plan.util; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java index 5cb1fd96ae..ed8a702d8f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java @@ -16,11 +16,12 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.verifier; import org.apache.tajo.catalog.Column; -import org.apache.tajo.engine.eval.*; -import org.apache.tajo.engine.planner.logical.LogicalNode; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.LogicalNode; import java.util.Set; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java similarity index 94% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java index c9932fa8e8..b6912a7af3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/LogicalPlanVerifier.java @@ -16,15 +16,20 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.verifier; import com.google.common.base.Preconditions; +import org.apache.tajo.OverridableConf; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.engine.planner.logical.*; -import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.Target; +import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import java.util.Stack; @@ -38,16 +43,17 @@ public LogicalPlanVerifier(TajoConf conf, CatalogService catalog) { } public static class Context { - QueryContext queryContext; + OverridableConf queryContext; VerificationState state; - public Context(QueryContext queryContext, VerificationState state) { + public Context(OverridableConf queryContext, VerificationState state) { this.queryContext = this.queryContext; this.state = state; } } - public VerificationState verify(QueryContext queryContext, VerificationState state, LogicalPlan plan) throws PlanningException { + public VerificationState verify(OverridableConf queryContext, VerificationState state, LogicalPlan plan) + throws PlanningException { Context context = new Context(queryContext, state); visit(context, plan, plan.getRootBlock()); return context.state; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java similarity index 92% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index f6d5540320..95e0f30aa9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -16,21 +16,25 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.verifier; +import org.apache.tajo.OverridableConf; +import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.plan.util.ExprFinder; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; import org.apache.tajo.util.TUtil; import java.util.Set; import java.util.Stack; -public class PreLogicalPlanVerifier extends BaseAlgebraVisitor { +public class PreLogicalPlanVerifier extends BaseAlgebraVisitor { private CatalogService catalog; public PreLogicalPlanVerifier(CatalogService catalog) { @@ -38,16 +42,17 @@ public PreLogicalPlanVerifier(CatalogService catalog) { } public static class Context { - QueryContext queryContext; + OverridableConf queryContext; VerificationState state; - public Context(QueryContext queryContext, VerificationState state) { + public Context(OverridableConf queryContext, VerificationState state) { this.queryContext = queryContext; this.state = state; } } - public VerificationState verify(QueryContext queryContext, VerificationState state, Expr expr) throws PlanningException { + public VerificationState verify(OverridableConf queryContext, VerificationState state, Expr expr) + throws PlanningException { Context context = new Context(queryContext, state); visit(context, new Stack(), expr); return context.state; @@ -141,7 +146,7 @@ private boolean assertRelationExistence(Context context, String tableName) { if (CatalogUtil.isFQTableName(tableName)) { qualifiedName = tableName; } else { - qualifiedName = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), tableName); + qualifiedName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), tableName); } if (!catalog.existsTable(qualifiedName)) { @@ -157,7 +162,7 @@ private boolean assertRelationNoExistence(Context context, String tableName) { if (CatalogUtil.isFQTableName(tableName)) { qualifiedName = tableName; } else { - qualifiedName = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), tableName); + qualifiedName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), tableName); } if(qualifiedName == null) { System.out.println("A"); @@ -263,7 +268,7 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws if (expr.hasTableName()) { String qualifiedName = expr.getTableName(); if (TajoConstants.EMPTY_STRING.equals(CatalogUtil.extractQualifier(expr.getTableName()))) { - qualifiedName = CatalogUtil.buildFQName(context.queryContext.getCurrentDatabase(), + qualifiedName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), expr.getTableName()); } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/VerificationState.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/VerificationState.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/VerificationState.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/verifier/VerificationState.java index 18882b8ad9..a27b2007d7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/VerificationState.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/VerificationState.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.verifier; import com.google.common.collect.Lists; import org.apache.commons.logging.Log; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/exception/VerifyException.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/VerifyException.java similarity index 90% rename from tajo-core/src/main/java/org/apache/tajo/engine/exception/VerifyException.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/verifier/VerifyException.java index beda78742b..2e7a9c273a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/exception/VerifyException.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/VerifyException.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package org.apache.tajo.engine.exception; +package org.apache.tajo.plan.verifier; -import org.apache.tajo.engine.planner.PlanningException; +import org.apache.tajo.plan.PlanningException; public class VerifyException extends PlanningException { public VerifyException(String message) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java index a4e90b432b..89eb4a8475 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/BasicLogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/BasicLogicalPlanVisitor.java @@ -16,9 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.visitor; -import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.*; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java index a7e5375c69..7065295db4 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/ExplainLogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/ExplainLogicalPlanVisitor.java @@ -16,10 +16,13 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.visitor; import org.apache.tajo.annotation.Nullable; -import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanString; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.*; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java similarity index 97% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java index 0a36610476..29807f179e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/LogicalPlanVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/LogicalPlanVisitor.java @@ -16,9 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.visitor; -import org.apache.tajo.engine.planner.logical.*; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.logical.*; import java.util.Stack; diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/SimpleAlgebraVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/SimpleAlgebraVisitor.java similarity index 98% rename from tajo-core/src/main/java/org/apache/tajo/engine/planner/SimpleAlgebraVisitor.java rename to tajo-plan/src/main/java/org/apache/tajo/plan/visitor/SimpleAlgebraVisitor.java index 8b34189efe..ad4ef1834d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/SimpleAlgebraVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/visitor/SimpleAlgebraVisitor.java @@ -16,9 +16,11 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan.visitor; import org.apache.tajo.algebra.*; +import org.apache.tajo.plan.PlanningException; +import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; import java.util.Stack; diff --git a/tajo-core/src/main/proto/Plan.proto b/tajo-plan/src/main/proto/Plan.proto similarity index 94% rename from tajo-core/src/main/proto/Plan.proto rename to tajo-plan/src/main/proto/Plan.proto index fa8cbb8c44..8639117c81 100644 --- a/tajo-core/src/main/proto/Plan.proto +++ b/tajo-plan/src/main/proto/Plan.proto @@ -16,9 +16,7 @@ * limitations under the License. */ -package tajo.plan; -option cc_generic_services = false; -option java_package = "org.apache.tajo.engine.plan.proto"; +option java_package = "org.apache.tajo.plan.serder"; option java_outer_classname = "PlanProto"; option java_generic_services = false; option java_generate_equals_and_hash = true; @@ -206,4 +204,17 @@ message Datum { message Interval { optional int32 month = 1 [default = 0]; optional int64 msec = 2; +} + +enum ShuffleType { + NONE_SHUFFLE = 0; + HASH_SHUFFLE = 1; + RANGE_SHUFFLE = 2; + SCATTERED_HASH_SHUFFLE = 3; +} + +enum TransmitType { + PUSH_TRANSMIT = 0; + PULL_TRANSMIT = 1; + FILE_WRITE = 2; } \ No newline at end of file diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestExprAnnotator.java b/tajo-plan/src/test/java/org/apache/tajo/plan/TestExprAnnotator.java similarity index 98% rename from tajo-core/src/test/java/org/apache/tajo/engine/planner/TestExprAnnotator.java rename to tajo-plan/src/test/java/org/apache/tajo/plan/TestExprAnnotator.java index bd88cd694e..cdf4c1ba9f 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestExprAnnotator.java +++ b/tajo-plan/src/test/java/org/apache/tajo/plan/TestExprAnnotator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; import org.apache.tajo.catalog.CatalogUtil; import org.junit.Test; diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalNode.java b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java similarity index 85% rename from tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalNode.java rename to tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java index 3fe75f0dc2..bff7efb0cd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/planner/TestLogicalNode.java +++ b/tajo-plan/src/test/java/org/apache/tajo/plan/TestLogicalNode.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package org.apache.tajo.engine.planner; +package org.apache.tajo.plan; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.CatalogUtil; @@ -24,20 +24,16 @@ import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.Type; -import org.apache.tajo.engine.planner.logical.GroupbyNode; -import org.apache.tajo.engine.planner.logical.JoinNode; -import org.apache.tajo.engine.planner.logical.LogicalNode; -import org.apache.tajo.engine.planner.logical.ScanNode; +import org.apache.tajo.plan.logical.GroupbyNode; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.ScanNode; import org.junit.Test; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class TestLogicalNode { - public static final void testCloneLogicalNode(LogicalNode n1) throws CloneNotSupportedException { - LogicalNode copy = (LogicalNode) n1.clone(); - assertTrue(n1.deepEquals(copy)); - } @Test public void testEquals() { diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 908fd1840d..97489daa55 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -727,6 +727,11 @@ tajo-core ${tajo.version} + + org.apache.tajo + tajo-plan + ${tajo.version} + org.apache.tajo tajo-core diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java index 07fa16b71c..1789cc987b 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/StorageUtil.java @@ -111,26 +111,6 @@ public static Path concatPath(Path parent, String...childs) { return new Path(parent, sb.toString()); } - public static KeyValueSet newPhysicalProperties(CatalogProtos.StoreType type) { - KeyValueSet options = new KeyValueSet(); - if (CatalogProtos.StoreType.CSV == type) { - options.set(CSVFILE_DELIMITER, DEFAULT_FIELD_DELIMITER); - } else if (CatalogProtos.StoreType.RCFILE == type) { - options.set(RCFILE_SERDE, DEFAULT_BINARY_SERDE); - } else if (CatalogProtos.StoreType.SEQUENCEFILE == type) { - options.set(SEQUENCEFILE_SERDE, DEFAULT_TEXT_SERDE); - options.set(SEQUENCEFILE_DELIMITER, DEFAULT_FIELD_DELIMITER); - } else if (type == CatalogProtos.StoreType.PARQUET) { - options.set(ParquetOutputFormat.BLOCK_SIZE, PARQUET_DEFAULT_BLOCK_SIZE); - options.set(ParquetOutputFormat.PAGE_SIZE, PARQUET_DEFAULT_PAGE_SIZE); - options.set(ParquetOutputFormat.COMPRESSION, PARQUET_DEFAULT_COMPRESSION_CODEC_NAME); - options.set(ParquetOutputFormat.ENABLE_DICTIONARY, PARQUET_DEFAULT_IS_DICTIONARY_ENABLED); - options.set(ParquetOutputFormat.VALIDATION, PARQUET_DEFAULT_IS_VALIDATION_ENABLED); - } - - return options; - } - static final String fileNamePatternV08 = "part-[0-9]*-[0-9]*"; static final String fileNamePatternV09 = "part-[0-9]*-[0-9]*-[0-9]*"; diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java index c1a96a5d08..51c612c242 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java @@ -108,7 +108,7 @@ public void testMultipleFiles() throws IOException { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(StorageUtil.newPhysicalProperties(storeType)); + meta.setOptions(CatalogUtil.newPhysicalProperties(storeType)); if (storeType == StoreType.AVRO) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_MULTIPLE_FILES_AVRO_SCHEMA); diff --git a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java index ef5388cde4..dca21af41e 100644 --- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java +++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java @@ -246,7 +246,7 @@ public void testProjection() throws IOException { schema.addColumn("score", Type.FLOAT4); TableMeta meta = CatalogUtil.newTableMeta(storeType); - meta.setOptions(StorageUtil.newPhysicalProperties(storeType)); + meta.setOptions(CatalogUtil.newPhysicalProperties(storeType)); if (storeType == StoreType.AVRO) { meta.putOption(StorageConstants.AVRO_SCHEMA_LITERAL, TEST_PROJECTION_AVRO_SCHEMA); @@ -314,7 +314,7 @@ public void testVariousTypes() throws IOException { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(StorageUtil.newPhysicalProperties(storeType)); + meta.setOptions(CatalogUtil.newPhysicalProperties(storeType)); if (storeType == StoreType.AVRO) { String path = FileUtil.getResourcePath("testVariousTypes.avsc").toString(); meta.putOption(StorageConstants.AVRO_SCHEMA_URL, path); @@ -380,7 +380,7 @@ public void testNullHandlingTypes() throws IOException { KeyValueSet options = new KeyValueSet(); TableMeta meta = CatalogUtil.newTableMeta(storeType, options); - meta.setOptions(StorageUtil.newPhysicalProperties(storeType)); + meta.setOptions(CatalogUtil.newPhysicalProperties(storeType)); meta.putOption(StorageConstants.CSVFILE_NULL, "\\\\N"); meta.putOption(StorageConstants.RCFILE_NULL, "\\\\N"); meta.putOption(StorageConstants.RCFILE_SERDE, TextSerializerDeserializer.class.getName()); diff --git a/tajo-yarn-pullserver/src/main/java/org/apache/tajo/storage/Tuple.java b/tajo-yarn-pullserver/src/main/java/org/apache/tajo/storage/Tuple.java deleted file mode 100644 index 7b5d9c0273..0000000000 --- a/tajo-yarn-pullserver/src/main/java/org/apache/tajo/storage/Tuple.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.tajo.datum.Datum; -import org.apache.tajo.datum.ProtobufDatum; - -public interface Tuple extends Cloneable { - - public int size(); - - public boolean contains(int fieldid); - - public boolean isNull(int fieldid); - - public void clear(); - - public void put(int fieldId, Datum value); - - public void put(int fieldId, Datum[] values); - - public void put(int fieldId, Tuple tuple); - - public void put(Datum[] values); - - public Datum get(int fieldId); - - public void setOffset(long offset); - - public long getOffset(); - - @SuppressWarnings("unused") - public boolean getBool(int fieldId); - - @SuppressWarnings("unused") - public byte getByte(int fieldId); - - @SuppressWarnings("unused") - public char getChar(int fieldId); - - public byte [] getBytes(int fieldId); - - @SuppressWarnings("unused") - public short getInt2(int fieldId); - - @SuppressWarnings("unused") - public int getInt4(int fieldId); - - @SuppressWarnings("unused") - public long getInt8(int fieldId); - - @SuppressWarnings("unused") - public float getFloat4(int fieldId); - - @SuppressWarnings("unused") - public double getFloat8(int fieldId); - - @SuppressWarnings("unused") - public String getText(int fieldId); - - @SuppressWarnings("unused") - public ProtobufDatum getProtobufDatum(int field); - - public Tuple clone() throws CloneNotSupportedException; - - public Datum[] getValues(); -} From 3c3bcce8e3ddac4480499be3edae69bb0d2554a3 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Sat, 25 Oct 2014 12:37:03 -0700 Subject: [PATCH 09/15] TAJO-1125: Separate logical plan and optimizer into a maven module. (missed jar copy) --- tajo-dist/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml index 24b1831699..d350889bf0 100644 --- a/tajo-dist/pom.xml +++ b/tajo-dist/pom.xml @@ -126,6 +126,7 @@ run cp -r $ROOT/tajo-common/target/tajo-common-${project.version}/* . run cp -r $ROOT/tajo-rpc/target/tajo-rpc-${project.version}/* . run cp -r $ROOT/tajo-algebra/target/tajo-algebra-${project.version}/* . + run cp -r $ROOT/tajo-plan/target/tajo-plan-${project.version}/* . run cp -r $ROOT/tajo-client/target/tajo-client-${project.version}/* . run cp -r $ROOT/tajo-catalog/target/tajo-catalog-${project.version}/* . run cp -r $ROOT/tajo-storage/target/tajo-storage-${project.version}/* . From b953102a48bb75136172a584d95191273a386903 Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Fri, 17 Oct 2014 18:53:13 +0900 Subject: [PATCH 10/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- .../tajo/validation/AbstractValidator.java | 60 ++++ .../tajo/validation/ConstraintViolation.java | 47 +++ .../ConstraintViolationException.java | 51 +++ .../tajo/validation/GroupValidator.java | 49 +++ .../tajo/validation/LengthValidator.java | 57 ++++ .../apache/tajo/validation/MaxValidator.java | 79 +++++ .../apache/tajo/validation/MinValidator.java | 84 +++++ .../tajo/validation/NotNullValidator.java | 47 +++ .../tajo/validation/PatternValidator.java | 58 ++++ .../tajo/validation/RangeValidator.java | 53 +++ .../org/apache/tajo/validation/Validator.java | 29 ++ .../apache/tajo/validation/Validators.java | 53 +++ .../tajo/conf/TestValidatorsForTajoConf.java | 26 ++ .../tajo/validation/TestValidators.java | 302 ++++++++++++++++++ 14 files changed, 995 insertions(+) create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/AbstractValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolation.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/GroupValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/LengthValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/MinValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/NotNullValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/RangeValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/Validator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/Validators.java create mode 100644 tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java create mode 100644 tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/AbstractValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/AbstractValidator.java new file mode 100644 index 0000000000..0f7082bc01 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/AbstractValidator.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; + +import org.apache.tajo.util.TUtil; + +public abstract class AbstractValidator implements Validator { + + protected abstract String getErrorMessage(T object); + protected abstract boolean validateInternal(T object); + protected abstract Collection getDependantValidators(); + + @Override + public Collection validate(T object) { + Collection violations = TUtil.newHashSet(); + + if (!validateInternal(object)) { + ConstraintViolation violation = new ConstraintViolation(); + violation.setMessage(getErrorMessage(object)); + violation.setValidatorClazz(this.getClass()); + violations.add(violation); + } + + for (Validator dependantValidator: getDependantValidators()) { + violations.addAll(dependantValidator.validate(object)); + } + + return violations; + } + + @Override + public void validate(T object, boolean generateThrow) { + Collection violations = validate(object); + + if (violations.size() > 0) { + if (generateThrow) { + throw new ConstraintViolationException(violations); + } + } + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolation.java b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolation.java new file mode 100644 index 0000000000..8d84aa7636 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolation.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +public class ConstraintViolation { + + private String message; + private Class validatorClazz; + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public Class getValidatorClazz() { + return validatorClazz; + } + + public void setValidatorClazz(Class validatorClazz) { + this.validatorClazz = validatorClazz; + } + + @Override + public String toString() { + return "ConstraintViolation [message=" + message + ", validatorClazz=" + validatorClazz + "]"; + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java new file mode 100644 index 0000000000..664ec09771 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; + +public class ConstraintViolationException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final Collection violations; + + public ConstraintViolationException(Collection violations) { + this.violations = violations; + } + + public Collection getViolations() { + return violations; + } + + @Override + public String getMessage() { + if (violations != null) { + String errorMessage = "ConstraintViolationException ["; + for (ConstraintViolation violation: violations) { + errorMessage += violation.getMessage() + ","; + } + errorMessage += "]"; + return errorMessage; + } else { + return super.getMessage(); + } + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/GroupValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/GroupValidator.java new file mode 100644 index 0000000000..db2ba51e1e --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/GroupValidator.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; + +public class GroupValidator extends AbstractValidator { + + private final Collection dependants; + + public GroupValidator(Collection validators) { + if (validators.size() == 0) { + throw new IllegalArgumentException("Needs at least 1 or more validators."); + } + this.dependants = validators; + } + + @Override + protected String getErrorMessage(T object) { + return ""; + } + + @Override + protected boolean validateInternal(T object) { + return true; + } + + @Override + protected Collection getDependantValidators() { + return dependants; + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/LengthValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/LengthValidator.java new file mode 100644 index 0000000000..e53634b167 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/LengthValidator.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class LengthValidator extends AbstractValidator { + + private final int maxLength; + + public LengthValidator(int maxLen) { + this.maxLength = maxLen; + } + + @Override + protected String getErrorMessage(T object) { + return "Length of " + object + " is greater than " + maxLength; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof CharSequence) { + result = ((CharSequence)object).length() <= maxLength; + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java new file mode 100644 index 0000000000..63c2352b09 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collection; +import java.util.Collections; + +import org.apache.commons.lang.math.NumberUtils; + +public class MaxValidator extends AbstractValidator { + + private final String maxValue; + + public MaxValidator(String maxValue) { + if (!NumberUtils.isNumber(maxValue)) { + throw new IllegalArgumentException(maxValue + " is not a Java number."); + } + + this.maxValue = maxValue; + } + + @Override + protected String getErrorMessage(T object) { + return object + " should be less than " + maxValue; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if ((object instanceof Byte) || (object instanceof Short) || (object instanceof Integer)) { + Integer objInteger = Integer.decode(object.toString()); + Integer maxInteger = Integer.decode(maxValue); + result = objInteger.compareTo(maxInteger) <= 0; + } else if (object instanceof Long) { + Long objLong = Long.decode(object.toString()); + Long maxLong = Long.decode(maxValue); + result = objLong.compareTo(maxLong) <= 0; + } else if (object instanceof BigInteger) { + BigInteger objInteger = (BigInteger) object; + BigInteger maxInteger = new BigInteger(maxValue); + result = objInteger.compareTo(maxInteger) <= 0; + } else if (object instanceof BigDecimal) { + BigDecimal objDecimal = (BigDecimal) object; + BigDecimal maxDecimal = new BigDecimal(maxValue); + result = objDecimal.compareTo(maxDecimal) <= 0; + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/MinValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/MinValidator.java new file mode 100644 index 0000000000..431fe9b26f --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/MinValidator.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collection; +import java.util.Collections; + +import org.apache.commons.lang.math.NumberUtils; + +public class MinValidator extends AbstractValidator { + + private final String minValue; + + public MinValidator(String minValue) { + if (!NumberUtils.isNumber(minValue)) { + throw new IllegalArgumentException(minValue + " is not a Java Number."); + } + + this.minValue = minValue; + } + + @Override + protected String getErrorMessage(T object) { + return object + " should be greater than " + minValue; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if ((object instanceof Byte) || (object instanceof Short) || (object instanceof Integer)) { + Integer objInteger = Integer.decode(object.toString()); + Integer minInteger = Integer.decode(minValue); + result = objInteger.compareTo(minInteger) >= 0; + } else if (object instanceof Long) { + Long objLong = Long.decode(object.toString()); + Long minLong = Long.decode(minValue); + result = objLong.compareTo(minLong) >= 0; + } else if ((object instanceof Float) || (object instanceof Double)) { + Double objDouble = Double.valueOf(object.toString()); + Double minDouble = Double.valueOf(minValue); + result = objDouble.compareTo(minDouble) >= 0; + } else if (object instanceof BigInteger) { + BigInteger objInteger = (BigInteger) object; + BigInteger minInteger = new BigInteger(minValue); + result = objInteger.compareTo(minInteger) >= 0; + } else if (object instanceof BigDecimal) { + BigDecimal objDecimal = (BigDecimal) object; + BigDecimal minDecimal = new BigDecimal(minValue); + result = objDecimal.compareTo(minDecimal) >= 0; + } + } + else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/NotNullValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/NotNullValidator.java new file mode 100644 index 0000000000..d0ace1ffa3 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/NotNullValidator.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class NotNullValidator extends AbstractValidator { + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + + @Override + protected String getErrorMessage(T object) { + return "Object is null"; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + result = true; + } + + return result; + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java new file mode 100644 index 0000000000..0c3a99a2a2 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; +import java.util.regex.Pattern; + +public class PatternValidator extends AbstractValidator { + + private final Pattern pattern; + + public PatternValidator(String patternString) { + pattern = Pattern.compile(patternString); + } + + @Override + protected String getErrorMessage(T object) { + return object + " does not match to the " + pattern.pattern(); + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof CharSequence) { + result = pattern.matcher((CharSequence) object).find(); + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/RangeValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/RangeValidator.java new file mode 100644 index 0000000000..ae6fd76a06 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/RangeValidator.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class RangeValidator extends AbstractValidator { + + private final String minValue; + private final AbstractValidator minValidator; + private final String maxValue; + private final AbstractValidator maxValidator; + + public RangeValidator(String minValue, String maxValue) { + this.minValue = minValue; + this.minValidator = new MinValidator(minValue); + this.maxValue = maxValue; + this.maxValidator = new MaxValidator(maxValue); + } + + @Override + protected String getErrorMessage(T object) { + return object + " is not a range of " + minValue + " and " + maxValue; + } + + @Override + protected boolean validateInternal(T object) { + return this.minValidator.validateInternal(object) & this.maxValidator.validateInternal(object); + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validator.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validator.java new file mode 100644 index 0000000000..8f16fda2aa --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validator.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; + +public interface Validator { + + public Collection validate(T object); + + public void validate(T object, boolean generateThrow); + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java new file mode 100644 index 0000000000..26056ad9ce --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import org.apache.tajo.util.TUtil; + +public class Validators { + + public static Validator groups(Validator...validators) { + return new GroupValidator(TUtil.newHashSet(validators)); + } + + public static Validator length(int maxLength) { + return new LengthValidator(maxLength); + } + + public static Validator max(String maxValue) { + return new MaxValidator(maxValue); + } + + public static Validator min(String minValue) { + return new MinValidator(minValue); + } + + public static Validator notNull() { + return new NotNullValidator(); + } + + public static Validator patternMatch(String regex) { + return new PatternValidator(regex); + } + + public static Validator range(String minValue, String maxValue) { + return new RangeValidator(minValue, maxValue); + } + +} diff --git a/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java b/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java new file mode 100644 index 0000000000..db5aae77c0 --- /dev/null +++ b/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java @@ -0,0 +1,26 @@ +package org.apache.tajo.conf; + +import static org.junit.Assert.*; + +import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.validation.ConstraintViolationException; +import org.junit.Test; + +public class TestValidatorsForTajoConf { + + @Test(expected=ConstraintViolationException.class) + public void testPathValidatorsForTajoConf_setVar() { + TajoConf conf = new TajoConf(); + + TajoConf.setVar(conf, ConfVars.ROOT_DIR, "Invalid path"); + fail(); + } + + @Test + public void testValidPathForTajoConf() { + TajoConf conf = new TajoConf(); + + TajoConf.setVar(conf, ConfVars.ROOT_DIR, "file:///tmp/tajo-${user.name}/"); + } + +} diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java new file mode 100644 index 0000000000..facb8c89c5 --- /dev/null +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -0,0 +1,302 @@ +package org.apache.tajo.validation; + +import static org.junit.Assert.*; +import static org.hamcrest.CoreMatchers.*; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; +import java.util.Collection; +import java.util.Random; +import java.util.UUID; + +import org.apache.tajo.util.TUtil; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.junit.Test; + +public class TestValidators { + + private class ValidatorClazzMatcher extends org.hamcrest.TypeSafeDiagnosingMatcher { + + private final org.hamcrest.Matcher> matcher; + + public ValidatorClazzMatcher(Matcher> matcher) { + this.matcher = matcher; + } + + @Override + public void describeTo(Description description) { + description + .appendText("validatorClazz property of ConstraintViolation class containing ") + .appendDescriptionOf(matcher); + } + + @Override + protected boolean matchesSafely(ConstraintViolation item, Description mismatchDescription) { + if (matcher.matches(item.getValidatorClazz())) { + return true; + } + matcher.describeMismatch(item, mismatchDescription); + return false; + } + + } + + private org.hamcrest.Matcher + hasAClass(Matcher> matcher) { + return new ValidatorClazzMatcher(matcher); + } + + @Test + public void testNotNullValidator() { + Object testValue = null; + + assertThat(new NotNullValidator().validateInternal(testValue), is(false)); + assertThat(new NotNullValidator().validate(testValue).size(), is(1)); + assertThat(new NotNullValidator().validate(testValue), + hasItem(new ValidatorClazzMatcher(equalTo(NotNullValidator.class)))); + } + + @Test + public void testMinValidator() { + byte byteValue; + short shortValue; + int intValue; + long longValue; + float floatValue; + double doubleValue; + BigInteger bigIntegerValue; + BigDecimal bigDecimalValue; + + byteValue = 2; + assertThat(new MinValidator(Byte.toString(Byte.MIN_VALUE)).validateInternal(byteValue), is(true)); + assertThat(new MinValidator(Byte.toString(Byte.MIN_VALUE)).validate(byteValue).size(), is(0)); + assertThat(new MinValidator(Byte.toString(Byte.MAX_VALUE)).validateInternal(byteValue), is(false)); + assertThat(new MinValidator(Byte.toString(Byte.MAX_VALUE)).validate(byteValue).size(), is(1)); + assertThat(new MinValidator(Byte.toString(Byte.MAX_VALUE)).validate(byteValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + shortValue = 3; + assertThat(new MinValidator(Short.toString(Short.MIN_VALUE)).validateInternal(shortValue), is(true)); + assertThat(new MinValidator(Short.toString(Short.MIN_VALUE)).validate(shortValue).size(), is(0)); + assertThat(new MinValidator(Short.toString(Short.MAX_VALUE)).validateInternal(shortValue), is(false)); + assertThat(new MinValidator(Short.toString(Short.MAX_VALUE)).validate(shortValue).size(), is(1)); + assertThat(new MinValidator(Short.toString(Short.MAX_VALUE)).validate(shortValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + intValue = 4; + assertThat(new MinValidator(Integer.toString(Integer.MIN_VALUE)).validateInternal(intValue), is(true)); + assertThat(new MinValidator(Integer.toString(Integer.MIN_VALUE)).validate(intValue).size(), is(0)); + assertThat(new MinValidator(Integer.toString(Integer.MAX_VALUE)).validateInternal(intValue), is(false)); + assertThat(new MinValidator(Integer.toString(Integer.MAX_VALUE)).validate(intValue).size(), is(1)); + assertThat(new MinValidator(Integer.toString(Integer.MAX_VALUE)).validate(intValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + longValue = 5; + assertThat(new MinValidator(Long.toString(Long.MIN_VALUE)).validateInternal(longValue), is(true)); + assertThat(new MinValidator(Long.toString(Long.MIN_VALUE)).validate(longValue).size(), is(0)); + assertThat(new MinValidator(Long.toString(Long.MAX_VALUE)).validateInternal(longValue), is(false)); + assertThat(new MinValidator(Long.toString(Long.MAX_VALUE)).validate(longValue).size(), is(1)); + assertThat(new MinValidator(Long.toString(Long.MAX_VALUE)).validate(longValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + floatValue = 4.7f; + assertThat(new MinValidator(Float.toString(Float.MIN_VALUE)).validateInternal(floatValue), is(true)); + assertThat(new MinValidator(Float.toString(Float.MIN_VALUE)).validate(floatValue).size(), is(0)); + assertThat(new MinValidator(Float.toString(Float.MAX_VALUE)).validateInternal(floatValue), is(false)); + assertThat(new MinValidator(Float.toString(Float.MAX_VALUE)).validate(floatValue).size(), is(1)); + assertThat(new MinValidator(Float.toString(Float.MAX_VALUE)).validate(floatValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + doubleValue = 7.5e10; + assertThat(new MinValidator(Double.toString(Double.MIN_VALUE)).validateInternal(doubleValue), is(true)); + assertThat(new MinValidator(Double.toString(Double.MIN_VALUE)).validate(doubleValue).size(), is(0)); + assertThat(new MinValidator(Double.toString(Double.MAX_VALUE)).validateInternal(doubleValue), is(false)); + assertThat(new MinValidator(Double.toString(Double.MAX_VALUE)).validate(doubleValue).size(), is(1)); + assertThat(new MinValidator(Double.toString(Double.MAX_VALUE)).validate(doubleValue), + hasItem(hasAClass(equalTo(MinValidator.class)))); + + bigIntegerValue = new BigInteger(10, new Random()); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validateInternal(bigIntegerValue), is(true)); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validate(bigIntegerValue).size(), is(0)); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validateInternal(bigIntegerValue), is(false)); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validate(bigIntegerValue).size(), is(1)); + assertThat(new MinValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validate(bigIntegerValue), hasItem(hasAClass(equalTo(MinValidator.class)))); + + bigDecimalValue = new BigDecimal(new BigInteger(10, new Random()), MathContext.DECIMAL64); + assertThat(new MinValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validateInternal(bigDecimalValue), is(true)); + assertThat(new MinValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validate(bigDecimalValue).size(), is(0)); + assertThat(new MinValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validateInternal(bigDecimalValue), is(false)); + assertThat(new MinValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validate(bigDecimalValue).size(), is(1)); + assertThat(new MinValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validate(bigDecimalValue), hasItem(hasAClass(equalTo(MinValidator.class)))); + } + + @Test + public void testMaxValidator() { + byte byteValue; + short shortValue; + int intValue; + long longValue; + float floatValue; + double doubleValue; + BigInteger bigIntegerValue; + BigDecimal bigDecimalValue; + + byteValue = 2; + assertThat(new MaxValidator(Byte.toString(Byte.MAX_VALUE)).validateInternal(byteValue), is(true)); + assertThat(new MaxValidator(Byte.toString(Byte.MAX_VALUE)).validate(byteValue).size(), is(0)); + assertThat(new MaxValidator(Byte.toString(Byte.MIN_VALUE)).validateInternal(byteValue), is(false)); + assertThat(new MaxValidator(Byte.toString(Byte.MIN_VALUE)).validate(byteValue).size(), is(1)); + assertThat(new MaxValidator(Byte.toString(Byte.MIN_VALUE)).validate(byteValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + shortValue = 3; + assertThat(new MaxValidator(Short.toString(Short.MAX_VALUE)).validateInternal(shortValue), is(true)); + assertThat(new MaxValidator(Short.toString(Short.MAX_VALUE)).validate(shortValue).size(), is(0)); + assertThat(new MaxValidator(Short.toString(Short.MIN_VALUE)).validateInternal(shortValue), is(false)); + assertThat(new MaxValidator(Short.toString(Short.MIN_VALUE)).validate(shortValue).size(), is(1)); + assertThat(new MaxValidator(Short.toString(Short.MIN_VALUE)).validate(shortValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + intValue = 4; + assertThat(new MaxValidator(Integer.toString(Integer.MAX_VALUE)).validateInternal(intValue), is(true)); + assertThat(new MaxValidator(Integer.toString(Integer.MAX_VALUE)).validate(intValue).size(), is(0)); + assertThat(new MaxValidator(Integer.toString(Integer.MIN_VALUE)).validateInternal(intValue), is(false)); + assertThat(new MaxValidator(Integer.toString(Integer.MIN_VALUE)).validate(intValue).size(), is(1)); + assertThat(new MaxValidator(Integer.toString(Integer.MIN_VALUE)).validate(intValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + longValue = 5; + assertThat(new MaxValidator(Long.toString(Long.MAX_VALUE)).validateInternal(longValue), is(true)); + assertThat(new MaxValidator(Long.toString(Long.MAX_VALUE)).validate(longValue).size(), is(0)); + assertThat(new MaxValidator(Long.toString(Long.MIN_VALUE)).validateInternal(longValue), is(false)); + assertThat(new MaxValidator(Long.toString(Long.MIN_VALUE)).validate(longValue).size(), is(1)); + assertThat(new MaxValidator(Long.toString(Long.MIN_VALUE)).validate(longValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + floatValue = 4.7f; + assertThat(new MaxValidator(Float.toString(Float.MAX_VALUE)).validateInternal(floatValue), is(true)); + assertThat(new MaxValidator(Float.toString(Float.MAX_VALUE)).validate(floatValue).size(), is(0)); + assertThat(new MaxValidator(Float.toString(Float.MIN_VALUE)).validateInternal(floatValue), is(false)); + assertThat(new MaxValidator(Float.toString(Float.MIN_VALUE)).validate(floatValue).size(), is(1)); + assertThat(new MaxValidator(Float.toString(Float.MIN_VALUE)).validate(floatValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + doubleValue = 7.5e10; + assertThat(new MaxValidator(Double.toString(Double.MAX_VALUE)).validateInternal(doubleValue), is(true)); + assertThat(new MaxValidator(Double.toString(Double.MAX_VALUE)).validate(doubleValue).size(), is(0)); + assertThat(new MaxValidator(Double.toString(Double.MIN_VALUE)).validateInternal(doubleValue), is(false)); + assertThat(new MaxValidator(Double.toString(Double.MIN_VALUE)).validate(doubleValue).size(), is(1)); + assertThat(new MaxValidator(Double.toString(Double.MIN_VALUE)).validate(doubleValue), + hasItem(hasAClass(equalTo(MaxValidator.class)))); + + bigIntegerValue = new BigInteger(10, new Random()); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validateInternal(bigIntegerValue), is(true)); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MAX_VALUE)).toString(10)) + .validate(bigIntegerValue).size(), is(0)); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validateInternal(bigIntegerValue), is(false)); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validate(bigIntegerValue).size(), is(1)); + assertThat(new MaxValidator(new BigInteger(Long.toString(Long.MIN_VALUE)).toString(10)) + .validate(bigIntegerValue), hasItem(hasAClass(equalTo(MaxValidator.class)))); + + bigDecimalValue = new BigDecimal(new BigInteger(10, new Random()), MathContext.DECIMAL64); + assertThat(new MaxValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validateInternal(bigDecimalValue), is(true)); + assertThat(new MaxValidator(new BigDecimal(Double.MAX_VALUE).toString()) + .validate(bigDecimalValue).size(), is(0)); + assertThat(new MaxValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validateInternal(bigDecimalValue), is(false)); + assertThat(new MaxValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validate(bigDecimalValue).size(), is(1)); + assertThat(new MaxValidator(new BigDecimal(Double.MIN_VALUE).toString()) + .validate(bigDecimalValue), hasItem(hasAClass(equalTo(MaxValidator.class)))); + } + + @Test + public void testPatternValidator() { + String schemeString = "http://tajo.apache.org"; + assertThat(new PatternValidator("^([a-zA-Z])+://").validateInternal(schemeString), is(true)); + assertThat(new PatternValidator("^([a-zA-Z])+://").validate(schemeString).size(), is(0)); + assertThat(new PatternValidator("([a-zA-Z])+://$").validateInternal(schemeString), is(false)); + assertThat(new PatternValidator("([a-zA-Z])+://$").validate(schemeString).size(), is(1)); + assertThat(new PatternValidator("([a-zA-Z])+://$").validate(schemeString), + hasItem(hasAClass(equalTo(PatternValidator.class)))); + } + + @Test + public void testLengthValidator() { + String shortString = "12345"; + String longString = UUID.randomUUID().toString(); + + assertThat(new LengthValidator(10).validateInternal(shortString), is(true)); + assertThat(new LengthValidator(10).validate(shortString).size(), is(0)); + assertThat(new LengthValidator(3).validateInternal(shortString), is(false)); + assertThat(new LengthValidator(3).validate(shortString).size(), is(1)); + assertThat(new LengthValidator(3).validate(shortString), + hasItem(hasAClass(equalTo(LengthValidator.class)))); + + assertThat(new LengthValidator(40).validateInternal(longString), is(true)); + assertThat(new LengthValidator(40).validate(longString).size(), is(0)); + assertThat(new LengthValidator(10).validateInternal(longString), is(false)); + assertThat(new LengthValidator(10).validate(longString).size(), is(1)); + assertThat(new LengthValidator(10).validate(longString), + hasItem(hasAClass(equalTo(LengthValidator.class)))); + } + + @Test + public void testGroupValidator() { + String httpUrl = "http://tajo.apache.org"; + Collection validators = null; + + validators = TUtil.newHashSet(); + validators.add(new PatternValidator("^[a-zA-Z]+://")); + validators.add(new LengthValidator(255)); + assertThat(new GroupValidator(validators).validate(httpUrl).size(), is(0)); + assertThat(new GroupValidator(validators).validate("tajo").size(), is(2)); + assertThat(new GroupValidator(validators).validate("tajo"), + hasItem(hasAClass(equalTo(PatternValidator.class)))); + + validators = TUtil.newHashSet(); + validators.add(new PatternValidator("^[a-zA-Z]+://")); + validators.add(new LengthValidator(7)); + assertThat(new GroupValidator(validators).validate(httpUrl).size(), is(1)); + assertThat(new GroupValidator(validators).validate(httpUrl), + hasItem(hasAClass(equalTo(LengthValidator.class)))); + } + + @Test + public void testRangeValidator() { + int intValue = 5; + + assertThat(new RangeValidator("0", Integer.toString(Integer.MAX_VALUE)).validateInternal(intValue), + is(true)); + assertThat(new RangeValidator("1024", "2048").validateInternal(intValue), is(false)); + } + + @Test(expected=ConstraintViolationException.class) + public void testExceptionThrow() { + Collection validators = null; + + validators = TUtil.newHashSet(); + validators.add(new PatternValidator("^[a-zA-Z]+://")); + validators.add(new LengthValidator(255)); + new GroupValidator(validators).validate("tajo", true); + + fail(); + } + +} From c510067ce5665a431230e5e56efa574f014bb85d Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Mon, 20 Oct 2014 16:13:01 +0900 Subject: [PATCH 11/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- tajo-common/pom.xml | 5 + .../main/java/org/apache/tajo/ConfigKey.java | 6 + .../java/org/apache/tajo/OverridableConf.java | 15 ++ .../java/org/apache/tajo/SessionVars.java | 21 +++ .../java/org/apache/tajo/conf/TajoConf.java | 149 ++++++++++++++--- .../java/org/apache/tajo/util/NumberUtil.java | 21 +++ .../apache/tajo/validation/MaxValidator.java | 4 + .../tajo/validation/PatternValidator.java | 7 +- .../apache/tajo/validation/Validators.java | 16 ++ .../tajo/conf/TestValidatorsForTajoConf.java | 81 +++++++++- .../tajo/validation/TestValidators.java | 152 +++++++++++++++++- .../tajo/engine/query/QueryContext.java | 11 ++ 12 files changed, 463 insertions(+), 25 deletions(-) diff --git a/tajo-common/pom.xml b/tajo-common/pom.xml index d857e0e716..820220c8f1 100644 --- a/tajo-common/pom.xml +++ b/tajo-common/pom.xml @@ -108,6 +108,11 @@ + + + src/test/resources/org/apache/tajo/conf/** + + org.apache.maven.plugins diff --git a/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java index b9d51ec76d..6329e77930 100644 --- a/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java +++ b/tajo-common/src/main/java/org/apache/tajo/ConfigKey.java @@ -18,6 +18,8 @@ package org.apache.tajo; +import org.apache.tajo.validation.Validator; + public interface ConfigKey { // Client can set or change variables of this mode. @@ -49,4 +51,8 @@ public String getPrefix() { public String keyname(); public ConfigType type(); + + public Class valueClass(); + + public Validator validator(); } diff --git a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java index c9cf7fac2c..288d5775db 100644 --- a/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/OverridableConf.java @@ -73,6 +73,9 @@ public TajoConf getConf() { } public void setBool(ConfigKey key, boolean val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } setBool(key.keyname(), val); } @@ -100,6 +103,9 @@ public boolean getBool(ConfigKey key) { } public void setInt(ConfigKey key, int val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } setInt(key.keyname(), val); } @@ -125,6 +131,9 @@ public int getInt(ConfigKey key) { } public void setLong(ConfigKey key, long val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } setLong(key.keyname(), val); } @@ -150,6 +159,9 @@ public long getLong(ConfigKey key) { } public void setFloat(ConfigKey key, float val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } setFloat(key.keyname(), val); } @@ -175,6 +187,9 @@ public float getFloat(ConfigKey key) { } public void put(ConfigKey key, String val) { + if (key.validator() != null) { + key.validator().validate(val, true); + } set(key.keyname(), val); } diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index 1229849819..fb6b75ec36 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -22,6 +22,8 @@ import java.util.Map; +import org.apache.tajo.validation.Validator; + import static org.apache.tajo.SessionVars.VariableMode.*; import static org.apache.tajo.conf.TajoConf.ConfVars; @@ -143,6 +145,9 @@ public enum SessionVars implements ConfigKey { private final ConfVars key; private final String description; private final VariableMode mode; + + private Class valClass; + private Validator validator; public static enum VariableMode { DEFAULT, // Client can set or change variables of this mode.. @@ -157,6 +162,12 @@ public static enum VariableMode { this.description = description; this.mode = mode; } + + SessionVars(ConfVars key, String description, VariableMode mode, Class valueClass, Validator validator) { + this(key, description, mode); + this.valClass = valueClass; + this.validator = validator; + } public String keyname() { return name(); @@ -213,4 +224,14 @@ public static SessionVars get(String keyname) { public static String handleDeprecatedName(String keyname) { return SessionVars.exists(keyname) ? SessionVars.get(keyname).keyname() : keyname; } + + @Override + public Class valueClass() { + return valClass; + } + + @Override + public Validator validator() { + return validator; + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 181ef2e536..94f690784f 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -19,20 +19,28 @@ package org.apache.tajo.conf; import com.google.common.base.Preconditions; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.ConfigKey; +import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.NumberUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.datetime.DateTimeConstants; +import org.apache.tajo.validation.ConstraintViolationException; +import org.apache.tajo.validation.Validator; +import org.apache.tajo.validation.Validators; import java.io.IOException; import java.io.PrintStream; import java.net.InetSocketAddress; import java.util.Map; +import java.util.Properties; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -43,6 +51,9 @@ public class TajoConf extends Configuration { private static final ReentrantReadWriteLock confLock = new ReentrantReadWriteLock(); private static final Lock writeLock = confLock.writeLock(); private static final Lock readLock = confLock.readLock(); + private final AtomicBoolean isReloaded = new AtomicBoolean(true); + + private static final Map vars = TUtil.newHashMap(); static { Configuration.addDefaultResource("catalog-default.xml"); @@ -53,12 +64,14 @@ public class TajoConf extends Configuration { Configuration.addDefaultResource("tajo-site.xml"); confStaticInit(); + + for (ConfVars confVars: ConfVars.values()) { + vars.put(confVars.keyname(), confVars); + } } private static final String EMPTY_VALUE = ""; - private static final Map vars = TUtil.newHashMap(); - public TajoConf() { super(); } @@ -143,8 +156,8 @@ public static enum ConfVars implements ConfigKey { /////////////////////////////////////////////////////////////////////////////////////// // a username for a running Tajo cluster - ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/"), - USERNAME("tajo.username", "${user.name}"), + ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/", Validators.pathUrl()), + USERNAME("tajo.username", "${user.name}", Validators.shellVar()), // Configurable System Directories WAREHOUSE_DIR("tajo.warehouse.directory", EMPTY_VALUE), @@ -154,16 +167,19 @@ public static enum ConfVars implements ConfigKey { SYSTEM_CONF_REPLICA_COUNT("tajo.system-conf.replica-count", 20), // Tajo Master Service Addresses - TAJO_MASTER_UMBILICAL_RPC_ADDRESS("tajo.master.umbilical-rpc.address", "localhost:26001"), - TAJO_MASTER_CLIENT_RPC_ADDRESS("tajo.master.client-rpc.address", "localhost:26002"), - TAJO_MASTER_INFO_ADDRESS("tajo.master.info-http.address", "0.0.0.0:26080"), + TAJO_MASTER_UMBILICAL_RPC_ADDRESS("tajo.master.umbilical-rpc.address", "localhost:26001", + Validators.networkAddr()), + TAJO_MASTER_CLIENT_RPC_ADDRESS("tajo.master.client-rpc.address", "localhost:26002", + Validators.networkAddr()), + TAJO_MASTER_INFO_ADDRESS("tajo.master.info-http.address", "0.0.0.0:26080", Validators.networkAddr()), // Tajo Master HA Configurations - TAJO_MASTER_HA_ENABLE("tajo.master.ha.enable", false), + TAJO_MASTER_HA_ENABLE("tajo.master.ha.enable", false, Validators.bool()), TAJO_MASTER_HA_MONITOR_INTERVAL("tajo.master.ha.monitor.interval", 5 * 1000), // 5 sec // Resource tracker service - RESOURCE_TRACKER_RPC_ADDRESS("tajo.resource-tracker.rpc.address", "localhost:26003"), + RESOURCE_TRACKER_RPC_ADDRESS("tajo.resource-tracker.rpc.address", "localhost:26003", + Validators.networkAddr()), RESOURCE_TRACKER_HEARTBEAT_TIMEOUT("tajo.resource-tracker.heartbeat.timeout-secs", 120 * 1000), // seconds // QueryMaster resource @@ -171,25 +187,26 @@ public static enum ConfVars implements ConfigKey { TAJO_QUERYMASTER_MEMORY_MB("tajo.qm.resource.memory-mb", 512), // Tajo Worker Service Addresses - WORKER_INFO_ADDRESS("tajo.worker.info-http.address", "0.0.0.0:28080"), - WORKER_QM_INFO_ADDRESS("tajo.worker.qm-info-http.address", "0.0.0.0:28081"), - WORKER_PEER_RPC_ADDRESS("tajo.worker.peer-rpc.address", "0.0.0.0:28091"), - WORKER_CLIENT_RPC_ADDRESS("tajo.worker.client-rpc.address", "0.0.0.0:28092"), - WORKER_QM_RPC_ADDRESS("tajo.worker.qm-rpc.address", "0.0.0.0:28093"), + WORKER_INFO_ADDRESS("tajo.worker.info-http.address", "0.0.0.0:28080", Validators.networkAddr()), + WORKER_QM_INFO_ADDRESS("tajo.worker.qm-info-http.address", "0.0.0.0:28081", Validators.networkAddr()), + WORKER_PEER_RPC_ADDRESS("tajo.worker.peer-rpc.address", "0.0.0.0:28091", Validators.networkAddr()), + WORKER_CLIENT_RPC_ADDRESS("tajo.worker.client-rpc.address", "0.0.0.0:28092", Validators.networkAddr()), + WORKER_QM_RPC_ADDRESS("tajo.worker.qm-rpc.address", "0.0.0.0:28093", Validators.networkAddr()), // Tajo Worker Temporal Directories - WORKER_TEMPORAL_DIR("tajo.worker.tmpdir.locations", "/tmp/tajo-${user.name}/tmpdir"), - WORKER_TEMPORAL_DIR_CLEANUP("tajo.worker.tmpdir.cleanup-at-startup", false), + WORKER_TEMPORAL_DIR("tajo.worker.tmpdir.locations", "/tmp/tajo-${user.name}/tmpdir", + Validators.pathUrl()), + WORKER_TEMPORAL_DIR_CLEANUP("tajo.worker.tmpdir.cleanup-at-startup", false, Validators.bool()), // Tajo Worker Resources WORKER_RESOURCE_AVAILABLE_CPU_CORES("tajo.worker.resource.cpu-cores", 1), WORKER_RESOURCE_AVAILABLE_MEMORY_MB("tajo.worker.resource.memory-mb", 1024), WORKER_RESOURCE_AVAILABLE_DISKS("tajo.worker.resource.disks", 1.0f), WORKER_EXECUTION_MAX_SLOTS("tajo.worker.parallel-execution.max-num", 2), - WORKER_RESOURCE_DFS_DIR_AWARE("tajo.worker.resource.dfs-dir-aware", false), + WORKER_RESOURCE_DFS_DIR_AWARE("tajo.worker.resource.dfs-dir-aware", false, Validators.bool()), // Tajo Worker Dedicated Resources - WORKER_RESOURCE_DEDICATED("tajo.worker.resource.dedicated", false), + WORKER_RESOURCE_DEDICATED("tajo.worker.resource.dedicated", false, Validators.bool()), WORKER_RESOURCE_DEDICATED_MEMORY_RATIO("tajo.worker.resource.dedicated-memory-ratio", 0.8f), // Tajo Worker History @@ -201,7 +218,7 @@ public static enum ConfVars implements ConfigKey { RESOURCE_MANAGER_CLASS("tajo.resource.manager", "org.apache.tajo.master.rm.TajoWorkerResourceManager"), // Catalog - CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005"), + CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005", Validators.networkAddr()), // for Yarn Resource Manager ---------------------------------------------- @@ -217,7 +234,7 @@ public static enum ConfVars implements ConfigKey { // Shuffle Configuration -------------------------------------------------- PULLSERVER_PORT("tajo.pullserver.port", 0), - SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false), + SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false, Validators.bool()), SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"), SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM("tajo.shuffle.fetcher.parallel-execution.max-num", 2), SHUFFLE_FETCHER_CHUNK_MAX_SIZE("tajo.shuffle.fetcher.chunk.max-size", 8192), @@ -231,7 +248,7 @@ public static enum ConfVars implements ConfigKey { ROWFILE_SYNC_INTERVAL("rowfile.sync.interval", 100), MINIMUM_SPLIT_SIZE("tajo.min.split.size", (long) 1), // for RCFile - HIVEUSEEXPLICITRCFILEHEADER("tajo.exec.rcfile.use.explicit.header", true), + HIVEUSEEXPLICITRCFILEHEADER("tajo.exec.rcfile.use.explicit.header", true, Validators.bool()), // RPC -------------------------------------------------------------------- RPC_POOL_MAX_IDLE("tajo.rpc.pool.idle.max", 10), @@ -377,6 +394,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = false; this.type = VarType.STRING; } + + ConfVars(String varname, String defaultVal, Validator validator) { + this(varname, defaultVal); + this.validator = validator; + } ConfVars(String varname, int defaultIntVal) { this.varname = varname; @@ -388,6 +410,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = false; this.type = VarType.INT; } + + ConfVars(String varname, int defaultIntVal, Validator validator) { + this(varname, defaultIntVal); + this.validator = validator; + } ConfVars(String varname, long defaultLongVal) { this.varname = varname; @@ -399,6 +426,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = false; this.type = VarType.LONG; } + + ConfVars(String varname, long defaultLongVal, Validator validator) { + this(varname, defaultLongVal); + this.validator = validator; + } ConfVars(String varname, float defaultFloatVal) { this.varname = varname; @@ -410,6 +442,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = false; this.type = VarType.FLOAT; } + + ConfVars(String varname, float defaultFloatVal, Validator validator) { + this(varname, defaultFloatVal); + this.validator = validator; + } ConfVars(String varname, boolean defaultBoolVal) { this.varname = varname; @@ -421,6 +458,11 @@ public static enum ConfVars implements ConfigKey { this.defaultBoolVal = defaultBoolVal; this.type = VarType.BOOLEAN; } + + ConfVars(String varname, boolean defaultBoolVal, Validator validator) { + this(varname, defaultBoolVal); + this.validator = validator; + } enum VarType { STRING { void checkType(String value) throws Exception { } }, @@ -446,6 +488,16 @@ public String keyname() { public ConfigType type() { return ConfigType.SYSTEM; } + + @Override + public Class valueClass() { + return valClass; + } + + @Override + public Validator validator() { + return validator; + } } public static int getIntVar(Configuration conf, ConfVars var) { @@ -635,4 +687,59 @@ public static Path getSystemConfPath(TajoConf conf) { return new Path(systemConfPathStr); } } + + private void validateProperty(String name, String value) throws ConstraintViolationException { + ConfigKey configKey = null; + + configKey = TajoConf.getConfVars(name); + if (configKey == null) { + configKey = SessionVars.get(name); + } + + if (configKey != null && configKey.validator() != null && configKey.valueClass() != null) { + Object valueObj = value; + if (Number.class.isAssignableFrom(configKey.valueClass())) { + valueObj = NumberUtil.numberValue(configKey.valueClass(), value); + if (valueObj == null) { + return; + } + } + + configKey.validator().validate(valueObj, true); + } + } + + @Override + protected synchronized Properties getProps() { + Properties properties = super.getProps(); + + if (isReloaded.getAndSet(false)) { + + for (String key: properties.stringPropertyNames()) { + String value = properties.getProperty(key); + + validateProperty(key, value); + } + } + + return properties; + } + + @Override + public synchronized void reloadConfiguration() { + super.reloadConfiguration(); + isReloaded.set(true); + } + + @Override + public void set(String name, String value, String source) { + validateProperty(name, value); + super.set(name, value, source); + } + + @Override + public void set(String name, String value) { + set(name, value, null); + } + } diff --git a/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java b/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java index d52b80497c..375a2e45ac 100644 --- a/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/util/NumberUtil.java @@ -18,6 +18,9 @@ package org.apache.tajo.util; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + public class NumberUtil { public static final double[] powersOf10 = { /* Table giving binary powers of 10. Entry */ @@ -408,4 +411,22 @@ private static int parseIntInternal(byte[] bytes, int start, int length, int off } return result; } + + public static Number numberValue(Class numberClazz, String value) { + Number returnNumber = null; + + if (numberClazz == null && value == null) { + return returnNumber; + } + + if (Number.class.isAssignableFrom(numberClazz)) { + try { + Constructor constructor = numberClazz.getConstructor(String.class); + returnNumber = (Number) constructor.newInstance(value); + } catch (Exception ignored) { + } + } + + return returnNumber; + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java index 63c2352b09..300c5ea7bb 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/MaxValidator.java @@ -55,6 +55,10 @@ protected boolean validateInternal(T object) { Long objLong = Long.decode(object.toString()); Long maxLong = Long.decode(maxValue); result = objLong.compareTo(maxLong) <= 0; + } else if ((object instanceof Float) || (object instanceof Double)) { + Double objDouble = Double.valueOf(object.toString()); + Double maxDouble = Double.valueOf(maxValue); + result = objDouble.compareTo(maxDouble) <= 0; } else if (object instanceof BigInteger) { BigInteger objInteger = (BigInteger) object; BigInteger maxInteger = new BigInteger(maxValue); diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java index 0c3a99a2a2..2a29671891 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PatternValidator.java @@ -41,7 +41,12 @@ protected boolean validateInternal(T object) { if (object != null) { if (object instanceof CharSequence) { - result = pattern.matcher((CharSequence) object).find(); + String valueString = object.toString(); + if (valueString.isEmpty()) { + result = true; + } else { + result = pattern.matcher((CharSequence) object).find(); + } } } else { result = true; diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java index 26056ad9ce..4184c9757e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java @@ -50,4 +50,20 @@ public static Validator range(String minValue, String maxValue) { return new RangeValidator(minValue, maxValue); } + public static Validator pathUrl() { + return new PathValidator(); + } + + public static Validator shellVar() { + return new ShellVariableValidator(); + } + + public static Validator networkAddr() { + return new NetworkAddressValidator(); + } + + public static Validator bool() { + return new BooleanValidator(); + } + } diff --git a/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java b/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java index db5aae77c0..cdf4588861 100644 --- a/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java +++ b/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java @@ -1,26 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.conf; import static org.junit.Assert.*; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.apache.hadoop.conf.Configuration; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.validation.ConstraintViolationException; +import org.junit.Before; import org.junit.Test; public class TestValidatorsForTajoConf { + + @Before + public void setUp() throws Exception { + CopyOnWriteArrayList defaultResources = new CopyOnWriteArrayList( + new String[] {"catalog-default.xml", "catalog-site.xml", + "storage-default.xml", "storage-site.xml", + "tajo-default.xml", "tajo-site.xml"}); + Field defaultResourcesField = Configuration.class.getDeclaredField("defaultResources"); + defaultResourcesField.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(defaultResourcesField, defaultResourcesField.getModifiers() & ~Modifier.FINAL); + defaultResourcesField.set(null, defaultResources); + } @Test(expected=ConstraintViolationException.class) public void testPathValidatorsForTajoConf_setVar() { TajoConf conf = new TajoConf(); TajoConf.setVar(conf, ConfVars.ROOT_DIR, "Invalid path"); - fail(); + fail("ConstraintViolationException is expected but it is not occurred."); } @Test public void testValidPathForTajoConf() { TajoConf conf = new TajoConf(); - TajoConf.setVar(conf, ConfVars.ROOT_DIR, "file:///tmp/tajo-${user.name}/"); + try { + TajoConf.setVar(conf, ConfVars.ROOT_DIR, "file:///tmp/tajo-${user.name}/"); + } catch (Exception e) { + e.printStackTrace(); + fail("Unexpected ConstraintViolationException has been occurred."); + } + } + + @Test(expected=ConstraintViolationException.class) + public void testAddDefaultResourceValidation1() { + TajoConf.addDefaultResource("org/apache/tajo/conf/InvalidConf1.xml"); + TajoConf conf = new TajoConf(); + + conf.getVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS); + fail("Validation check has failed"); + } + + @Test(expected=ConstraintViolationException.class) + public void testAddDefaultResourceValidation2() { + TajoConf.addDefaultResource("org/apache/tajo/conf/InvalidConf2.xml"); + TajoConf conf = new TajoConf(); + + conf.getBoolVar(ConfVars.TAJO_MASTER_HA_ENABLE); + fail("Validation check has failed"); + } + + @Test(expected=ConstraintViolationException.class) + public void testSetConf1() { + TajoConf conf = new TajoConf(); + + conf.setVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, "9440"); + fail("Validation check has failed"); + } + + @Test(expected=ConstraintViolationException.class) + public void testSetConf2() { + TajoConf conf = new TajoConf(); + + conf.set(ConfVars.TAJO_MASTER_HA_ENABLE.keyname(), "yes"); + fail("Validation check has failed"); } } diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java index facb8c89c5..4d79b6bb39 100644 --- a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.tajo.validation; import static org.junit.Assert.*; @@ -266,7 +284,7 @@ public void testGroupValidator() { validators.add(new PatternValidator("^[a-zA-Z]+://")); validators.add(new LengthValidator(255)); assertThat(new GroupValidator(validators).validate(httpUrl).size(), is(0)); - assertThat(new GroupValidator(validators).validate("tajo").size(), is(2)); + assertThat(new GroupValidator(validators).validate("tajo").size(), is(1)); assertThat(new GroupValidator(validators).validate("tajo"), hasItem(hasAClass(equalTo(PatternValidator.class)))); @@ -298,5 +316,137 @@ public void testExceptionThrow() { fail(); } + + @Test + public void testPathValidator() { + String validUrl = "file:///tmp/tajo-$root/"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + validUrl = "file:///tmp/tajo-${user.name}/"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + validUrl = "file:///C:/Windows/System32"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + validUrl = "/C:/Windows/system32/driver"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + validUrl = "/tmp/tajo-root/"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + + String invalidUrl = "t!ef:///tmp/tajo-root"; + assertThat(new PathValidator().validateInternal(invalidUrl), is(false)); + assertThat(new PathValidator().validate(invalidUrl).size(), is(1)); + assertThat(new PathValidator().validate(invalidUrl), + hasItem(hasAClass(equalTo(PathValidator.class)))); + + invalidUrl = "This is not a valid url."; + assertThat(new PathValidator().validateInternal(invalidUrl), is(false)); + assertThat(new PathValidator().validate(invalidUrl).size(), is(1)); + assertThat(new PathValidator().validate(invalidUrl), + hasItem(hasAClass(equalTo(PathValidator.class)))); + } + + @Test + public void testShellVariableValidator() { + String validVariable = "${user.name}"; + assertThat(new ShellVariableValidator().validateInternal(validVariable), is(true)); + assertThat(new ShellVariableValidator().validate(validVariable).size(), is(0)); + + validVariable = "$SHELL"; + assertThat(new ShellVariableValidator().validateInternal(validVariable), is(true)); + assertThat(new ShellVariableValidator().validate(validVariable).size(), is(0)); + + validVariable = "STRING"; + assertThat(new ShellVariableValidator().validateInternal(validVariable), is(true)); + assertThat(new ShellVariableValidator().validate(validVariable).size(), is(0)); + + String invalidVariable = "Invalid Shell Variable Name"; + assertThat(new ShellVariableValidator().validateInternal(invalidVariable), is(false)); + assertThat(new ShellVariableValidator().validate(invalidVariable).size(), is(1)); + assertThat(new ShellVariableValidator().validate(invalidVariable), + hasItem(hasAClass(equalTo(ShellVariableValidator.class)))); + } + + @Test + public void testNetworkAddressValidator() { + String validNetworkAddress = "localhost:5000"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "192.168.0.1:5000"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "0.0.0.0:28094"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "tajo.apache.org"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "192.168.122.1"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + String invalidNetAddr = "5000"; + assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr), + hasItem(hasAClass(equalTo(NetworkAddressValidator.class)))); + + invalidNetAddr = "192.168."; + assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr), + hasItem(hasAClass(equalTo(NetworkAddressValidator.class)))); + + invalidNetAddr = "localhost:98765"; + assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); + assertThat(new NetworkAddressValidator().validate(invalidNetAddr), + hasItem(hasAClass(equalTo(NetworkAddressValidator.class)))); + } + + @Test + public void testBooleanValidator() { + String validBoolean = "true"; + assertThat(new BooleanValidator().validateInternal(validBoolean), is(true)); + assertThat(new BooleanValidator().validate(validBoolean).size(), is(0)); + + validBoolean = "false"; + assertThat(new BooleanValidator().validateInternal(validBoolean), is(true)); + assertThat(new BooleanValidator().validate(validBoolean).size(), is(0)); + + assertThat(new BooleanValidator().validateInternal(true), is(true)); + assertThat(new BooleanValidator().validate(true).size(), is(0)); + + assertThat(new BooleanValidator().validateInternal(false), is(true)); + assertThat(new BooleanValidator().validate(false).size(), is(0)); + + String invalidBoolean = "yes"; + assertThat(new BooleanValidator().validateInternal(invalidBoolean), is(false)); + assertThat(new BooleanValidator().validate(invalidBoolean).size(), is(1)); + assertThat(new BooleanValidator().validate(invalidBoolean), + hasItem(hasAClass(equalTo(BooleanValidator.class)))); + + invalidBoolean = "nope"; + assertThat(new BooleanValidator().validateInternal(invalidBoolean), is(false)); + assertThat(new BooleanValidator().validate(invalidBoolean).size(), is(1)); + assertThat(new BooleanValidator().validate(invalidBoolean), + hasItem(hasAClass(equalTo(BooleanValidator.class)))); + + invalidBoolean = "invalid"; + assertThat(new BooleanValidator().validateInternal(invalidBoolean), is(false)); + assertThat(new BooleanValidator().validate(invalidBoolean).size(), is(1)); + assertThat(new BooleanValidator().validate(invalidBoolean), + hasItem(hasAClass(equalTo(BooleanValidator.class)))); + } } diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java index 86fd355d72..d8f7f080b8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/query/QueryContext.java @@ -25,6 +25,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.master.session.Session; +import org.apache.tajo.validation.Validator; import org.apache.tajo.plan.logical.NodeType; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; @@ -56,6 +57,16 @@ public String keyname() { public ConfigType type() { return ConfigType.QUERY; } + + @Override + public Class valueClass() { + return null; + } + + @Override + public Validator validator() { + return null; + } } public QueryContext(TajoConf conf) { From 9d666ef75662c36e825b6bcdc1cd22e30d6c59af Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Mon, 20 Oct 2014 16:13:53 +0900 Subject: [PATCH 12/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- .../tajo/validation/BooleanValidator.java | 57 +++++++++++++++ .../validation/NetworkAddressValidator.java | 73 +++++++++++++++++++ .../apache/tajo/validation/PathValidator.java | 32 ++++++++ .../validation/ShellVariableValidator.java | 32 ++++++++ .../org/apache/tajo/conf/InvalidConf1.xml | 7 ++ .../org/apache/tajo/conf/InvalidConf2.xml | 7 ++ 6 files changed, 208 insertions(+) create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/BooleanValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/ShellVariableValidator.java create mode 100644 tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf1.xml create mode 100644 tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf2.xml diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/BooleanValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/BooleanValidator.java new file mode 100644 index 0000000000..b72a5fde03 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/BooleanValidator.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class BooleanValidator extends AbstractValidator { + + @Override + protected String getErrorMessage(T object) { + return object + " is not a valid boolean representation."; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof Boolean) { + result = true; + } else if (object instanceof CharSequence) { + String valueString = object.toString(); + if (Boolean.FALSE.toString().equalsIgnoreCase(valueString) || + Boolean.TRUE.toString().equalsIgnoreCase(valueString)) { + result = true; + } + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java new file mode 100644 index 0000000000..31565646fc --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; +import java.util.regex.Pattern; + +public class NetworkAddressValidator extends AbstractValidator { + + private final Pattern hostnamePattern; + private final Pattern portNumberPattern; + + public NetworkAddressValidator() { + hostnamePattern = Pattern.compile( + "^(?:[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}|[a-zA-Z][a-zA-Z0-9.-_]+)$"); + portNumberPattern = Pattern.compile("^[1-6]?[0-9]{1,4}$"); + } + + @Override + protected String getErrorMessage(T object) { + return object + " is not a valid network address representation."; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof CharSequence) { + String valueString = object.toString(); + if (valueString.isEmpty()) { + + } else { + int separatorIdx = valueString.indexOf(':'); + + if (separatorIdx > -1) { + result = (hostnamePattern.matcher(valueString.substring(0, separatorIdx)).find() & + portNumberPattern.matcher(valueString.substring(separatorIdx + 1)).find()); + } else { + result = hostnamePattern.matcher(valueString).find(); + } + } + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java new file mode 100644 index 0000000000..aad66a2fe6 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +public class PathValidator extends PatternValidator { + + public PathValidator() { + super("^(?:[a-zA-Z][a-zA-Z0-9+-.]+://[a-zA-Z-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$"); + } + + @Override + protected String getErrorMessage(T object) { + return object + " is not valid path."; + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ShellVariableValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/ShellVariableValidator.java new file mode 100644 index 0000000000..cfc0cc93ff --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ShellVariableValidator.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +public class ShellVariableValidator extends PatternValidator { + + public ShellVariableValidator() { + super("^[\\\\$]?[\\\\{]?[a-zA-Z0-9.]+[\\\\}]?$"); + } + + @Override + protected String getErrorMessage(T object) { + return object + " is not a valid shell variable"; + } + +} diff --git a/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf1.xml b/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf1.xml new file mode 100644 index 0000000000..c9a7c01e3d --- /dev/null +++ b/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf1.xml @@ -0,0 +1,7 @@ + + + + tajo.master.umbilical-rpc.address + localhost:98475 + + \ No newline at end of file diff --git a/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf2.xml b/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf2.xml new file mode 100644 index 0000000000..674a16887b --- /dev/null +++ b/tajo-common/src/test/resources/org/apache/tajo/conf/InvalidConf2.xml @@ -0,0 +1,7 @@ + + + + tajo.master.ha.enable + yes + + \ No newline at end of file From 1b5088bcf515f2e6706be1d1dbec515a9885b8e4 Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Mon, 20 Oct 2014 21:47:27 +0900 Subject: [PATCH 13/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- .../java/org/apache/tajo/SessionVars.java | 42 ++++++++------ .../java/org/apache/tajo/conf/TajoConf.java | 40 +++++++------ .../tajo/validation/ClassValidator.java | 57 +++++++++++++++++++ .../ConstraintViolationException.java | 7 ++- .../validation/NetworkAddressValidator.java | 42 ++++++++++++-- .../apache/tajo/validation/PathValidator.java | 2 +- .../apache/tajo/validation/Validators.java | 4 ++ .../tajo/validation/TestValidators.java | 42 ++++++++++++-- 8 files changed, 190 insertions(+), 46 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/ClassValidator.java diff --git a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java index fb6b75ec36..dbe949acc4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/SessionVars.java +++ b/tajo-common/src/main/java/org/apache/tajo/SessionVars.java @@ -23,6 +23,7 @@ import java.util.Map; import org.apache.tajo.validation.Validator; +import org.apache.tajo.validation.Validators; import static org.apache.tajo.SessionVars.VariableMode.*; import static org.apache.tajo.conf.TajoConf.ConfVars; @@ -42,8 +43,8 @@ public enum SessionVars implements ConfigKey { //------------------------------------------------------------------------------- // Server Side Only Variables //------------------------------------------------------------------------------- - SESSION_ID(ConfVars.$EMPTY, "session variable", SERVER_SIDE_VAR), - SESSION_LAST_ACCESS_TIME(ConfVars.$EMPTY, "last access time", SERVER_SIDE_VAR), + SESSION_ID(ConfVars.$EMPTY, "session variable", SERVER_SIDE_VAR, String.class, Validators.notNull()), + SESSION_LAST_ACCESS_TIME(ConfVars.$EMPTY, "last access time", SERVER_SIDE_VAR, Long.class, Validators.min("0")), USERNAME(ConfVars.USERNAME, "username", SERVER_SIDE_VAR), CLIENT_HOST(ConfVars.$EMPTY, "client hostname", SERVER_SIDE_VAR), @@ -55,7 +56,8 @@ public enum SessionVars implements ConfigKey { //------------------------------------------------------------------------------- // Client -------------------------------------------------------- - SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "session expiry time (secs)", DEFAULT), + SESSION_EXPIRY_TIME(ConfVars.$CLIENT_SESSION_EXPIRY_TIME, "session expiry time (secs)", DEFAULT, + Integer.class, Validators.min("0")), // Command line interface and its behavior -------------------------------- CLI_COLUMNS(ConfVars.$CLI_MAX_COLUMN, "Sets the width for the wrapped format", CLI_SIDE_VAR), @@ -89,29 +91,36 @@ public enum SessionVars implements ConfigKey { // for distributed query strategies BROADCAST_TABLE_SIZE_LIMIT(ConfVars.$DIST_QUERY_BROADCAST_JOIN_THRESHOLD, "limited size (bytes) of broadcast table", - DEFAULT), + DEFAULT, Long.class, Validators.min("0")), - JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size (mb) ", DEFAULT), + JOIN_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_JOIN_TASK_VOLUME, "join task input size (mb) ", DEFAULT, + Integer.class, Validators.min("1")), SORT_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_SORT_TASK_VOLUME, "sort task input size (mb)", DEFAULT), GROUPBY_TASK_INPUT_SIZE(ConfVars.$DIST_QUERY_GROUPBY_TASK_VOLUME, "group by task input size (mb)", DEFAULT), - JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join (mb)", DEFAULT), - GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort (mb)", DEFAULT), + JOIN_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_JOIN_PARTITION_VOLUME, "shuffle output size for join (mb)", DEFAULT, + Integer.class, Validators.min("1")), + GROUPBY_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_GROUPBY_PARTITION_VOLUME, "shuffle output size for sort (mb)", DEFAULT, + Integer.class, Validators.min("1")), TABLE_PARTITION_PER_SHUFFLE_SIZE(ConfVars.$DIST_QUERY_TABLE_PARTITION_VOLUME, - "shuffle output size for partition table write (mb)", DEFAULT), + "shuffle output size for partition table write (mb)", DEFAULT, Long.class, Validators.min("1")), - GROUPBY_MULTI_LEVEL_ENABLED(ConfVars.$GROUPBY_MULTI_LEVEL_ENABLED, "Multiple level groupby enabled", DEFAULT), + GROUPBY_MULTI_LEVEL_ENABLED(ConfVars.$GROUPBY_MULTI_LEVEL_ENABLED, "Multiple level groupby enabled", DEFAULT, + Boolean.class, Validators.bool()), // for physical Executors - EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort (mb)", DEFAULT), - HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash join (mb)", DEFAULT), + EXTSORT_BUFFER_SIZE(ConfVars.$EXECUTOR_EXTERNAL_SORT_BUFFER_SIZE, "sort buffer size for external sort (mb)", DEFAULT, + Long.class, Validators.min("0")), + HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash join (mb)", DEFAULT, + Long.class, Validators.min("0")), INNER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_INNER_HASH_JOIN_SIZE_THRESHOLD, - "limited size for hash inner join (mb)", DEFAULT), + "limited size for hash inner join (mb)", DEFAULT, Long.class, Validators.min("0")), OUTER_HASH_JOIN_SIZE_LIMIT(ConfVars.$EXECUTOR_OUTER_HASH_JOIN_SIZE_THRESHOLD, "limited size for hash outer join (mb)", - DEFAULT), + DEFAULT, Long.class, Validators.min("0")), HASH_GROUPBY_SIZE_LIMIT(ConfVars.$EXECUTOR_GROUPBY_INMEMORY_HASH_THRESHOLD, "limited size for hash groupby (mb)", - DEFAULT), - MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size (mb). 0 means infinite.", DEFAULT), + DEFAULT, Long.class, Validators.min("0")), + MAX_OUTPUT_FILE_SIZE(ConfVars.$MAX_OUTPUT_FILE_SIZE, "Maximum per-output file size (mb). 0 means infinite.", DEFAULT, + Long.class, Validators.min("0")), NULL_CHAR(ConfVars.$CSVFILE_NULL, "null char of text file output", DEFAULT), CODEGEN(ConfVars.$CODEGEN, "Runtime code generation enabled (experiment)", DEFAULT), @@ -120,7 +129,8 @@ public enum SessionVars implements ConfigKey { "If true, a running query will be terminated when an overflow or divide-by-zero occurs.", DEFAULT), // ResultSet ---------------------------------------------------------------- - FETCH_ROWNUM(ConfVars.$RESULT_SET_FETCH_ROWNUM, "Sets the number of rows at a time from Master", DEFAULT), + FETCH_ROWNUM(ConfVars.$RESULT_SET_FETCH_ROWNUM, "Sets the number of rows at a time from Master", DEFAULT, + Integer.class, Validators.min("0")), //------------------------------------------------------------------------------- // Only for Unit Testing diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 94f690784f..32fe7f8e6d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -156,15 +156,17 @@ public static enum ConfVars implements ConfigKey { /////////////////////////////////////////////////////////////////////////////////////// // a username for a running Tajo cluster - ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/", Validators.pathUrl()), - USERNAME("tajo.username", "${user.name}", Validators.shellVar()), + ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/", + Validators.groups(Validators.notNull(), Validators.pathUrl())), + USERNAME("tajo.username", "${user.name}", + Validators.groups(Validators.notNull(), Validators.shellVar())), // Configurable System Directories - WAREHOUSE_DIR("tajo.warehouse.directory", EMPTY_VALUE), - STAGING_ROOT_DIR("tajo.staging.directory", "/tmp/tajo-${user.name}/staging"), + WAREHOUSE_DIR("tajo.warehouse.directory", EMPTY_VALUE, Validators.pathUrl()), + STAGING_ROOT_DIR("tajo.staging.directory", "/tmp/tajo-${user.name}/staging", Validators.pathUrl()), - SYSTEM_CONF_PATH("tajo.system-conf.path", EMPTY_VALUE), - SYSTEM_CONF_REPLICA_COUNT("tajo.system-conf.replica-count", 20), + SYSTEM_CONF_PATH("tajo.system-conf.path", EMPTY_VALUE, Validators.pathUrl()), + SYSTEM_CONF_REPLICA_COUNT("tajo.system-conf.replica-count", 20, Validators.min("1")), // Tajo Master Service Addresses TAJO_MASTER_UMBILICAL_RPC_ADDRESS("tajo.master.umbilical-rpc.address", "localhost:26001", @@ -183,8 +185,8 @@ public static enum ConfVars implements ConfigKey { RESOURCE_TRACKER_HEARTBEAT_TIMEOUT("tajo.resource-tracker.heartbeat.timeout-secs", 120 * 1000), // seconds // QueryMaster resource - TAJO_QUERYMASTER_DISK_SLOT("tajo.qm.resource.disk.slots", 0.0f), - TAJO_QUERYMASTER_MEMORY_MB("tajo.qm.resource.memory-mb", 512), + TAJO_QUERYMASTER_DISK_SLOT("tajo.qm.resource.disk.slots", 0.0f, Validators.min("0.0f")), + TAJO_QUERYMASTER_MEMORY_MB("tajo.qm.resource.memory-mb", 512, Validators.min("64")), // Tajo Worker Service Addresses WORKER_INFO_ADDRESS("tajo.worker.info-http.address", "0.0.0.0:28080", Validators.networkAddr()), @@ -199,15 +201,16 @@ public static enum ConfVars implements ConfigKey { WORKER_TEMPORAL_DIR_CLEANUP("tajo.worker.tmpdir.cleanup-at-startup", false, Validators.bool()), // Tajo Worker Resources - WORKER_RESOURCE_AVAILABLE_CPU_CORES("tajo.worker.resource.cpu-cores", 1), - WORKER_RESOURCE_AVAILABLE_MEMORY_MB("tajo.worker.resource.memory-mb", 1024), + WORKER_RESOURCE_AVAILABLE_CPU_CORES("tajo.worker.resource.cpu-cores", 1, Validators.min("1")), + WORKER_RESOURCE_AVAILABLE_MEMORY_MB("tajo.worker.resource.memory-mb", 1024, Validators.min("64")), WORKER_RESOURCE_AVAILABLE_DISKS("tajo.worker.resource.disks", 1.0f), WORKER_EXECUTION_MAX_SLOTS("tajo.worker.parallel-execution.max-num", 2), WORKER_RESOURCE_DFS_DIR_AWARE("tajo.worker.resource.dfs-dir-aware", false, Validators.bool()), // Tajo Worker Dedicated Resources WORKER_RESOURCE_DEDICATED("tajo.worker.resource.dedicated", false, Validators.bool()), - WORKER_RESOURCE_DEDICATED_MEMORY_RATIO("tajo.worker.resource.dedicated-memory-ratio", 0.8f), + WORKER_RESOURCE_DEDICATED_MEMORY_RATIO("tajo.worker.resource.dedicated-memory-ratio", 0.8f, + Validators.range("0.0f", "1.0f")), // Tajo Worker History WORKER_HISTORY_EXPIRE_PERIOD("tajo.worker.history.expire-interval-minutes", 12 * 60), // 12 hours @@ -215,7 +218,8 @@ public static enum ConfVars implements ConfigKey { WORKER_HEARTBEAT_TIMEOUT("tajo.worker.heartbeat.timeout", 120 * 1000), // 120 sec // Resource Manager - RESOURCE_MANAGER_CLASS("tajo.resource.manager", "org.apache.tajo.master.rm.TajoWorkerResourceManager"), + RESOURCE_MANAGER_CLASS("tajo.resource.manager", "org.apache.tajo.master.rm.TajoWorkerResourceManager", + Validators.groups(Validators.notNull(), Validators.clazz())), // Catalog CATALOG_ADDRESS("tajo.catalog.client-rpc.address", "localhost:26005", Validators.networkAddr()), @@ -224,16 +228,16 @@ public static enum ConfVars implements ConfigKey { // for Yarn Resource Manager ---------------------------------------------- /** how many launching TaskRunners in parallel */ - YARN_RM_QUERY_MASTER_MEMORY_MB("tajo.querymaster.memory-mb", 512), + YARN_RM_QUERY_MASTER_MEMORY_MB("tajo.querymaster.memory-mb", 512, Validators.min("64")), YARN_RM_QUERY_MASTER_DISKS("tajo.yarn-rm.querymaster.disks", 1), YARN_RM_TASKRUNNER_LAUNCH_PARALLEL_NUM("tajo.yarn-rm.parallel-task-runner-launcher-num", 16), YARN_RM_WORKER_NUMBER_PER_NODE("tajo.yarn-rm.max-worker-num-per-node", 8), // Query Configuration - QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60), + QUERY_SESSION_TIMEOUT("tajo.query.session.timeout-sec", 60, Validators.min("0")), // Shuffle Configuration -------------------------------------------------- - PULLSERVER_PORT("tajo.pullserver.port", 0), + PULLSERVER_PORT("tajo.pullserver.port", 0, Validators.range("0", "65535")), SHUFFLE_SSL_ENABLED_KEY("tajo.pullserver.ssl.enabled", false, Validators.bool()), SHUFFLE_FILE_FORMAT("tajo.shuffle.file-format", "RAW"), SHUFFLE_FETCHER_PARALLEL_EXECUTION_MAX_NUM("tajo.shuffle.fetcher.parallel-execution.max-num", 2), @@ -322,9 +326,9 @@ public static enum ConfVars implements ConfigKey { $DIST_QUERY_SORT_TASK_VOLUME("tajo.dist-query.sort.task-volume-mb", 128), $DIST_QUERY_GROUPBY_TASK_VOLUME("tajo.dist-query.groupby.task-volume-mb", 128), - $DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128), - $DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256), - $DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256), + $DIST_QUERY_JOIN_PARTITION_VOLUME("tajo.dist-query.join.partition-volume-mb", 128, Validators.min("1")), + $DIST_QUERY_GROUPBY_PARTITION_VOLUME("tajo.dist-query.groupby.partition-volume-mb", 256, Validators.min("1")), + $DIST_QUERY_TABLE_PARTITION_VOLUME("tajo.dist-query.table-partition.task-volume-mb", 256, Validators.min("1")), $GROUPBY_MULTI_LEVEL_ENABLED("tajo.dist-query.groupby.multi-level-aggr", true), diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ClassValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/ClassValidator.java new file mode 100644 index 0000000000..1b35e6d195 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ClassValidator.java @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +import java.util.Collection; +import java.util.Collections; + +public class ClassValidator extends AbstractValidator { + + @Override + protected String getErrorMessage(T object) { + return "ClassLoader cannot find " + object + " class."; + } + + @Override + protected boolean validateInternal(T object) { + boolean result = false; + + if (object != null) { + if (object instanceof CharSequence) { + String valueString = object.toString(); + try { + Class.forName(valueString); + result = true; + } catch (ClassNotFoundException e) { + result = false; + } + } + } else { + result = true; + } + + return result; + } + + @Override + protected Collection getDependantValidators() { + return Collections.emptySet(); + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java index 664ec09771..cad6184567 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/ConstraintViolationException.java @@ -38,8 +38,13 @@ public Collection getViolations() { public String getMessage() { if (violations != null) { String errorMessage = "ConstraintViolationException ["; + int elemIdx = 1; + int elemCount = violations.size(); for (ConstraintViolation violation: violations) { - errorMessage += violation.getMessage() + ","; + errorMessage += violation.getMessage(); + if (elemIdx > elemCount) { + errorMessage += ","; + } } errorMessage += "]"; return errorMessage; diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java index 31565646fc..7477c4ca28 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/NetworkAddressValidator.java @@ -22,6 +22,8 @@ import java.util.Collections; import java.util.regex.Pattern; +import org.apache.http.conn.util.InetAddressUtils; + public class NetworkAddressValidator extends AbstractValidator { private final Pattern hostnamePattern; @@ -29,7 +31,7 @@ public class NetworkAddressValidator extends AbstractValidator { public NetworkAddressValidator() { hostnamePattern = Pattern.compile( - "^(?:[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}.[1-2]?[0-9]{1,2}|[a-zA-Z][a-zA-Z0-9.-_]+)$"); + "^[a-zA-Z][-a-zA-Z0-9\\\\._]+$"); portNumberPattern = Pattern.compile("^[1-6]?[0-9]{1,4}$"); } @@ -46,15 +48,43 @@ protected boolean validateInternal(T object) { if (object instanceof CharSequence) { String valueString = object.toString(); if (valueString.isEmpty()) { - + result = true; } else { int separatorIdx = valueString.indexOf(':'); - + String hostOrIpAddress = null; + if (separatorIdx > -1) { - result = (hostnamePattern.matcher(valueString.substring(0, separatorIdx)).find() & - portNumberPattern.matcher(valueString.substring(separatorIdx + 1)).find()); + if (valueString.indexOf(':', separatorIdx+1) > -1) { + // it is IPV6 representation. + int leftBracketsIdx = valueString.indexOf('['); + int rightBracketsIdx = valueString.indexOf(']'); + int periodIdx = valueString.indexOf('.'); + + if ((leftBracketsIdx > -1) && (rightBracketsIdx > -1) && + (valueString.length() > (rightBracketsIdx+1)) && + valueString.charAt(rightBracketsIdx+1) == ':') { + hostOrIpAddress = valueString.substring(leftBracketsIdx+1, rightBracketsIdx); + separatorIdx = rightBracketsIdx+1; + } else if ((periodIdx > -1)) { + hostOrIpAddress = valueString.substring(0, periodIdx); + separatorIdx = periodIdx; + } else { + separatorIdx = valueString.lastIndexOf(':'); + hostOrIpAddress = valueString.substring(0, separatorIdx); + } + } else { + hostOrIpAddress = valueString.substring(0, separatorIdx); + } } else { - result = hostnamePattern.matcher(valueString).find(); + hostOrIpAddress = valueString; + } + + result = ((hostnamePattern.matcher(hostOrIpAddress).find()) | + InetAddressUtils.isIPv4Address(hostOrIpAddress) | + InetAddressUtils.isIPv6Address(hostOrIpAddress)); + + if (separatorIdx > -1) { + result &= portNumberPattern.matcher(valueString.substring(separatorIdx + 1)).find(); } } } diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java index aad66a2fe6..a76587cd69 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/PathValidator.java @@ -21,7 +21,7 @@ public class PathValidator extends PatternValidator { public PathValidator() { - super("^(?:[a-zA-Z][a-zA-Z0-9+-.]+://[a-zA-Z-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$"); + super("^(?:[a-zA-Z][a-zA-Z0-9+-.]+:[/]{1,2}[a-zA-Z-.]*[:0-9]*)?(?:/?[a-zA-Z]:)?[/a-zA-Z0-9-_\\\\.\\\\$\\\\{\\\\}]*$"); } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java index 4184c9757e..cc0c82c0f2 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java @@ -66,4 +66,8 @@ public static Validator bool() { return new BooleanValidator(); } + public static Validator clazz() { + return new ClassValidator(); + } + } diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java index 4d79b6bb39..13370d6c02 100644 --- a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -31,13 +31,14 @@ import org.apache.tajo.util.TUtil; import org.hamcrest.Description; import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeDiagnosingMatcher; import org.junit.Test; public class TestValidators { - private class ValidatorClazzMatcher extends org.hamcrest.TypeSafeDiagnosingMatcher { + private class ValidatorClazzMatcher extends TypeSafeDiagnosingMatcher { - private final org.hamcrest.Matcher> matcher; + private final Matcher> matcher; public ValidatorClazzMatcher(Matcher> matcher) { this.matcher = matcher; @@ -327,6 +328,10 @@ public void testPathValidator() { assertThat(new PathValidator().validateInternal(validUrl), is(true)); assertThat(new PathValidator().validate(validUrl).size(), is(0)); + validUrl = "file:/home/tajo/test-data/TestExternalSortExec"; + assertThat(new PathValidator().validateInternal(validUrl), is(true)); + assertThat(new PathValidator().validate(validUrl).size(), is(0)); + validUrl = "file:///C:/Windows/System32"; assertThat(new PathValidator().validateInternal(validUrl), is(true)); assertThat(new PathValidator().validate(validUrl).size(), is(0)); @@ -387,7 +392,7 @@ public void testNetworkAddressValidator() { assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); - validNetworkAddress = "tajo.apache.org"; + validNetworkAddress = "Tajo-Test.apache.org"; assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); @@ -395,13 +400,25 @@ public void testNetworkAddressValidator() { assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + validNetworkAddress = "[2001:db8::ff00:42:8329]:20089"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "2001:db8::ff00:42:8330:20089"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + + validNetworkAddress = "2001:db8::ff00:42:8331.20089"; + assertThat(new NetworkAddressValidator().validateInternal(validNetworkAddress), is(true)); + assertThat(new NetworkAddressValidator().validate(validNetworkAddress).size(), is(0)); + String invalidNetAddr = "5000"; assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); assertThat(new NetworkAddressValidator().validate(invalidNetAddr), hasItem(hasAClass(equalTo(NetworkAddressValidator.class)))); - invalidNetAddr = "192.168."; + invalidNetAddr = "192.168.:"; assertThat(new NetworkAddressValidator().validateInternal(invalidNetAddr), is(false)); assertThat(new NetworkAddressValidator().validate(invalidNetAddr).size(), is(1)); assertThat(new NetworkAddressValidator().validate(invalidNetAddr), @@ -448,5 +465,22 @@ public void testBooleanValidator() { assertThat(new BooleanValidator().validate(invalidBoolean), hasItem(hasAClass(equalTo(BooleanValidator.class)))); } + + @Test + public void testClassValidator() { + String clazzName = "org.apache.tajo.validation.ClassValidator"; + assertThat(new ClassValidator().validateInternal(clazzName), is(true)); + assertThat(new ClassValidator().validate(clazzName).size(), is(0)); + + clazzName = "org.apache.tajo.ConfigKey"; + assertThat(new ClassValidator().validateInternal(clazzName), is(true)); + assertThat(new ClassValidator().validate(clazzName).size(), is(0)); + + String invalidClazzName = "invalid-.class.name"; + assertThat(new ClassValidator().validateInternal(invalidClazzName), is(false)); + assertThat(new ClassValidator().validate(invalidClazzName).size(), is(1)); + assertThat(new ClassValidator().validate(invalidClazzName), + hasItem(hasAClass(equalTo(ClassValidator.class)))); + } } From 7c92470a100732434e3aa453978c5b498eecca6e Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Mon, 20 Oct 2014 22:02:30 +0900 Subject: [PATCH 14/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- .../test/java/org/apache/tajo/validation/TestValidators.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java index 13370d6c02..a104874ee2 100644 --- a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -62,7 +62,7 @@ protected boolean matchesSafely(ConstraintViolation item, Description mismatchDe } - private org.hamcrest.Matcher + private Matcher hasAClass(Matcher> matcher) { return new ValidatorClazzMatcher(matcher); } @@ -74,7 +74,7 @@ public void testNotNullValidator() { assertThat(new NotNullValidator().validateInternal(testValue), is(false)); assertThat(new NotNullValidator().validate(testValue).size(), is(1)); assertThat(new NotNullValidator().validate(testValue), - hasItem(new ValidatorClazzMatcher(equalTo(NotNullValidator.class)))); + hasItem(hasAClass(equalTo(NotNullValidator.class)))); } @Test From b49df082a32d2bb5a987c7444b0b4b0e94fa8836 Mon Sep 17 00:00:00 2001 From: Jihun Kang Date: Mon, 27 Oct 2014 17:52:58 +0900 Subject: [PATCH 15/15] TAJO-1114: Improve ConfVars (SessionVar) to take a validator interface to check its input. --- .../java/org/apache/tajo/conf/TajoConf.java | 63 +---------- .../tajo/validation/JavaStringValidator.java | 32 ++++++ .../apache/tajo/validation/Validators.java | 4 + .../tajo/conf/TestValidatorsForTajoConf.java | 103 ------------------ .../tajo/validation/TestValidators.java | 17 +++ 5 files changed, 55 insertions(+), 164 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/validation/JavaStringValidator.java delete mode 100644 tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java diff --git a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java index 32fe7f8e6d..70977248c5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java +++ b/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java @@ -24,13 +24,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.ConfigKey; -import org.apache.tajo.SessionVars; import org.apache.tajo.TajoConstants; import org.apache.tajo.util.NetUtils; -import org.apache.tajo.util.NumberUtil; import org.apache.tajo.util.TUtil; import org.apache.tajo.util.datetime.DateTimeConstants; -import org.apache.tajo.validation.ConstraintViolationException; import org.apache.tajo.validation.Validator; import org.apache.tajo.validation.Validators; @@ -38,9 +35,7 @@ import java.io.PrintStream; import java.net.InetSocketAddress; import java.util.Map; -import java.util.Properties; import java.util.TimeZone; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -51,7 +46,6 @@ public class TajoConf extends Configuration { private static final ReentrantReadWriteLock confLock = new ReentrantReadWriteLock(); private static final Lock writeLock = confLock.writeLock(); private static final Lock readLock = confLock.readLock(); - private final AtomicBoolean isReloaded = new AtomicBoolean(true); private static final Map vars = TUtil.newHashMap(); @@ -159,7 +153,7 @@ public static enum ConfVars implements ConfigKey { ROOT_DIR("tajo.rootdir", "file:///tmp/tajo-${user.name}/", Validators.groups(Validators.notNull(), Validators.pathUrl())), USERNAME("tajo.username", "${user.name}", - Validators.groups(Validators.notNull(), Validators.shellVar())), + Validators.groups(Validators.notNull(), Validators.javaString())), // Configurable System Directories WAREHOUSE_DIR("tajo.warehouse.directory", EMPTY_VALUE, Validators.pathUrl()), @@ -387,6 +381,7 @@ public static enum ConfVars implements ConfigKey { public final boolean defaultBoolVal; private final VarType type; + private Validator validator; ConfVars(String varname, String defaultVal) { this.varname = varname; @@ -692,58 +687,4 @@ public static Path getSystemConfPath(TajoConf conf) { } } - private void validateProperty(String name, String value) throws ConstraintViolationException { - ConfigKey configKey = null; - - configKey = TajoConf.getConfVars(name); - if (configKey == null) { - configKey = SessionVars.get(name); - } - - if (configKey != null && configKey.validator() != null && configKey.valueClass() != null) { - Object valueObj = value; - if (Number.class.isAssignableFrom(configKey.valueClass())) { - valueObj = NumberUtil.numberValue(configKey.valueClass(), value); - if (valueObj == null) { - return; - } - } - - configKey.validator().validate(valueObj, true); - } - } - - @Override - protected synchronized Properties getProps() { - Properties properties = super.getProps(); - - if (isReloaded.getAndSet(false)) { - - for (String key: properties.stringPropertyNames()) { - String value = properties.getProperty(key); - - validateProperty(key, value); - } - } - - return properties; - } - - @Override - public synchronized void reloadConfiguration() { - super.reloadConfiguration(); - isReloaded.set(true); - } - - @Override - public void set(String name, String value, String source) { - validateProperty(name, value); - super.set(name, value, source); - } - - @Override - public void set(String name, String value) { - set(name, value, null); - } - } diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/JavaStringValidator.java b/tajo-common/src/main/java/org/apache/tajo/validation/JavaStringValidator.java new file mode 100644 index 0000000000..a8b1806549 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/validation/JavaStringValidator.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.validation; + +public class JavaStringValidator extends PatternValidator { + + public JavaStringValidator() { + super("^(?sU:\\p{Graph})+$"); + } + + @Override + protected String getErrorMessage(T object) { + return object + " is not a valid string"; + } + +} diff --git a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java index cc0c82c0f2..acd4876a7e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java +++ b/tajo-common/src/main/java/org/apache/tajo/validation/Validators.java @@ -70,4 +70,8 @@ public static Validator clazz() { return new ClassValidator(); } + public static Validator javaString() { + return new JavaStringValidator(); + } + } diff --git a/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java b/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java deleted file mode 100644 index cdf4588861..0000000000 --- a/tajo-common/src/test/java/org/apache/tajo/conf/TestValidatorsForTajoConf.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.conf; - -import static org.junit.Assert.*; - -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.util.concurrent.CopyOnWriteArrayList; - -import org.apache.hadoop.conf.Configuration; -import org.apache.tajo.conf.TajoConf.ConfVars; -import org.apache.tajo.validation.ConstraintViolationException; -import org.junit.Before; -import org.junit.Test; - -public class TestValidatorsForTajoConf { - - @Before - public void setUp() throws Exception { - CopyOnWriteArrayList defaultResources = new CopyOnWriteArrayList( - new String[] {"catalog-default.xml", "catalog-site.xml", - "storage-default.xml", "storage-site.xml", - "tajo-default.xml", "tajo-site.xml"}); - Field defaultResourcesField = Configuration.class.getDeclaredField("defaultResources"); - defaultResourcesField.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); - modifiersField.setAccessible(true); - modifiersField.setInt(defaultResourcesField, defaultResourcesField.getModifiers() & ~Modifier.FINAL); - defaultResourcesField.set(null, defaultResources); - } - - @Test(expected=ConstraintViolationException.class) - public void testPathValidatorsForTajoConf_setVar() { - TajoConf conf = new TajoConf(); - - TajoConf.setVar(conf, ConfVars.ROOT_DIR, "Invalid path"); - fail("ConstraintViolationException is expected but it is not occurred."); - } - - @Test - public void testValidPathForTajoConf() { - TajoConf conf = new TajoConf(); - - try { - TajoConf.setVar(conf, ConfVars.ROOT_DIR, "file:///tmp/tajo-${user.name}/"); - } catch (Exception e) { - e.printStackTrace(); - fail("Unexpected ConstraintViolationException has been occurred."); - } - } - - @Test(expected=ConstraintViolationException.class) - public void testAddDefaultResourceValidation1() { - TajoConf.addDefaultResource("org/apache/tajo/conf/InvalidConf1.xml"); - TajoConf conf = new TajoConf(); - - conf.getVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS); - fail("Validation check has failed"); - } - - @Test(expected=ConstraintViolationException.class) - public void testAddDefaultResourceValidation2() { - TajoConf.addDefaultResource("org/apache/tajo/conf/InvalidConf2.xml"); - TajoConf conf = new TajoConf(); - - conf.getBoolVar(ConfVars.TAJO_MASTER_HA_ENABLE); - fail("Validation check has failed"); - } - - @Test(expected=ConstraintViolationException.class) - public void testSetConf1() { - TajoConf conf = new TajoConf(); - - conf.setVar(ConfVars.TAJO_MASTER_UMBILICAL_RPC_ADDRESS, "9440"); - fail("Validation check has failed"); - } - - @Test(expected=ConstraintViolationException.class) - public void testSetConf2() { - TajoConf conf = new TajoConf(); - - conf.set(ConfVars.TAJO_MASTER_HA_ENABLE.keyname(), "yes"); - fail("Validation check has failed"); - } - -} diff --git a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java index a104874ee2..dc4850cc39 100644 --- a/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java +++ b/tajo-common/src/test/java/org/apache/tajo/validation/TestValidators.java @@ -482,5 +482,22 @@ public void testClassValidator() { assertThat(new ClassValidator().validate(invalidClazzName), hasItem(hasAClass(equalTo(ClassValidator.class)))); } + + @Test + public void testStringValidator() { + String validAsciiString = "tajo-root900"; + assertThat(new JavaStringValidator().validateInternal(validAsciiString), is(true)); + assertThat(new JavaStringValidator().validate(validAsciiString).size(), is(0)); + + String validMultibyteString = "타조009"; + assertThat(new JavaStringValidator().validateInternal(validMultibyteString), is(true)); + assertThat(new JavaStringValidator().validate(validMultibyteString).size(), is(0)); + + String invalidAsciiString = " inva - "; + assertThat(new JavaStringValidator().validateInternal(invalidAsciiString), is(false)); + assertThat(new JavaStringValidator().validate(invalidAsciiString).size(), is(1)); + assertThat(new JavaStringValidator().validate(invalidAsciiString), + hasItem(hasAClass(equalTo(JavaStringValidator.class)))); + } }