Skip to content

Commit

Permalink
[CALCITE-3329] Implement osquery for OS adapter
Browse files Browse the repository at this point in the history
  • Loading branch information
XuQianJin-Stars committed Dec 17, 2023
1 parent 8c640da commit 94110de
Show file tree
Hide file tree
Showing 26 changed files with 1,198 additions and 230 deletions.
13 changes: 8 additions & 5 deletions bom/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,13 @@ dependencies {
// runtime means "the dependency is only for runtime, not for compilation"
// In other words, marking dependency as "runtime" would avoid accidental
// dependency on it during compilation
apiv("cn.hutool:hutool-all")
apiv("com.alibaba.database:innodb-java-reader")
apiv("com.beust:jcommander")
apiv("org.checkerframework:checker-qual", "checkerframework")
apiv("com.datastax.oss:java-driver-core", "cassandra-java-driver-core")
apiv("org.locationtech.jts:jts-core")
apiv("org.locationtech.jts.io:jts-io-common")
apiv("org.locationtech.proj4j:proj4j")
apiv("org.locationtech.proj4j:proj4j-epsg", "proj4j")
apiv("com.fasterxml.jackson.core:jackson-databind")
apiv("com.github.kstyrc:embedded-redis")
apiv("com.github.oshi:oshi-core")
apiv("com.github.stephenc.jcip:jcip-annotations")
apiv("com.google.errorprone:error_prone_annotations", "errorprone")
apiv("com.google.errorprone:error_prone_type_annotations", "errorprone")
Expand All @@ -75,6 +72,11 @@ dependencies {
apiv("com.yahoo.datasketches:sketches-core")
apiv("commons-codec:commons-codec")
apiv("commons-io:commons-io")
apiv("org.checkerframework:checker-qual", "checkerframework")
apiv("org.locationtech.jts:jts-core")
apiv("org.locationtech.jts.io:jts-io-common")
apiv("org.locationtech.proj4j:proj4j")
apiv("org.locationtech.proj4j:proj4j-epsg", "proj4j")
apiv("de.bwaldvogel:mongo-java-server", "mongo-java-server")
apiv("de.bwaldvogel:mongo-java-server-core", "mongo-java-server")
apiv("de.bwaldvogel:mongo-java-server-memory-backend", "mongo-java-server")
Expand All @@ -93,6 +95,7 @@ dependencies {
apiv("net.hydromatic:sql-logic-test")
apiv("net.hydromatic:tpcds", "hydromatic.tpcds")
apiv("net.java.dev.jna:jna")
apiv("net.java.dev.jna:jna-platform")
apiv("net.sf.opencsv:opencsv")
apiv("org.apache.calcite.avatica:avatica-core", "calcite.avatica")
apiv("org.apache.calcite.avatica:avatica-server", "calcite.avatica")
Expand Down
5 changes: 4 additions & 1 deletion gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ hamcrest.version=2.1
hsqldb.version=2.7.2
httpclient.version=4.5.9
httpcore.version=4.4.11
hutool-all.version=5.8.23
hydromatic.tpcds.version=0.4
immutables.version=2.8.8
innodb-java-reader.version=1.0.10
Expand All @@ -129,7 +130,8 @@ jcommander.version=1.72
jedis.version=3.3.0
jetty.version=9.4.46.v20220331
jmh.version=1.12
jna.version=5.7.0
jna.version=5.14.0
jna-platform.version=5.14.0
joda-time.version=2.8.1
json-path.version=2.8.0
jsr305.version=3.0.2
Expand All @@ -147,6 +149,7 @@ mysql-connector-java.version=5.1.20
natty.version=0.13
ojdbc8.version=19.3.0.0
opencsv.version=2.3
oshi-core.version=6.4.9
pig.version=0.16.0
pigunit.version=0.16.0
postgresql.version=9.3-1102-jdbc41
Expand Down
11 changes: 11 additions & 0 deletions plus/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,20 @@ dependencies {

implementation("com.google.guava:guava")
implementation("com.teradata.tpcds:tpcds")
implementation("cn.hutool:hutool-all")
implementation("io.prestosql.tpch:tpch")
implementation("net.hydromatic:chinook-data-hsqldb")
implementation("net.hydromatic:tpcds")
implementation("net.java.dev.jna:jna")
implementation("net.java.dev.jna:jna-platform")

implementation("com.github.oshi:oshi-core") {
exclude("log4j", "log4j")
.because("log4j is already present in the classpath")
exclude("org.slf4j", "slf4j-api")
.because("slf4j is already present in the classpath")
}

implementation("org.apache.calcite.avatica:avatica-server")
implementation("org.hsqldb:hsqldb::jdk8")

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.calcite.adapter.os;

import org.apache.calcite.config.CalciteConnectionConfig;
import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.Statistic;
import org.apache.calcite.schema.Statistics;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.util.ImmutableBitSet;

import com.google.common.collect.ImmutableList;

import org.checkerframework.checker.nullness.qual.Nullable;

/**
* Abstract base class for implementations of OS table functions.
*/
abstract class AbstractBaseScannableTable implements ScannableTable {
protected AbstractBaseScannableTable() {
}

@Override public Statistic getStatistic() {
return Statistics.of(1000d, ImmutableList.of(ImmutableBitSet.of(1)));
}

@Override public Schema.TableType getJdbcTableType() {
return Schema.TableType.TABLE;
}

@Override public boolean isRolledUp(String column) {
return false;
}

@Override public boolean rolledUpColumnValidInsideAgg(String column, SqlCall call,
@Nullable SqlNode parent, @Nullable CalciteConnectionConfig config) {
return true;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.calcite.adapter.os;

import org.apache.calcite.DataContext;
import org.apache.calcite.linq4j.AbstractEnumerable;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.sql.type.SqlTypeName;

import org.checkerframework.checker.nullness.qual.Nullable;

/**
* Table function that executes the OS "cpu_info".
*/
public class CpuInfoTableFunction {
private CpuInfoTableFunction() {
}

public static ScannableTable eval(boolean b) {
return new AbstractBaseScannableTable() {
@Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
return new AbstractEnumerable<Object[]>() {
@Override public Enumerator<Object[]> enumerator() {
return new OsQueryEnumerator("cpu_info");
}
};
}

@Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
return typeFactory.builder()
.add("device_id", SqlTypeName.VARCHAR)
.add("model", SqlTypeName.VARCHAR)
.add("manufacturer", SqlTypeName.VARCHAR)
.add("number_of_cores", SqlTypeName.INTEGER)
.add("logical_processors", SqlTypeName.INTEGER)
.add("address_width", SqlTypeName.INTEGER)
.add("max_clock_speed", SqlTypeName.BIGINT)
.add("socket_designation", SqlTypeName.INTEGER)
.add("cpu_load", SqlTypeName.DOUBLE)
.build();
}
};
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to you under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.calcite.adapter.os;

import org.apache.calcite.DataContext;
import org.apache.calcite.linq4j.AbstractEnumerable;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.sql.type.SqlTypeName;

import org.checkerframework.checker.nullness.qual.Nullable;

/**
* Table function that executes the OS "cpu_info".
*/
public class CpuTimeTableFunction {
private CpuTimeTableFunction() {
}

public static ScannableTable eval(boolean b) {
return new AbstractBaseScannableTable() {
@Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
return new AbstractEnumerable<Object[]>() {
@Override public Enumerator<Object[]> enumerator() {
return new OsQueryEnumerator("cpu_time");
}
};
}

@Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
return typeFactory.builder()
.add("idle", SqlTypeName.BIGINT)
.add("nice", SqlTypeName.BIGINT)
.add("irq", SqlTypeName.BIGINT)
.add("soft_irq", SqlTypeName.BIGINT)
.add("steal", SqlTypeName.BIGINT)
.add("system", SqlTypeName.BIGINT)
.add("user", SqlTypeName.BIGINT)
.add("io_wait", SqlTypeName.BIGINT)
.build();
}
};
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,11 @@
package org.apache.calcite.adapter.os;

import org.apache.calcite.DataContext;
import org.apache.calcite.config.CalciteConnectionConfig;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.Statistic;
import org.apache.calcite.schema.Statistics;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.ImmutableBitSet;

import com.google.common.collect.ImmutableList;

import org.checkerframework.checker.nullness.qual.Nullable;

Expand All @@ -39,10 +30,11 @@
* to compute file sizes.
*/
public class DuTableFunction {
private DuTableFunction() {}
private DuTableFunction() {
}

public static ScannableTable eval(boolean b) {
return new ScannableTable() {
return new AbstractBaseScannableTable() {
@Override public Enumerable<@Nullable Object[]> scan(DataContext root) {
return Processes.processLines("du", "-ak")
.select(a0 -> {
Expand All @@ -57,24 +49,6 @@ public static ScannableTable eval(boolean b) {
.add("path", SqlTypeName.VARCHAR)
.build();
}

@Override public Statistic getStatistic() {
return Statistics.of(1000d, ImmutableList.of(ImmutableBitSet.of(1)));
}

@Override public Schema.TableType getJdbcTableType() {
return Schema.TableType.TABLE;
}

@Override public boolean isRolledUp(String column) {
return false;
}

@Override public boolean rolledUpColumnValidInsideAgg(String column, SqlCall call,
@Nullable SqlNode parent, @Nullable CalciteConnectionConfig config) {
return true;
}
};
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,13 @@

import org.apache.calcite.DataContext;
import org.apache.calcite.adapter.java.JavaTypeFactory;
import org.apache.calcite.config.CalciteConnectionConfig;
import org.apache.calcite.linq4j.AbstractEnumerable;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.Enumerator;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.schema.ScannableTable;
import org.apache.calcite.schema.Schema;
import org.apache.calcite.schema.Statistic;
import org.apache.calcite.schema.Statistics;
import org.apache.calcite.sql.SqlCall;
import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Util;

import com.google.common.collect.ImmutableList;
Expand All @@ -52,15 +45,17 @@ public class FilesTableFunction {

private static final BigDecimal THOUSAND = BigDecimal.valueOf(1000L);

private FilesTableFunction() {}
private FilesTableFunction() {
}

/** Evaluates the function.
/**
* Evaluates the function.
*
* @param path Directory in which to start the search. Typically '.'
* @return Table that can be inspected, planned, and evaluated
*/
public static ScannableTable eval(final String path) {
return new ScannableTable() {
return new AbstractBaseScannableTable() {
@Override public RelDataType getRowType(RelDataTypeFactory typeFactory) {
return typeFactory.builder()
.add("access_time", SqlTypeName.TIMESTAMP) // %A@ sec since epoch
Expand Down Expand Up @@ -267,24 +262,6 @@ private Object field(String field, String value) {
}
};
}

@Override public Statistic getStatistic() {
return Statistics.of(1000d, ImmutableList.of(ImmutableBitSet.of(1)));
}

@Override public Schema.TableType getJdbcTableType() {
return Schema.TableType.TABLE;
}

@Override public boolean isRolledUp(String column) {
return false;
}

@Override public boolean rolledUpColumnValidInsideAgg(String column, SqlCall call,
@Nullable SqlNode parent, @Nullable CalciteConnectionConfig config) {
return true;
}
};
}

}

0 comments on commit 94110de

Please sign in to comment.