[Issue #170]: implemented scan operator.

pixels-lambda/1-update-code.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# update code
+#aws lambda update-function-code --function-name io.pixelsdb.pixels.lambda.Worker --architectures arm64\
+#  --s3-bucket "tiannan-test"  --s3-key "pixels-lambda-0.1.0-SNAPSHOT.jar"\
+#  --zip-file fileb://target/pixels-lambda-0.1.0-SNAPSHOT.jar
+
+# change configuration
+#aws lambda update-function-configuration --function-name io.pixelsdb.pixels.lambda.Worker  --handler io.pixelsdb.pixels.lambda.Worker
+
+# upload jar to s3
+aws s3 cp ./target/pixels-lambda-0.1.0-SNAPSHOT-full.jar s3://tiannan-test/pixels-lambda-0.1.0-SNAPSHOT-full.jar
+# update the jar file directly from s3
+aws lambda update-function-code --function-name Worker --architectures arm64\
+  --s3-bucket "tiannan-test"  --s3-key "pixels-lambda-0.1.0-SNAPSHOT-full.jar"

pixels-lambda/2-invoke.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+aws lambda invoke --function-name Worker \
+  --payload '{ "fileNames":["pixels-tpch-orders-v-0-order/20220312072707_0.pxl", "pixels-tpch-orders-v-0-order/20220312072714_1.pxl"],
+   "cols":["o_orderkey", "o_custkey", "o_orderstatus", "o_orderdate"]}' \
+  --cli-binary-format raw-in-base64-out response.json #output file

pixels-lambda/pom.xml

@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>pixels</artifactId>
+        <groupId>io.pixelsdb</groupId>
+        <version>0.1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>pixels-lambda</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+        <mainClass>io.pixelsdb.pixels.lambda.QueryExecutor</mainClass>
+    </properties>
+
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>bom</artifactId>
+                <version>2.17.61</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+    <dependencies>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>lambda</artifactId>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/software.amazon.awssdk/s3 -->
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3</artifactId>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-lambda -->
+<!--        <dependency>-->
+<!--            <groupId>com.amazonaws</groupId>-->
+<!--            <artifactId>aws-java-sdk-lambda</artifactId>-->
+<!--            <version>1.12.159</version>-->
+<!--        </dependency>-->
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-lambda-java-core</artifactId>
+            <version>1.2.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.2.4</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>io.pixelsdb</groupId>
+            <artifactId>pixels-core</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.pixelsdb</groupId>
+            <artifactId>pixels-common</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>io.pixelsdb</groupId>
+            <artifactId>pixels-cache</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.etcd</groupId>
+            <artifactId>jetcd-core</artifactId>
+            <optional>true</optional>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-nop -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-nop</artifactId>
+            <version>1.7.36</version>
+            <scope>test</scope>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.curator/apache-curator -->
+<!--        <dependency>-->
+<!--            <groupId>org.apache.curator</groupId>-->
+<!--            <artifactId>apache-curator</artifactId>-->
+<!--            <version>5.2.0</version>-->
+<!--            <type>pom</type>-->
+<!--        </dependency>-->
+        <!-- https://mvnrepository.com/artifact/org.apache.curator/curator-test -->
+<!--        <dependency>-->
+<!--            <groupId>org.apache.curator</groupId>-->
+<!--            <artifactId>curator-test</artifactId>-->
+<!--            <version>5.2.0</version>-->
+<!--            <scope>test</scope>-->
+<!--        </dependency>-->
+<!--        <dependency>-->
+<!--            <groupId>software.amazon.awssdk</groupId>-->
+<!--            <artifactId>apache-client</artifactId>-->
+<!--            <optional>true</optional>-->
+<!--        </dependency>-->
+        <!-- https://mvnrepository.com/artifact/org.apache.curator/curator-client -->
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-client</artifactId>
+            <version>5.2.0</version>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>aws-crt-client</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-common</artifactId>
+            <optional>true</optional>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.2</version>
+                <configuration>
+                    <createDependencyReducedPom>false</createDependencyReducedPom>
+                </configuration>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+<!--                            <finalName>pixels-daemon</finalName>-->
+                            <outputDirectory>${project.parent.basedir}/pixels-lambda/target</outputDirectory>
+                            <shadedArtifactAttached>true</shadedArtifactAttached>
+                            <shadedClassifierName>full</shadedClassifierName>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>${mainClass}</mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

pixels-lambda/response.json

@@ -0,0 +1 @@
+"bc8e8389-82eb-4674-8f26-9597389bf82dfile0,bc8e8389-82eb-4674-8f26-9597389bf82dfile1"

pixels-lambda/src/main/java/io/pixelsdb/pixels/lambda/LambdaEvent.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2022 PixelsDB.
+ *
+ * This file is part of Pixels.
+ *
+ * Pixels is free software: you can redistribute it and/or modify
+ * it under the terms of the Affero GNU General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Pixels is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * Affero GNU General Public License for more details.
+ *
+ * You should have received a copy of the Affero GNU General Public
+ * License along with Pixels.  If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+
+package io.pixelsdb.pixels.lambda;
+
+import java.util.ArrayList;
+
+public class LambdaEvent {
+    ArrayList<String> fileNames;
+    ArrayList<String> cols;
+
+    public LambdaEvent(ArrayList<String> fileNames, ArrayList<String> cols) {
+        this.fileNames = fileNames;
+        this.cols = cols;
+    }
+}

pixels-lambda/src/main/java/io/pixelsdb/pixels/lambda/QueryExecutor.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright 2022 PixelsDB.
+ *
+ * This file is part of Pixels.
+ *
+ * Pixels is free software: you can redistribute it and/or modify
+ * it under the terms of the Affero GNU General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Pixels is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * Affero GNU General Public License for more details.
+ *
+ * You should have received a copy of the Affero GNU General Public
+ * License along with Pixels.  If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+
+package io.pixelsdb.pixels.lambda;
+
+import java.util.ArrayList;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
+import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.ListIterator;
+
+public class QueryExecutor {
+    public static void main(String[] args) {
+        /* use lambda function to do the scan */
+        ArrayList<String> filesToScan = new ArrayList<String>();
+
+
+        ArrayList<String> cols = new ArrayList<String>() {
+            {
+                add("o_orderkey");
+                add("o_custkey");
+                add("o_orderstatus");
+                add("o_orderdate");
+            }
+        };
+        String[] colsArr = cols.toArray(new String[0]);
+
+        //for testing on local
+//        filesToScan.add("pixels-tpch-orders-v-0-order/20220312072707_0.pxl");
+//        filesToScan.add("pixels-tpch-orders-v-0-order/20220312072714_1.pxl");
+//        filesToScan.add("pixels-tpch-orders-v-0-order/20220312072720_2.pxl");
+//        filesToScan.add("pixels-tpch-orders-v-0-order/20220312072727_3.pxl");
+
+        // get the list of files in bucket to scan
+        String bucketName = "pixels-tpch-orders-v-0-order";
+        Region region = Region.US_EAST_2;
+        S3Client s3 = S3Client.builder()
+                .region(region)
+                .build();
+        filesToScan = listBucketObjects(s3, bucketName);
+//        for (String obj:listBucketObjects(s3, bucketName).subList(0,128)) {
+//            filesToScan.add(obj);
+//        }
+        String[] filesToScanArr = filesToScan.toArray(new String[0]);
+        s3.close();
+        System.out.println(Arrays.toString(filesToScanArr));
+
+
+        Scan scanLambda = new Scan();
+        Scan scanNoLambda = new Scan();
+        int numFilePerWorker=1;
+        if (args.length==1) numFilePerWorker = Integer.parseInt(args[0]);
+
+        /* use Pixels reader to scan tables directly */
+//        System.out.println("begin scan tables directly without using lambda");
+//        long startTimeNoLambda = System.nanoTime();
+//
+//        scanNoLambda.readersReadFiles(filesToScanArr, colsArr);
+//
+//        long endTimeNoLambda = System.nanoTime();
+//        double scanTimeNoLambda = 1.0*(endTimeNoLambda - startTimeNoLambda)/Math.pow(10, 9);
+
+        /* use lambda function to do the scan */
+        System.out.println("begin using lambda to scan files");
+        long startTimeLambda = System.nanoTime();
+        scanLambda.scan(numFilePerWorker, filesToScan, cols);
+        long endTimeLambda = System.nanoTime();
+        double scanTimeLambda = 1.0*(endTimeLambda - startTimeLambda)/Math.pow(10, 9);
+
+        System.out.println("scanTimeLambda:" + scanTimeLambda);
+        //System.out.println("scanTimeNoLambda:" + scanTimeNoLambda);
+    }
+
+    public static ArrayList<String> listBucketObjects(S3Client s3, String bucketName) {
+
+        ArrayList<String> filesToScan = new ArrayList<String>();
+        try {
+            ListObjectsRequest listObjects = ListObjectsRequest
+                    .builder()
+                    .bucket(bucketName)
+                    .build();
+
+            ListObjectsResponse res = s3.listObjects(listObjects);
+            List<S3Object> objects = res.contents();
+
+            for (ListIterator iterVals = objects.listIterator(); iterVals.hasNext(); ) {
+                S3Object myValue = (S3Object) iterVals.next();
+                filesToScan.add(bucketName + "/" + myValue.key());
+            }
+
+        } catch (S3Exception e) {
+            System.err.println(e.awsErrorDetails().errorMessage());
+            System.exit(1);
+        }
+        return filesToScan;
+    }
+}