[Metadata] refractor metadata entity with protobuf code-gen

.github/workflows/flink-cdc-test.yml

@@ -40,6 +40,10 @@ jobs:
         run: |
           python -m pip install --upgrade pip setuptools wheel
           pip install pymysql cryptography jproperties --no-cache-dir
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal

.github/workflows/maven-test.yml

@@ -52,7 +52,7 @@ jobs:
 
   spark-test-1:
     runs-on: ubuntu-latest
-    needs: [build-linux-x86_64]
+    needs: [ build-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -76,42 +76,46 @@ jobs:
           - 5432:5432
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up JDK 8
-      uses: actions/setup-java@v3
-      with:
-        java-version: '8'
-        distribution: 'temurin'
-        cache: maven
-    - name: Install psql
-      run: sudo apt-get install -y postgresql-client-14
-    - name: Init PG
-      run: |
-        ./script/meta_init_for_local_test.sh -j 2
-    - uses: actions/download-artifact@v3
-      with:
-        name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
-        path: ./native-io/target/release/
-    - name: Build with Maven
-      run: |
-        mvn -B test -pl lakesoul-spark -am -Pcross-build -Pparallel-test --file pom.xml -Dtest='UpdateScalaSuite,AlterTableByNameSuite,ReadSuite,UpdateSQLSuite,ParquetNativeFilterSuite,DeleteScalaSuite,DeleteSQLSuite,ParquetV2FilterSuite,ParquetScanSuite,UpsertSuiteBase' -Dsurefire.failIfNoSpecifiedTests=false
-    - name: Generate Report Site
-      if: always()
-      run: |
-        mvn surefire-report:report-only -pl lakesoul-spark -am
-    - name: Upload Test Report
-      if: always()
-      continue-on-error: true
-      uses: actions/upload-artifact@v3
-      with:
-        name: maven-test-report-artifact-spark-1
-        path: lakesoul-spark/target/site
-        retention-days: 5
-        if-no-files-found: error
+      - uses: actions/checkout@v3
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - name: Install psql
+        run: sudo apt-get install -y postgresql-client-14
+      - name: Init PG
+        run: |
+          ./script/meta_init_for_local_test.sh -j 2
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
+      - uses: actions/download-artifact@v3
+        with:
+          name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
+          path: ./native-io/target/release/
+      - name: Build with Maven
+        run: |
+          mvn -B test -pl lakesoul-spark -am -Pcross-build -Pparallel-test --file pom.xml -Dtest='UpdateScalaSuite,AlterTableByNameSuite,ReadSuite,UpdateSQLSuite,ParquetNativeFilterSuite,DeleteScalaSuite,DeleteSQLSuite,ParquetV2FilterSuite,ParquetScanSuite,UpsertSuiteBase' -Dsurefire.failIfNoSpecifiedTests=false
+      - name: Generate Report Site
+        if: always()
+        run: |
+          mvn surefire-report:report-only -pl lakesoul-spark -am
+      - name: Upload Test Report
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@v3
+        with:
+          name: maven-test-report-artifact-spark-1
+          path: lakesoul-spark/target/site
+          retention-days: 5
+          if-no-files-found: error
 
   spark-test-2:
     runs-on: ubuntu-latest
-    needs: [build-linux-x86_64]
+    needs: [ build-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -147,6 +151,10 @@ jobs:
       - name: Init PG
         run: |
           ./script/meta_init_for_local_test.sh -j 2
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
@@ -170,7 +178,7 @@ jobs:
 
   flink-test-1:
     runs-on: ubuntu-latest
-    needs: [build-linux-x86_64]
+    needs: [ build-linux-x86_64 ]
 
     services:
       # Label used to access the service container
@@ -206,6 +214,10 @@ jobs:
       - name: Init PG
         run: |
           PGPASSWORD=lakesoul_test psql -h localhost -p 5432 -U lakesoul_test -f script/meta_init.sql lakesoul_test
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
       - uses: actions/download-artifact@v3
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test

.github/workflows/native-build.yml

@@ -105,7 +105,7 @@ jobs:
 
   build-maven-package:
     runs-on: ubuntu-latest
-    needs: [build-linux-x86_64, build-windows-x86_64, build-macos-x86_64]
+    needs: [ build-linux-x86_64, build-windows-x86_64, build-macos-x86_64 ]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/download-artifact@v3
@@ -126,6 +126,10 @@ jobs:
           java-version: '8'
           distribution: 'temurin'
           cache: maven
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
       - name: Build with Maven
         run: |
           MAVEN_OPTS="-Xmx4000m" mvn -q -B package --file pom.xml -Pcross-build -DskipTests -Dmaven.test.skip=true

.gitignore

@@ -13,5 +13,6 @@ dependency-reduced-pom.xml
 /lakesoul-spark/spark-warehouse/
 dependency-reduced-pom.xml
 /lakesoul-spark/spark-warehouse/
+lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/entity/
 /script/benchmark/work-dir/*.jar
 *.DS_Store

lakesoul-common/pom.xml

@@ -45,6 +45,25 @@
                     </execution>
                 </executions>
             </plugin>
+
+            <plugin>
+                <groupId>org.xolstice.maven.plugins</groupId>
+                <artifactId>protobuf-maven-plugin</artifactId>
+                <version>${protobuf.plugin.version}</version>
+                <configuration>
+                    <attachProtoSources>false</attachProtoSources>
+                    <clearOutputDirectory>false</clearOutputDirectory>
+                    <protoSourceRoot>${basedir}/../native-metadata/proto/src</protoSourceRoot>
+                    <outputDirectory>${basedir}/src/main/java</outputDirectory>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 
@@ -54,6 +73,8 @@
         <scala.binary.version>2.12</scala.binary.version>
         <mysql.version>8.0.30</mysql.version>
         <local.scope>provided</local.scope>
+        <protobuf.plugin.version>0.6.1</protobuf.plugin.version>
+        <protoc.version>3.5.1</protoc.version>
     </properties>
 
     <dependencies>
@@ -106,6 +127,12 @@
             <scope>compile</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.google.protobuf</groupId>
+            <artifactId>protobuf-java</artifactId>
+            <version>3.22.0</version>
+        </dependency>
+
     </dependencies>
 
 </project>

lakesoul-common/src/main/java/com/dmetasoul/lakesoul/meta/DBConfig.java

@@ -21,6 +21,10 @@ public abstract class DBConfig {
 
     static int MAX_COMMIT_ATTEMPTS = 5;
 
+    public static String LAKESOUL_DEFAULT_NAMESPACE = "default";
+
+    public static String LAKESOUL_NAMESPACE_LEVEL_SPLITTER = ".";
+
     public static String LAKESOUL_NULL_STRING = "__L@KE$OUL_NULL__";
 
     public static String LAKESOUL_EMPTY_STRING = "__L@KE$OUL_EMPTY_STRING__";