Skip to content

Commit

Permalink
[KYUUBI-SHADED #29] [KYUUBI-SHADED #28] Step 2/2: Overwrite SnapStrea…
Browse files Browse the repository at this point in the history
…m to remove deps of snappy in ZK client 3.6

### _Why are the changes needed?_

This is step 2 of #28

> 2. remove the Snappy support (simply throw `UnsupportedOperationException`) in `org.apache.zookeeper.server.persistence.SnapStream` and snappy deps

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request

- [x] Verified through apache/kyuubi#5783

Closes #29 from pan3793/snappy-2.

a109d1c [Cheng Pan] [KYUUBI-SHADED #28] Step 1/2: Overwrite SnapStream to remove deps of snappy in ZK client 3.6

Authored-by: Cheng Pan <chengpan@apache.org>
Signed-off-by: Cheng Pan <chengpan@apache.org>
  • Loading branch information
pan3793 committed Dec 4, 2023
1 parent 06eb6e7 commit a51fe58
Show file tree
Hide file tree
Showing 4 changed files with 8 additions and 36 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ under the License.
<zookeeper.version>3.6.4</zookeeper.version>
<curator.version>5.4.0</curator.version>
<netty.version>4.1.91.Final</netty.version>
<snappy.version>1.1.8.4</snappy.version>
</properties>

<dependencyManagement>
Expand Down Expand Up @@ -161,11 +160,5 @@ under the License.
</exclusion>
</exclusions>
</dependency>
<dependency>
<!-- required by kyuubi embedded Zookeeper server to bootstrap -->
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>${snappy.version}</version>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.zip.Adler32;
import java.util.zip.CheckedInputStream;
import java.util.zip.CheckedOutputStream;
Expand All @@ -40,9 +39,6 @@
import org.apache.zookeeper.common.AtomicFileOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xerial.snappy.SnappyCodec;
import org.xerial.snappy.SnappyInputStream;
import org.xerial.snappy.SnappyOutputStream;

/** Represent the Stream used in serialize and deserialize the Snapshot. */
public class SnapStream {
Expand Down Expand Up @@ -106,8 +102,8 @@ public static CheckedInputStream getInputStream(File file) throws IOException {
is = new GZIPInputStream(fis);
break;
case SNAPPY:
is = new SnappyInputStream(fis);
break;
throw new UnsupportedOperationException(
"[KYUUBI-SHADED #28] " + ZOOKEEPER_SHAPSHOT_STREAM_MODE + " does not support snappy");
case CHECKED:
default:
is = new BufferedInputStream(fis);
Expand All @@ -131,8 +127,8 @@ public static CheckedOutputStream getOutputStream(File file, boolean fsync) thro
os = new GZIPOutputStream(fos);
break;
case SNAPPY:
os = new SnappyOutputStream(fos);
break;
throw new UnsupportedOperationException(
"[KYUUBI-SHADED #28] " + ZOOKEEPER_SHAPSHOT_STREAM_MODE + " does not support snappy");
case CHECKED:
default:
os = new BufferedOutputStream(fos);
Expand Down Expand Up @@ -184,8 +180,8 @@ public static boolean isValidSnapshot(File file) throws IOException {
isValid = isValidGZipStream(file);
break;
case SNAPPY:
isValid = isValidSnappyStream(file);
break;
throw new UnsupportedOperationException(
"[KYUUBI-SHADED #28] " + ZOOKEEPER_SHAPSHOT_STREAM_MODE + " does not support snappy");
case CHECKED:
default:
isValid = isValidCheckedStream(file);
Expand Down Expand Up @@ -252,20 +248,8 @@ private static boolean isValidGZipStream(File f) throws IOException {
* @throws IOException
*/
private static boolean isValidSnappyStream(File f) throws IOException {
byte[] byteArray = new byte[SnappyCodec.MAGIC_LEN];
try (FileInputStream fis = new FileInputStream(f)) {
if (SnappyCodec.MAGIC_LEN != fis.read(byteArray, 0, SnappyCodec.MAGIC_LEN)) {
LOG.error("Read incorrect number of bytes from {}", f.getName());
return false;
}
ByteBuffer bb = ByteBuffer.wrap(byteArray);
byte[] magicHeader = new byte[SnappyCodec.MAGIC_LEN];
bb.get(magicHeader, 0, SnappyCodec.MAGIC_LEN);
return Arrays.equals(magicHeader, SnappyCodec.getMagicHeader());
} catch (FileNotFoundException e) {
LOG.error("Unable to open file {}", f.getName(), e);
return false;
}
throw new UnsupportedOperationException(
"[KYUUBI-SHADED #28] " + ZOOKEEPER_SHAPSHOT_STREAM_MODE + " does not support snappy");
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,3 @@ This project bundles the following dependencies under the Apache Software Licens
- org.apache.curator:curator-recipes:5.4.0
- org.apache.zookeeper:zookeeper-jute:3.6.4
- org.apache.zookeeper:zookeeper:3.6.4
- org.xerial.snappy:xerial-java:1.1.8.4
4 changes: 0 additions & 4 deletions kyuubi-relocated-zookeeper-parent/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,6 @@ under the License.
<pattern>com.google</pattern>
<shadedPattern>${shading.prefix}.google</shadedPattern>
</relocation>
<relocation>
<pattern>org.xerial.snappy</pattern>
<shadedPattern>${shading.prefix}.snappy</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
Expand Down

0 comments on commit a51fe58

Please sign in to comment.