Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,14 @@

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.io.CharStreams;
import org.apache.spark.network.shuffle.protocol.ExecutorShuffleInfo;
import org.apache.spark.network.util.MapConfigProvider;
import org.apache.spark.network.util.TransportConf;
import org.apache.spark.network.shuffle.ExternalShuffleBlockResolver.AppExecId;
import org.apache.spark.util.SparkStreamUtils$;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -83,23 +82,17 @@ public void testSortShuffleBlocks() throws IOException {

try (InputStream block0Stream = resolver.getBlockData(
"app0", "exec0", 0, 0, 0).createInputStream()) {
String block0 =
CharStreams.toString(new InputStreamReader(block0Stream, StandardCharsets.UTF_8));
assertEquals(sortBlock0, block0);
assertEquals(sortBlock0, SparkStreamUtils$.MODULE$.toString(block0Stream));
}

try (InputStream block1Stream = resolver.getBlockData(
"app0", "exec0", 0, 0, 1).createInputStream()) {
String block1 =
CharStreams.toString(new InputStreamReader(block1Stream, StandardCharsets.UTF_8));
assertEquals(sortBlock1, block1);
assertEquals(sortBlock1, SparkStreamUtils$.MODULE$.toString(block1Stream));
}

try (InputStream blocksStream = resolver.getContinuousBlocksData(
"app0", "exec0", 0, 0, 0, 2).createInputStream()) {
String blocks =
CharStreams.toString(new InputStreamReader(blocksStream, StandardCharsets.UTF_8));
assertEquals(sortBlock0 + sortBlock1, blocks);
assertEquals(sortBlock0 + sortBlock1, SparkStreamUtils$.MODULE$.toString(blocksStream));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,8 @@

package org.apache.spark.deploy

import java.io._
import java.nio.charset.StandardCharsets

import com.google.common.io.CharStreams

import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
import org.apache.spark.internal.config._
import org.apache.spark.network.shuffle.{ExternalBlockHandler, ExternalShuffleBlockResolver}
Expand Down Expand Up @@ -110,7 +107,7 @@ abstract class ExternalShuffleServiceDbSuite extends SparkFunSuite {
blockResolver = blockHandler.getBlockResolver

val block0Stream = blockResolver.getBlockData("app0", "exec0", 0, 0, 0).createInputStream
val block0 = CharStreams.toString(new InputStreamReader(block0Stream, StandardCharsets.UTF_8))
val block0 = Utils.toString(block0Stream)
block0Stream.close()
assert(sortBlock0 == block0)
// pass
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@

package org.apache.spark.network.netty

import java.io.InputStreamReader
import java.nio._
import java.nio.charset.StandardCharsets
import java.util.concurrent.TimeUnit
Expand All @@ -26,7 +25,6 @@ import scala.concurrent.Promise
import scala.concurrent.duration._
import scala.util.{Failure, Success, Try}

import com.google.common.io.CharStreams
import org.mockito.Mockito._
import org.scalatest.matchers.must.Matchers
import org.scalatest.matchers.should.Matchers._
Expand All @@ -40,7 +38,7 @@ import org.apache.spark.network.buffer.{ManagedBuffer, NioManagedBuffer}
import org.apache.spark.network.shuffle.BlockFetchingListener
import org.apache.spark.serializer.{JavaSerializer, SerializerManager}
import org.apache.spark.storage.{BlockId, ShuffleBlockId}
import org.apache.spark.util.{SslTestUtils, ThreadUtils}
import org.apache.spark.util.{SslTestUtils, ThreadUtils, Utils}

class NettyBlockTransferSecuritySuite extends SparkFunSuite with MockitoSugar with Matchers {

Expand Down Expand Up @@ -150,9 +148,7 @@ class NettyBlockTransferSecuritySuite extends SparkFunSuite with MockitoSugar wi

val result = fetchBlock(exec0, exec1, "1", blockId) match {
case Success(buf) =>
val actualString = CharStreams.toString(
new InputStreamReader(buf.createInputStream(), StandardCharsets.UTF_8))
actualString should equal(blockString)
Utils.toString(buf.createInputStream()) should equal(blockString)
buf.release()
Success(())
case Failure(t) =>
Expand Down
4 changes: 4 additions & 0 deletions dev/checkstyle.xml
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,10 @@
<property name="format" value="FileUtils\.deleteQuietly"/>
<property name="message" value="Use deleteQuietly of JavaUtils/SparkFileUtils/Utils instead." />
</module>
<module name="RegexpSinglelineJava">
<property name="format" value="CharStreams\.toString"/>
<property name="message" value="Use toString of SparkStreamUtils or Utils instead." />
</module>
<!-- support structured logging -->
<module name="RegexpSinglelineJava">
<property name="format" value="org\.slf4j\.(Logger|LoggerFactory)" />
Expand Down
5 changes: 5 additions & 0 deletions scalastyle-config.xml
Original file line number Diff line number Diff line change
Expand Up @@ -722,6 +722,11 @@ This file is divided into 3 sections:
<customMessage>Use toString of SparkStreamUtils or Utils instead.</customMessage>
</check>

<check customId="charstreamstostring" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
<parameters><parameter name="regex">\bCharStreams\.toString\b</parameter></parameters>
<customMessage>Use toString of SparkStreamUtils or Utils instead.</customMessage>
</check>

<check customId="ioutilswrite" level="error" class="org.scalastyle.file.RegexChecker" enabled="true">
<parameters><parameter name="regex">\bIOUtils\.write\b</parameter></parameters>
<customMessage>Use Java `write` instead.</customMessage>
Expand Down