Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove old test to make Spark run out of memory #160

Merged
merged 2 commits into from
Oct 22, 2018
Merged

Remove old test to make Spark run out of memory #160

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
103e0ec
Removed an old test I added to check if Spark ran out of memory when …
Jauntbox Oct 22, 2018
0fdd309
Merge branch 'master' into km/remove-bad-test
tovbinm Oct 22, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 1 addition & 31 deletions core/src/test/scala/com/salesforce/op/stages/impl/preparators/SanityCheckerTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -532,37 +532,7 @@ class SanityCheckerTest extends OpEstimatorSpec[OPVector, BinaryModel[RealNN, OP
featuresToDrop, featuresWithNaNCorr)
}

// TODO: Not sure if we should do this test since it may not fail if spark settings are changed
it should "fail (due to a Kryo buffer overflow) when calculating a large (5k x 5k) correlation matrix " in {
val numHashes = 5000

val vectorized = textMap.vectorize(
shouldPrependFeatureName = TransmogrifierDefaults.PrependFeatureName,
cleanText = false,
cleanKeys = TransmogrifierDefaults.CleanKeys,
others = Array.empty,
trackNulls = TransmogrifierDefaults.TrackNulls,
numHashes = numHashes
)

val checkedFeatures = new SanityChecker()
.setCheckSample(1.0)
.setRemoveBadFeatures(true)
.setRemoveFeatureGroup(true)
.setProtectTextSharedHash(true)
.setFeatureLabelCorrOnly(false)
.setMinCorrelation(0.0)
.setMaxCorrelation(0.8)
.setMaxCramersV(0.8)
.setInput(targetResponse, vectorized)
.getOutput()

checkedFeatures.originStage shouldBe a[SanityChecker]

intercept[SparkException](new OpWorkflow().setResultFeatures(vectorized, checkedFeatures).transform(textData))
}

it should "not fail when calculating feature-label correlations on that same 5k element feature vector" in {
it should "not fail when calculating feature-label correlations on a 5k element feature vector" in {
val numHashes = 5000

val vectorized = textMap.vectorize(
Expand Down