Skip to content
Permalink
Browse files

Refactor build

  • Loading branch information...
mhamilton723 committed Jul 1, 2019
1 parent 77d7cb4 commit 5742c0e164d54f3b87e2e9007c249d45944f61ec
@@ -0,0 +1,31 @@
name := "mmlspark"
version := "0.17.1"
scalaVersion := "2.11.12"

val sparkVersion = "2.4.0"
val sprayVersion = "1.3.4"
val cntkVersion = "2.4"

libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion % "compile",
"org.apache.spark" %% "spark-mllib" % sparkVersion % "compile",
"org.scalactic" %% "scalactic" % "3.0.5",
"org.scalatest" %% "scalatest" % "3.0.5",
"io.spray" %% "spray-json" % "1.3.2",
"com.microsoft.cntk" % "cntk" % cntkVersion,
"org.openpnp" % "opencv" % "3.2.0-1",
"com.jcraft" % "jsch" % "0.1.54",
"com.jcraft" % "jsch" % "0.1.54",
"org.apache.httpcomponents" % "httpclient" % "4.5.6",
"com.microsoft.ml.lightgbm" % "lightgbmlib" % "2.2.350"
)

lazy val IntegrationTest2 = config("it").extend(Test)

val settings = Seq((scalastyleConfig in Test) := baseDirectory.value / "scalastyle-test-config.xml") ++
inConfig(IntegrationTest2)(Defaults.testSettings)

lazy val mmlspark = (project in file("."))
.configs(IntegrationTest2)
.enablePlugins(ScalaUnidocPlugin)
.settings(settings: _*)
@@ -0,0 +1,22 @@
resources:
- repo: self
trigger:
- build-refactor
jobs:

- job: UnitTests
pool:
vmImage: ubuntu-16.04
steps:
- task: AzureCLI@1
displayName: 'Unit Test'
inputs:
azureSubscription: 'Findable Incubation (ca9d21ff-2a46-4e8b-bf06-8d65242342e5)'
scriptLocation: inlineScript
inlineScript: 'sbt test'
- task: PublishTestResults@2
displayName: 'Publish Test Results **/test-reports/*.xml'
inputs:
testResultsFiles: '**/test-reports/*.xml'
failTaskOnFailedTests: true
condition: succeededOrFailed()
@@ -0,0 +1 @@
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9")
@@ -0,0 +1 @@
sbt.version = 1.2.8
@@ -0,0 +1 @@
libraryDependencies ++= Seq("io.spray" %% "spray-json" % "1.3.4")
@@ -0,0 +1,2 @@
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.2")
@@ -0,0 +1 @@
sbt.version=1.2.6
@@ -0,0 +1,79 @@
<scalastyle>
<name>Scalastyle Module Configuration (test)</name>
<check level="error" class="org.scalastyle.file.FileLengthChecker" enabled="true"><parameters>
<parameter name="maxFileLength">800</parameter></parameters></check>
<check level="error" class="org.scalastyle.file.FileTabChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.file.FileLineLengthChecker" enabled="true"><parameters>
<parameter name="maxLineLength">120</parameter></parameters></check>
<check level="error" class="org.scalastyle.file.NewLineAtEofChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.file.RegexChecker" enabled="true"><parameters>
<parameter name="regex">


</parameter></parameters></check>
<check level="warning" class="org.scalastyle.scalariform.TokenChecker" enabled="true"><parameters>
<parameter name="regex">.{33}</parameter></parameters></check>
<check level="error" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true"><parameters>
<parameter name="header">^// Copyright \(C\) Microsoft Corporation\. All rights reserved\.
// Licensed under the MIT License\. See LICENSE in project root for information\.

package (?:com\.microsoft\.ml\.spark|org\.apache\.spark|com\.microsoft\.CNTK|com\.microsoft\.ml\.lightgbm)[.
]</parameter>
<parameter name="regex">true</parameter></parameters></check>
<check level="error" class="org.scalastyle.file.WhitespaceEndOfLineChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.SpacesAfterPlusChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.SpacesBeforePlusChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.NoWhitespaceBeforeLeftBracketChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.NoWhitespaceAfterLeftBracketChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.EmptyClassChecker" enabled="true"></check>
<check level="warning" class="org.scalastyle.scalariform.EnsureSingleSpaceAfterTokenChecker" enabled="true"><parameters>
<parameter name="tokens">COLON</parameter></parameters></check>
<check level="error" class="org.scalastyle.scalariform.EnsureSingleSpaceBeforeTokenChecker" enabled="true"></check>
<check level="warning" class="org.scalastyle.scalariform.DisallowSpaceAfterTokenChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.DisallowSpaceBeforeTokenChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.ClassNamesChecker" enabled="true"><parameters>
<parameter name="regex">^[A-Z][A-Za-z0-9]*$</parameter></parameters></check>
<check level="error" class="org.scalastyle.scalariform.ObjectNamesChecker" enabled="true"><parameters>
<parameter name="regex">^[A-Za-z][A-Za-z0-9]*$</parameter></parameters></check>
<check level="error" class="org.scalastyle.scalariform.PackageObjectNamesChecker" enabled="true"><parameters>
<parameter name="regex">^[a-z][A-Za-z]*$</parameter></parameters></check>
<check level="warning" class="org.scalastyle.scalariform.FieldNamesChecker" enabled="true"><parameters>
<parameter name="regex">^([a-z][A-Za-z0-9]*| *\( *)$</parameter></parameters></check>
<check level="warning" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="true"><parameters>
<parameter name="regex">^[a-z][A-Za-z0-9]*(_=)?$</parameter></parameters></check>
<check level="warning" class="org.scalastyle.scalariform.ClassTypeParameterChecker" enabled="true"><parameters>
<parameter name="regex">^[A-Z_]$</parameter></parameters></check>
<check level="error" class="org.scalastyle.scalariform.EqualsHashCodeChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.IllegalImportsChecker" enabled="true"><parameters>
<parameter name="illegalImports">sun._</parameter></parameters></check>
<check level="error" class="org.scalastyle.scalariform.DeprecatedJavaChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true"><parameters>
<parameter name="maxParameters">12</parameter></parameters></check>
<check level="warning" class="org.scalastyle.scalariform.MethodLengthChecker" enabled="true"><parameters>
<parameter name="maxLength">50</parameter></parameters></check>
<check level="error" class="org.scalastyle.scalariform.NumberOfTypesChecker" enabled="true"><parameters>
<parameter name="maxTypes">40</parameter></parameters></check>
<check level="error" class="org.scalastyle.scalariform.NumberOfMethodsInTypeChecker" enabled="true"><parameters>
<parameter name="maxMethods">50</parameter></parameters></check>
<check level="warning" class="org.scalastyle.scalariform.CyclomaticComplexityChecker" enabled="true"><parameters>
<parameter name="maximum">10</parameter></parameters></check>
<check level="error" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.UppercaseLChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.ProcedureDeclarationChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.RedundantIfChecker" enabled="true"></check>
<check level="warning" class="org.scalastyle.scalariform.WhileChecker" enabled="true"></check>
<check level="warning" class="org.scalastyle.scalariform.ReturnChecker" enabled="true"></check>
<check level="warning" class="org.scalastyle.scalariform.NullChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.NoCloneChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.StructuralTypeChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.CovariantEqualsChecker" enabled="true"></check>
<check level="error" class="org.scalastyle.scalariform.ImportOrderChecker" enabled="true"><parameters>
<parameter name="groups">our,scala,java,other</parameter>
<parameter name="our">com.microsoft.ml.spark[.].+</parameter>
<parameter name="scala">scala[.].+</parameter>
<parameter name="java">java[.].+</parameter>
<parameter name="other">.+</parameter></parameters></check>
<check level="error" class="org.scalastyle.scalariform.SimplifyBooleanExpressionChecker" enabled="true"></check>
<check level="warning" class="org.scalastyle.scalariform.NotImplementedErrorUsage" enabled="true"></check>
</scalastyle>

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

@@ -3,21 +3,13 @@

package com.microsoft.ml.spark.core.utils

import java.io.{IOException, InputStream, ObjectInputStream, ObjectStreamClass}
import java.net.{URL, URLClassLoader}
import java.util.jar.JarFile
import java.io.{InputStream, ObjectInputStream, ObjectStreamClass}

import com.microsoft.ml.spark.core.env.FileUtilities
import com.microsoft.ml.spark.core.env.FileUtilities.File
import org.scalatest.exceptions.TestFailedException

import scala.reflect.{ClassTag, _}
import com.microsoft.ml.spark.core.env.FileUtilities._
import org.spark_project.guava.reflect.ClassPath
import org.spark_project.guava.reflect.ClassPath.ClassInfo

import collection.JavaConverters._
import scala.util.Try
import scala.collection.JavaConverters._
import scala.reflect.{ClassTag, _}

/** Contains logic for loading classes. */
object JarLoadingUtils {
@@ -27,10 +19,10 @@ object JarLoadingUtils {
filename.substring(0, classNameEnd).replace('/', '.')
}

private lazy val allClasses = {
private lazy val allClasses = {
ClassPath.from(getClass.getClassLoader)
//.getTopLevelClassesRecursive("com.microsoft").asScala.toList
.getResources().asScala.toList
.getResources.asScala.toList
.map(ri => className(ri.getResourceName))
.filter(_.startsWith("com.microsoft.ml"))
.flatMap { cn =>

This file was deleted.

0 comments on commit 5742c0e

Please sign in to comment.
You can’t perform that action at this time.