Skip to content
Permalink
Browse files

Scalding updates for Scala 2.12 (#1646)

* start upgrade to 2.12

* Bump bijection, chill

* Add Scala 2.12 related repl code

* Scala 2.12 version bumps + _equals fix

* Suppress Option[Partial] warnings

* Add fallback for scrooge-serializer on Scala 2.10

* Add scala 2.12 build targets

* Fix parquet-scrooge and parquet tests on 2.12

* Drop support for 2.10

* Fix scalding-serialization tests

* Get thrift macros to compile with latest Scalacheck

* Fix scalding-db tests

* Exclude guava from scrooge imports

* Use mima 0.1.14

* Update scalding-thrift-macros jdk version to java8

* Fix OrderedSerialization test failures

* Break boxed lambdas into 2 lists to get around Scala 2.12 bug

* Try catching CNF in LineNumber

* Drop line numbers from hashJoin scalding hadoop test

* Switch default to jdk8 as scrooge deps have been built with that

* Update scald.rb to pick up 2.12

* Try latest jline version

* Try and fix merge issues

* Wrap Execution line better

* Remove scrooge method

* Drop _equals method

* Try private[ser] for Boxed issues

* Reorder hashCode and equals
  • Loading branch information...
Piyush Narang
Piyush Narang committed Mar 23, 2017
1 parent 0b07c4b commit 12aaa7e1f4315c2227080f18856003d2780e7e8f
Showing with 189 additions and 184 deletions.
  1. +20 −20 .travis.yml
  2. +56 −94 build.sbt
  3. +4 −5 project/plugins.sbt
  4. +8 −2 scalding-core/src/main/scala/com/twitter/scalding/Execution.scala
  5. +2 −1 scalding-core/src/main/scala/com/twitter/scalding/GroupBuilder.scala
  6. +2 −1 scalding-core/src/main/scala/com/twitter/scalding/JobTest.scala
  7. +13 −2 scalding-core/src/main/scala/com/twitter/scalding/LineNumber.scala
  8. +1 −1 scalding-core/src/main/scala/com/twitter/scalding/Mode.scala
  9. +7 −7 scalding-core/src/main/scala/com/twitter/scalding/Operations.scala
  10. +1 −0 scalding-core/src/main/scala/com/twitter/scalding/TestTapFactory.scala
  11. +1 −1 scalding-core/src/main/scala/com/twitter/scalding/TuplePacker.scala
  12. +1 −0 scalding-core/src/main/scala/com/twitter/scalding/macros/impl/TypeDescriptorProviderImpl.scala
  13. +1 −1 scalding-core/src/main/scala/com/twitter/scalding/mathematics/Matrix2.scala
  14. +1 −1 scalding-core/src/main/scala/com/twitter/scalding/reducer_estimation/Common.scala
  15. +1 −1 scalding-core/src/main/scala/com/twitter/scalding/serialization/WrappedSerialization.scala
  16. +1 −1 scalding-core/src/main/scala/com/twitter/scalding/typed/CoGrouped.scala
  17. +1 −0 scalding-core/src/main/scala/com/twitter/scalding/typed/TypedPipe.scala
  18. +8 −1 scalding-db/src/test/scala/com/twitter/scalding/db/macros/MacrosUnitTests.scala
  19. +2 −2 scalding-hadoop-test/src/test/scala/com/twitter/scalding/platform/PlatformExecutionTest.scala
  20. +7 −6 scalding-hadoop-test/src/test/scala/com/twitter/scalding/platform/PlatformTest.scala
  21. +3 −3 ...g-hraven/src/main/scala/com/twitter/scalding/hraven/reducer_estimation/HRavenHistoryService.scala
  22. +0 −8 scalding-parquet-scrooge-fixtures/src/test/resources/test.thrift
  23. +11 −0 scalding-repl/src/main/scala-2.12/com/twitter/scalding/ILoopCompat.scala
  24. +13 −3 scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Boxed.scala
  25. +1 −1 scalding-serialization/src/main/scala/com/twitter/scalding/serialization/Writer.scala
  26. +6 −6 .../scala/com/twitter/scalding/serialization/macros/impl/ordered_serialization/SealedTraitLike.scala
  27. +1 −1 ...witter/scalding/serialization/macros/impl/ordered_serialization/providers/ProductOrderedBuf.scala
  28. +2 −2 ...ialization/src/test/scala/com/twitter/scalding/serialization/macros/MacroOrderingProperties.scala
  29. +2 −2 ...erialization/src/test/scala/com/twitter/scalding/serialization/macros/TraversableHelperLaws.scala
  30. +6 −6 ...cros/src/main/scala/com/twitter/scalding/thrift/macros/impl/ordered_serialization/UnionLike.scala
  31. +4 −5 scalding-thrift-macros/src/test/scala/com/twitter/scalding/thrift/macros/ScroogeGenerators.scala
  32. +2 −0 scripts/scald.rb
@@ -1,5 +1,5 @@
language: scala
jdk: oraclejdk7
jdk: oraclejdk8
sudo: false

before_install:
@@ -27,55 +27,55 @@ addons:
matrix:
include:
#BASE TESTS
- scala: 2.10.6
- scala: 2.11.8
env: BUILD="base" TEST_TARGET="scalding-args scalding-date maple"
script: "scripts/run_test.sh"

- scala: 2.11.8
- scala: 2.12.1
env: BUILD="base" TEST_TARGET="scalding-args scalding-date maple"
script: "scripts/run_test.sh"

- scala: 2.10.6
- scala: 2.11.8
env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons scalding-parquet scalding-parquet-scrooge"
script: "scripts/run_test.sh"

- scala: 2.11.8
- scala: 2.12.1
env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons scalding-parquet scalding-parquet-scrooge"
script: "scripts/run_test.sh"

- scala: 2.10.6
- scala: 2.11.8
env: BUILD="base" TEST_TARGET="scalding-core scalding-jdbc scalding-json scalding-db"
script: "scripts/run_test.sh"

- scala: 2.11.8
- scala: 2.12.1
env: BUILD="base" TEST_TARGET="scalding-core scalding-jdbc scalding-json scalding-db"
script: "scripts/run_test.sh"

- scala: 2.10.6
- scala: 2.11.8
env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
script: "scripts/run_test.sh"

- scala: 2.11.8
- scala: 2.12.1
env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
script: "scripts/run_test.sh"

- scala: 2.10.6
- scala: 2.11.8
env: BUILD="base" TEST_TARGET="scalding-serialization"
script: "scripts/run_test.sh"

- scala: 2.11.8
- scala: 2.12.1
env: BUILD="base" TEST_TARGET="scalding-serialization"
script: "scripts/run_test.sh"

- scala: 2.10.6
- scala: 2.11.8
env: BUILD="base" TEST_TARGET="scalding-thrift-macros"
script: "scripts/run_test.sh"

- scala: 2.11.8
- scala: 2.12.1
env: BUILD="base" TEST_TARGET="scalding-thrift-macros"
script: "scripts/run_test.sh"

- scala: 2.10.6
- scala: 2.11.8
env: BUILD="test tutorials and matrix tutorials and repl" TEST_TARGET="scalding-repl"
script:
- "scripts/run_test.sh"
@@ -84,7 +84,7 @@ matrix:
- "scripts/build_assembly_no_test.sh scalding-assembly"
- "scripts/test_matrix_tutorials.sh"

- scala: 2.11.8
- scala: 2.12.1
env: BUILD="test tutorials and matrix tutorials and repl" TEST_TARGET="scalding-repl"
script:
- "scripts/run_test.sh"
@@ -93,20 +93,20 @@ matrix:
- "scripts/build_assembly_no_test.sh scalding-assembly"
- "scripts/test_matrix_tutorials.sh"

- scala: 2.10.6
env: BUILD="test repl and typed tutorials"
- scala: 2.11.8
env: BUILD="test repl and typed tutorials and microsite"
script:
- ./sbt ++$TRAVIS_SCALA_VERSION clean docs/makeMicrosite
- "scripts/build_assembly_no_test.sh scalding-repl"
- "scripts/test_repl_tutorial.sh"
- "scripts/build_assembly_no_test.sh scalding-core"
- "scripts/test_typed_tutorials.sh"
- "scripts/build_assembly_no_test.sh execution-tutorial"
- "scripts/test_execution_tutorial.sh"

- scala: 2.11.8
env: BUILD="test repl and typed tutorials and microsite"
- scala: 2.12.1
env: BUILD="test repl and typed tutorials"
script:
- ./sbt ++$TRAVIS_SCALA_VERSION clean docs/makeMicrosite
- "scripts/build_assembly_no_test.sh scalding-repl"
- "scripts/test_repl_tutorial.sh"
- "scripts/build_assembly_no_test.sh scalding-core"
Oops, something went wrong.

0 comments on commit 12aaa7e

Please sign in to comment.
You can’t perform that action at this time.