Permalink
Browse files

Refactoring

  • Loading branch information...
alexarchambault committed Nov 26, 2016
1 parent e843ee8 commit 860e48920d8a45fabe6e15e2a2613fb6270f9203
Showing with 636 additions and 6,751 deletions.
  1. +0 −170 api/src/main/scala/jupyter/api/API.scala
  2. +0 −69 api/src/main/scala/jupyter/api/Base64.scala
  3. +17 −0 api/src/main/scala/jupyter/api/JupyterApi.scala
  4. +29 −22 build.sbt
  5. +2 −1 cli/src/main/resources/logback.xml
  6. +31 −41 cli/src/main/scala/jupyter/scala/JupyterScala.scala
  7. BIN coursier
  8. +0 −445 examples/libraries/PureCSV.ipynb
  9. +0 −773 examples/libraries/Simulacrum.ipynb
  10. +0 −331 examples/libraries/psp-std.ipynb
  11. +0 −851 examples/tests/spark-1.2-local.ipynb
  12. +0 −870 examples/tests/spark-1.3-local.ipynb
  13. +0 −2,197 examples/tutorials/Macrology.ipynb
  14. BIN jupyter-scala
  15. BIN jupyter-scala-2.10
  16. +0 −106 kernel/src/main/scala/jupyter/scala/APIImpl.scala
  17. +0 −39 kernel/src/main/scala/jupyter/scala/BridgeConfig.scala
  18. +217 −0 kernel/src/main/scala/jupyter/scala/Interp.scala
  19. +0 −231 kernel/src/main/scala/jupyter/scala/ScalaInterpreter.scala
  20. +0 −22 kernel/src/main/scala/jupyter/scala/WebDisplay.scala
  21. +0 −22 kernel/src/main/scala/jupyter/scala/config/ScalaModule.scala
  22. +0 −23 kernel/src/test/resources/log4j.properties
  23. +0 −9 kernel/src/test/scala-2.10/jupyter/scala/LocalSpark11Tests.scala
  24. +0 −131 kernel/src/test/scala/jupyter/scala/InterpreterChecker.scala
  25. +0 −69 kernel/src/test/scala/jupyter/scala/Tests.scala
  26. +0 −46 kernel/src/test/scala/jupyter/scala/nbtest/Definitions.scala
  27. +0 −30 kernel/src/test/scala/jupyter/scala/nbtest/JsonCodecs.scala
  28. +0 −22 kernel/src/test/scala/jupyter/scala/nbtest/NbTest.scala
  29. +0 −157 kernel/src/test/scala/jupyter/scala/nbtest/Notebook.scala
  30. +0 −14 kernel/src/test/scala/jupyter/scala/package.scala
  31. +0 −21 launch
  32. +0 −23 launch-2.10
  33. +1 −15 project/run-tests
  34. +183 −0 spark/src/main/scala/jupyter/spark/Spark.scala
  35. +9 −0 spark/src/main/scala/jupyter/spark/package.scala
  36. +140 −0 spark/src/main/scala/org/apache/spark/repl/ExecutorClassLoader.scala
  37. +6 −0 spark/src/main/scala/spark/repl/Main.scala
  38. +1 −1 version.sbt
@@ -1,170 +0,0 @@
package jupyter.api
import ammonite.api.{ Eval, Setup }
import ammonite.tprint.TPrint
import _root_.pprint.{ PPrint, Config }
import scala.reflect.runtime.universe.WeakTypeTag
trait API {
/**
* Tools related to loading external scripts and code
*/
implicit def classpath: ammonite.api.Classpath
implicit def setup: Setup
implicit def eval: Eval
/**
* Exposes some internals of the current interpreter
*/
implicit def interpreter: ammonite.api.Interpreter
/**
* Controls how things are pretty-printed
*/
implicit var pprintConfig: Config
/**
* Prettyprint the given `value` with no truncation. Optionally takes
* a number of lines to print.
*/
def show[T](
t: T,
width: Integer = null,
height: Integer = 0,
indent: Integer = null,
colors: _root_.pprint.Colors = null
)(implicit
cfg: Config = Config.Defaults.PPrintConfig,
pprint: PPrint[T]
): Unit
/**
* Opaque container of the currently processed Jupyter message.
*
* Required to send display data or Jupyter comm messages. Opaque
* not to add extra dependencies.
*/
implicit def evidence: Evidence
/**
* Jupyter publishing helper
*
* Allows to push display items to the front-end or to communicate with
* widgets through Jupyter comms (WIP)
*/
implicit def publish: jupyter.api.Publish[Evidence]
val display: Display = new Display {}
def printValue[T, U](
value: => T,
dummy: => U,
ident: String,
custom: Option[String]
)(implicit
cfg: Config,
tprint: TPrint[U],
pprint: PPrint[T],
tpe: WeakTypeTag[T]
): Iterator[String]
}
class APIHolder {
@transient var shell0: API = null
@transient lazy val shell = shell0
}
object APIHolder {
def initReplBridge(holder: Class[APIHolder], api: API) =
holder
.getDeclaredMethods
.find(_.getName == "shell0_$eq")
.get
.invoke(null, api)
}
/**
* Opaque container of a Jupyter message. Opaque not to add
* extra dependencies.
*/
final class Evidence private[jupyter] (private[jupyter] val underlying: Any)
trait Display {
import Base64._
/*
* FIXME The publish.display method only accepts data of type String.
* Support should be added for Seq[String] (base64 encoded data looks better
* this way) and Json.
*/
def html(html: String)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
publish.display("", "text/html" -> html)
}
def html(node: scala.xml.Node)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
html(node.toString())
}
def markdown(md: String)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
publish.display("", "text/markdown" -> md)
}
def md(md: String)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
markdown(md)
}
def svg(svg: String)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
publish.display("", "image/svg+xml" -> svg)
}
def svg(node: scala.xml.Node)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
svg(node.toString())
}
def png(data: Array[Byte])
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
publish.display("", "image/png" -> data.toBase64)
}
def png(data: java.awt.image.BufferedImage)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
import java.io.ByteArrayOutputStream
import javax.imageio.ImageIO
val baos: ByteArrayOutputStream = new ByteArrayOutputStream()
ImageIO.write(data, "png", baos)
png(baos.toByteArray)
}
def jpg(data: Array[Byte])
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
publish.display("", "image/jpeg" -> data.toBase64)
}
def jpg(data: java.awt.image.BufferedImage)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
import java.io.ByteArrayOutputStream
import javax.imageio.ImageIO
val baos: ByteArrayOutputStream = new ByteArrayOutputStream()
ImageIO.write(data, "jpg", baos)
jpg(baos.toByteArray)
}
def latex(latex: String)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
publish.display("", "text/latex" -> latex)
}
def pdf(data: Array[Byte])
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
publish.display("", "application/pdf" -> data.toBase64)
}
def javascript(code: String)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
publish.display("", "application/javascript" -> code)
}
def js(code: String)
(implicit publish: jupyter.api.Publish[Evidence], ev: Evidence): Unit = {
javascript(code)
}
}
@@ -1,69 +0,0 @@
package jupyter.api
import scala.collection.immutable.HashMap
/**
* Base64 encoder
* @author Mark Lister
* This software is distributed under the 2-Clause BSD license. See the
* LICENSE file in the root of the repository.
*
* Copyright (c) 2014 - 2015 Mark Lister
*
* The repo for this Base64 encoder lives at https://github.com/marklister/base64
* Please send your issues, suggestions and pull requests there.
*
*/
private[api] object Base64 {
private[this] val zero = Array(0, 0).map(_.toByte)
class B64Scheme(val encodeTable: IndexedSeq[Char]) {
lazy val decodeTable = HashMap(encodeTable.zipWithIndex: _ *)
}
lazy val base64 = new B64Scheme(('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ Seq('+', '/'))
lazy val base64Url = new B64Scheme(base64.encodeTable.dropRight(2) ++ Seq('-', '_'))
implicit class Encoder(b: Array[Byte]) {
lazy val pad = (3 - b.length % 3) % 3
def toBase64(implicit scheme: B64Scheme = base64): String = {
def sixBits(x: Array[Byte]): Array[Int] = {
val a = (x(0) & 0xfc) >> 2
val b = ((x(0) & 0x3) << 4) | ((x(1) & 0xf0) >> 4)
val c = ((x(1) & 0xf) << 2) | ((x(2) & 0xc0) >> 6)
val d = (x(2)) & 0x3f
Array(a, b, c, d)
}
((b ++ zero.take(pad)).grouped(3)
.flatMap(sixBits)
.map(scheme.encodeTable)
.toArray
.dropRight(pad) :+ "=" * pad)
.mkString
}
}
implicit class Decoder(s: String) {
lazy val cleanS = s.reverse.dropWhile(_ == '=').reverse
lazy val pad = s.length - cleanS.length
def toByteArray(implicit scheme: B64Scheme = base64): Array[Byte] = {
def threeBytes(s: String): Array[Byte] = {
val r = s.map(scheme.decodeTable(_)).foldLeft(0)((a, b) => (a << 6) | b)
Array((r >> 16).toByte, (r >> 8).toByte, r.toByte)
}
if (pad > 2 || s.length % 4 != 0) throw new java.lang.IllegalArgumentException("Invalid Base64 String:" + s)
try {
(cleanS + "A" * pad)
.grouped(4)
.map(threeBytes)
.flatten
.toArray
.dropRight(pad)
} catch {case e:NoSuchElementException => throw new java.lang.IllegalArgumentException("Invalid Base64 String:" + s) }
}
}
}
@@ -0,0 +1,17 @@
package jupyter.api
import ammonite.repl.FullRuntimeAPI
import ammonite.runtime.APIHolder
trait JupyterApi {
/**
* Jupyter publishing helper
*
* Allows to push display items to the front-end or to communicate with
* widgets through Jupyter comms (WIP)
*/
implicit def publish: jupyter.api.Publish
}
class JupyterAPIHolder
object JupyterAPIHolder extends APIHolder[FullRuntimeAPI with JupyterApi]
View
@@ -1,16 +1,15 @@
val ammoniumVersion = "0.4.0"
val jupyterKernelVersion = "0.3.0"
val ammoniumVersion = "0.8.1-SNAPSHOT"
val jupyterKernelVersion = "0.4.0-SNAPSHOT"
lazy val `scala-api` = project.in(file("api"))
.settings(commonSettings)
.settings(
libraryDependencies ++= Seq(
"com.github.alexarchambault.ammonium" % "interpreter-api" % ammoniumVersion cross CrossVersion.full,
"com.github.alexarchambault.jupyter" %% "kernel-api" % jupyterKernelVersion,
"org.jupyter-scala" % "ammonite-runtime" % ammoniumVersion cross CrossVersion.full,
"org.jupyter-scala" %% "kernel-api" % jupyterKernelVersion,
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
"com.github.alexarchambault.ammonium" % "tprint" % ammoniumVersion cross CrossVersion.full,
"com.lihaoyi" %% "pprint" % "0.3.9"
"com.lihaoyi" %% "pprint" % "0.4.2"
),
libraryDependencies ++= {
if (scalaVersion.value startsWith "2.10.")
@@ -37,47 +36,54 @@ lazy val `scala-kernel` = project.in(file("kernel"))
.settings(testSettings)
.settings(
libraryDependencies ++= Seq(
"com.github.alexarchambault.jupyter" %% "kernel" % jupyterKernelVersion,
"com.github.alexarchambault.ammonium" % "interpreter" % ammoniumVersion cross CrossVersion.full,
"com.github.alexarchambault.ammonium" % "shell-tests" % ammoniumVersion % "test" cross CrossVersion.full
"org.jupyter-scala" %% "kernel" % jupyterKernelVersion,
"org.jupyter-scala" % "ammonite-compiler" % ammoniumVersion cross CrossVersion.full
),
libraryDependencies ++= Seq(
"com.github.alexarchambault.jupyter" %% "kernel" % jupyterKernelVersion
).map(_ % "test" classifier "tests"),
libraryDependencies ++= {
if (scalaBinaryVersion.value == "2.10")
Seq("org.scalamacros" % "paradise" % "2.0.1" % "plugin->default(compile)" cross CrossVersion.full)
Seq("org.scalamacros" % "paradise" % "2.1.0" % "plugin->default(compile)" cross CrossVersion.full)
else
Seq()
}
)
lazy val `scala-cli` = project.in(file("cli"))
lazy val `scala-cli` = project.in(file("cli"))
.dependsOn(`scala-kernel`)
.settings(commonSettings)
.settings(packAutoSettings)
.settings(
libraryDependencies ++= Seq(
"com.github.alexarchambault" %% "case-app" % "1.0.0-RC2",
"com.github.alexarchambault" %% "case-app" % "1.1.2",
"ch.qos.logback" % "logback-classic" % "1.1.7"
),
libraryDependencies ++= {
if (scalaBinaryVersion.value == "2.10")
Seq(compilerPlugin("org.scalamacros" % "paradise" % "2.0.1" cross CrossVersion.full))
Seq(compilerPlugin("org.scalamacros" % "paradise" % "2.1.0" cross CrossVersion.full))
else
Seq()
}
)
lazy val `jupyter-scala` = project.in(file("."))
lazy val spark = project
.dependsOn(`scala-api`)
.settings(commonSettings)
.settings(
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-sql" % "1.6.2" % "provided",
"org.eclipse.jetty" % "jetty-server" % "8.1.14.v20131031",
"io.get-coursier" %% "coursier-cli" % "1.0.0-M14-7"
)
)
lazy val `jupyter-scala` = project
.in(file("."))
.settings(commonSettings)
.settings(noPublishSettings)
.aggregate(`scala-api`, `scala-kernel`, `scala-cli`)
.dependsOn(`scala-api`, `scala-kernel`, `scala-cli`)
.aggregate(`scala-api`, `scala-kernel`, `scala-cli`, spark)
lazy val commonSettings = Seq(
organization := "com.github.alexarchambault.jupyter",
organization := "org.jupyter-scala",
scalacOptions ++= Seq("-deprecation", "-unchecked", "-feature"),
resolvers ++= Seq(
"Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases",
@@ -93,11 +99,12 @@ lazy val commonSettings = Seq(
javaOptions in Test ++= Seq(
"-Xmx3172M",
"-Xms3172M"
)
),
resolvers += Resolver.jcenterRepo
) ++ publishSettings
lazy val testSettings = Seq(
libraryDependencies += "com.lihaoyi" %% "utest" % "0.3.0" % "test",
libraryDependencies += "com.lihaoyi" %% "utest" % "0.4.4" % "test",
testFrameworks += new TestFramework("utest.runner.Framework")
)
@@ -17,11 +17,12 @@
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%msg%n</pattern>
<pattern>[%thread] %d{HH:mm:ss.SSS} %highlight(%-5level) %cyan(%logger{15}) - %msg %n</pattern>
</encoder>
</appender>
<root level="debug">
<appender-ref ref="FILE" />
<appender-ref ref="STDOUT" />
</root>
</configuration>
Oops, something went wrong.

0 comments on commit 860e489

Please sign in to comment.