source-from-url

.scalafix.conf

@@ -13,14 +13,14 @@ rules = [
 ]
 
 DisableSyntax.noVars = true
-DisableSyntax.noThrows = true
+DisableSyntax.noThrows = false
 DisableSyntax.noNulls = true
 DisableSyntax.noReturns = true
 DisableSyntax.noAsInstanceOf = true
 DisableSyntax.noIsInstanceOf = true
 DisableSyntax.noXml = true
 DisableSyntax.noDefaultArgs = true
-DisableSyntax.noFinalVal = true
+DisableSyntax.noFinalVal = false
 DisableSyntax.noFinalize = true
 DisableSyntax.noValPatterns = true
 DisableSyntax.noUniversalEquality = true

.scalafmt.conf

@@ -1,2 +1,2 @@
 style=IntelliJ
-maxColumn=80
+maxColumn=160

build.sbt

@@ -22,6 +22,7 @@ scapegoatVersion in ThisBuild := "1.3.8"
 scalacOptions ++= scalacOptionsFor(scalaVersion.value)
 val ScalaPropsVersion = "0.5.5"
 val SloggingVersion = "0.6.1"
+val ScalazVersion = "7.2.28"
 
 val SharedSettings = Seq(
   scalaVersion := "2.11.12",
@@ -36,7 +37,10 @@ val SharedSettings = Seq(
     "com.github.scalaprops" %%% "scalaprops" % ScalaPropsVersion % "test,it",
     "com.github.scalaprops" %%% "scalaprops-scalazlaws" % ScalaPropsVersion % "test,it",
   ),
-  libraryDependencies += "org.scalaz" %%% "scalaz-core" % "7.2.27",
+  libraryDependencies += "org.scalaz" %%% "scalaz-core" % ScalazVersion,
+  libraryDependencies += "org.scalaz" %% "scalaz-concurrent" % ScalazVersion,
+  libraryDependencies += "org.scalaz" %%% "scalaz-effect" % ScalazVersion,
+  libraryDependencies += "org.scalaz" %% "scalaz-ioeffect" % "2.10.1",
   libraryDependencies ++= Seq(
     "biz.enef" %%% "slogging" % SloggingVersion,
   ),
@@ -48,6 +52,7 @@ val SharedSettings = Seq(
   addCompilerPlugin(scalafixSemanticdb),
   wartremoverErrors ++= Warts.unsafe,
   scapegoatVersion := "1.3.8",
+  trapExit := false,
 )
 
 val jsSettings = Seq(

project/plugins.sbt

@@ -4,7 +4,7 @@ addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.4")
 addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "0.5.0")
 addSbtPlugin("org.portable-scala" % "sbt-scala-native-crossproject" % "0.5.0")
 addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.24")
-addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.8")
+addSbtPlugin("org.scala-native" % "sbt-scala-native" % "0.3.9")
 // coverage
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")
 addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.2.7")

re/jvm/src/main/scala/pl/writeonly/re/main/Main.scala

@@ -1,11 +1,14 @@
 package pl.writeonly.re.main
 
 import pl.writeonly.re.shared.core.Core
+import pl.writeonly.re.shared.hyde.impl.Hyde
 import slogging._
 
 object Main extends App {
   LoggerConfig.factory = SLF4JLoggerFactory()
 
   LoggerConfig.level = LogLevel.TRACE
   Core.apply("JVM")
+
+  Hyde.main(args)
 }

re/native/src/main/scala/pl/writeonly/re/main/Main.scala

@@ -1,6 +1,7 @@
 package pl.writeonly.re.main
 
 import pl.writeonly.re.shared.core._
+import pl.writeonly.re.shared.hyde.impl.Hyde
 import slogging._
 import slogging.TerminalLoggerFactory.TerminalControlCode
 
@@ -14,4 +15,6 @@ object Main extends App {
   Core.apply("Native")
 
   StrictLoggingCore.rainbow()
+
+  Hyde.main(args)
 }

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/State.scala

@@ -0,0 +1,9 @@
+package pl.writeonly.re.shared.hyde.common.states
+
+trait State {
+  def showResult(): Unit
+
+  def showStep(): Unit
+
+  def isEmptyNextInternalUrls: Boolean
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/api/AbstractAPIState.scala

@@ -0,0 +1,34 @@
+package pl.writeonly.re.shared.hyde.common.states.api
+
+import pl.writeonly.re.shared.hyde.common.states.State
+import pl.writeonly.re.shared.hyde.common.url.typed.InternalUrls
+import pl.writeonly.re.shared.hyde.common.url.urls._
+import scalaz.Scalaz._
+
+abstract class AbstractAPIState(data: UrlsWithThrowableList) extends State {
+
+  type HP
+
+  final type SET = Set[HP]
+
+  type NextState
+
+  override def showStep(): Unit = data.showStep()
+
+  def showResult(): Unit = data.showResult()
+
+  def isEmptyNextInternalUrls: Boolean = data.isEmptyNextInternalUrls
+
+  override def toString: String = data.toString
+
+  final def newState(set: SET): NextState = set |> nextData |> nextState
+
+  def nextData(set: SET): UrlsWithThrowableList
+
+  def nextState(data: UrlsWithThrowableList): NextState
+
+  protected def nextUrls: InternalUrls = data.nextUrls
+
+  def throwableList: ThrowableList = data.throwableList
+
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/api/DisjunctionAPIState.scala

@@ -0,0 +1,30 @@
+package pl.writeonly.re.shared.hyde.common.states.api
+
+import pl.writeonly.re.shared.hyde.common.url._
+import pl.writeonly.re.shared.hyde.common.url.urls._
+import pl.writeonly.re.shared.hyde.sourcepage.thirdparty._
+
+abstract class DisjunctionAPIState(data: UrlsWithThrowableList)(implicit d: Domain) extends AbstractAPIState(data) {
+  override type HP = SourcePageDisjunction
+
+  def nextData(set: SourcePageDisjunctionSet): UrlsWithThrowableList = {
+
+    val partitioned = set.partition(_.isRight)
+
+    val newWrappedUrls: WrappedUrlSet = partitioned._1
+      .flatMap(DisjunctionAPIState.sourcePageDisjunctionToWrappedUrlSet)
+
+    val newThrowableList: ThrowableList = partitioned._2.toList
+      .flatMap(_.swap.toOption.toList)
+
+    val newUrls = NewUrls(newWrappedUrls)
+
+    data.next(newUrls, newThrowableList)
+  }
+
+}
+
+object DisjunctionAPIState {
+  val sourcePageDisjunctionToWrappedUrlSet: SourcePageDisjunction => WrappedUrlSet =
+    _.map(_.getWrappedUrlSet).toOption.toSet.flatten
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/api/EitherAPIState.scala

@@ -0,0 +1,29 @@
+package pl.writeonly.re.shared.hyde.common.states.api
+
+import pl.writeonly.re.shared.hyde.common.url._
+import pl.writeonly.re.shared.hyde.common.url.urls._
+import pl.writeonly.re.shared.hyde.sourcepage.std._
+
+abstract class EitherAPIState(data: UrlsWithThrowableList)(implicit d: Domain) extends AbstractAPIState(data) {
+  override type HP = SourcePageEither
+
+  def nextData(set: SourcePageEitherSet): UrlsWithThrowableList = {
+
+    val partitioned = set.partition(_.isRight)
+
+    val newWrappedUrls: WrappedUrlSet = partitioned._1
+      .flatMap(EitherAPIState.sourcePageEitherToWrappedUrlSet)
+
+    val newThrowableList: ThrowableList = partitioned._2.toList
+      .flatMap(_.left.toOption.toList)
+
+    val newUrls = NewUrls(newWrappedUrls)
+
+    data.next(newUrls, newThrowableList)
+  }
+}
+
+object EitherAPIState {
+  val sourcePageEitherToWrappedUrlSet: SourcePageEither => WrappedUrlSet =
+    _.right.map(_.getWrappedUrlSet).right.toOption.toSet.flatten
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/api/package.scala

@@ -0,0 +1,19 @@
+package pl.writeonly.re.shared.hyde.common.states
+
+import scala.concurrent.Future
+
+import scalaz.concurrent.Task
+import scalaz.effect.IO
+import scalaz.ioeffect.{IO => IOIO, Task => IOTask}
+
+package object api {
+  type ParallelStateFuture = Future[EitherAPIState]
+
+  type ParallelStateIO = IO[DisjunctionAPIState]
+
+  type ParallelStateTask = Task[DisjunctionAPIState]
+
+  type ParallelStateIOIO = IOIO[Throwable, DisjunctionAPIState]
+
+  type ParallelStateIOTask = IOTask[DisjunctionAPIState]
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/notexception/AbstractFunctionState.scala

@@ -0,0 +1,12 @@
+package pl.writeonly.re.shared.hyde.common.states.notexception
+
+import pl.writeonly.re.shared.hyde.common.url.typed.InternalUrl
+
+trait AbstractFunctionState extends AbstractNewSetState {
+
+  final def newSet: SET = nextUrls.map(impureFunction)
+
+  def impureFunction: HPFromInternalUrl
+
+  type HPFromInternalUrl = InternalUrl => HP
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/notexception/AbstractNewSetState.scala

@@ -0,0 +1,8 @@
+package pl.writeonly.re.shared.hyde.common.states.notexception
+
+trait AbstractNewSetState extends AbstractNextState {
+
+  final def next: AbstractNextState = newState(newSet)
+
+  def newSet: SET
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/notexception/AbstractNextState.scala

@@ -0,0 +1,20 @@
+package pl.writeonly.re.shared.hyde.common.states.notexception
+
+import scala.annotation.tailrec
+
+import pl.writeonly.re.shared.hyde.common.states.api.AbstractAPIState
+import pl.writeonly.re.shared.hyde.common.url.urls.UrlsWithThrowableList
+
+trait AbstractNextState extends AbstractAPIState {
+
+  type NextState = AbstractNextState
+
+  def next: NextState
+
+  def nextState(data: UrlsWithThrowableList): AbstractNextState
+}
+
+object AbstractNextState {
+  @tailrec
+  def run(state: AbstractNextState): AbstractNextState = if (state.isEmptyNextInternalUrls) state else run(state.next)
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/notexception/AbstractNextStateObject.scala

@@ -0,0 +1,11 @@
+package pl.writeonly.re.shared.hyde.common.states.notexception
+
+import pl.writeonly.re.shared.hyde.common.url.Domain
+import scalaz.Scalaz._
+
+trait AbstractNextStateObject {
+
+  def fromDomain(implicit d: Domain): AbstractNextState
+
+  def apply(domain: String): AbstractNextState = fromDomain(new Domain(domain)) |> AbstractNextState.run
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/oo/AbstractOOState.scala

@@ -0,0 +1,36 @@
+package pl.writeonly.re.shared.hyde.common.states.oo
+
+import scala.annotation.tailrec
+
+import pl.writeonly.re.shared.hyde.common.states.State
+import pl.writeonly.re.shared.hyde.common.url._
+import pl.writeonly.re.shared.hyde.common.url.typed.InternalUrl
+import pl.writeonly.re.shared.hyde.common.url.urls._
+import pl.writeonly.re.shared.hyde.sourcepage.SourcePage
+import scalaz.Scalaz._
+
+abstract class AbstractOOState(urls: Urls)(implicit d: Domain) extends State {
+
+  override final def showResult(): Unit = urls.showResult()
+
+  override final def showStep(): Unit = urls.showStep()
+
+  override final def isEmptyNextInternalUrls: Boolean = urls.isEmptyNextInternalUrls
+
+  final def next: AbstractOOState = getWrappedUrlSet |> NewUrls.apply |> urls.next |> nextState
+
+  final def getWrappedUrlSet: WrappedUrlSet =
+    urls.nextUrls
+      .map(map)
+      .flatMap(_.getWrappedUrlSet)
+
+  def nextState(urls: Urls): AbstractOOState
+
+  def map: InternalUrl => SourcePage
+
+}
+
+object AbstractOOState {
+  @tailrec
+  def run(state: AbstractOOState): AbstractOOState = if (state.isEmptyNextInternalUrls) state else run(state.next)
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/states/oo/AbstractOOStateObject.scala

@@ -0,0 +1,11 @@
+package pl.writeonly.re.shared.hyde.common.states.oo
+
+import pl.writeonly.re.shared.hyde.common.url.Domain
+import scalaz.Scalaz._
+
+trait AbstractOOStateObject {
+
+  def fromDomain(implicit d: Domain): AbstractOOState
+
+  def apply(domain: String): AbstractOOState = fromDomain(new Domain(domain)) |> AbstractOOState.run
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/url/Domain.scala

@@ -0,0 +1,20 @@
+package pl.writeonly.re.shared.hyde.common.url
+
+import pl.writeonly.re.shared.hyde.common.url.typed.InternalUrl
+
+class Domain(domain: WrappedUrl) {
+
+  def this(url: String) = this(new WrappedUrl(url))
+
+  println("domain " + toString)
+
+  override def toString: String = domain.toString
+
+  def toInternalUrl: InternalUrl = InternalUrl(domain)(this)
+
+  def toInternalUrl(url: WrappedUrl): InternalUrl = new InternalUrl(domain.append(url))
+
+  def isInternalUrl(url: WrappedUrl): Boolean = url.isInternalUrl(domain)
+
+  def toWrappedUrl: WrappedUrl = domain
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/url/WrappedUrl.scala

@@ -0,0 +1,49 @@
+package pl.writeonly.re.shared.hyde.common.url
+
+import java.net._
+
+import pl.writeonly.re.shared.hyde.common.url.exception._
+import pl.writeonly.re.shared.hyde.common.url.typed._
+
+case class WrappedUrl(private val url: String) extends Ordered[WrappedUrl] {
+
+  override def toString: String = url
+
+  override def compare(that: WrappedUrl): Int = url.compareTo(that.url)
+
+  def append(that: WrappedUrl): WrappedUrl = new WrappedUrl(url + that.url)
+
+  def toURL: URL = new URL(url)
+
+  def toURI: URI = URI.create(url)
+
+  def toException(e: Throwable): UrlException = new UrlException(url, e)
+
+  def toNonInternalException: NonInternalUrlException = new NonInternalUrlException(url)
+
+  def toNonRelativeException: NonRelativeUrlException = new NonRelativeUrlException(url)
+
+  def isRelativeUrl: Boolean = url.startsWith("/")
+
+  def isInternalUrl(implicit d: Domain): Boolean = d.isInternalUrl(this)
+
+  def isInternalUrl(d: WrappedUrl): Boolean = url.startsWith(d.url)
+
+  def toInternalUrl(implicit d: Domain): InternalUrl = InternalUrl(this)
+
+  def makeInternalUrl(implicit d: Domain): InternalUrl = d.toInternalUrl(this)
+
+  def toExternalUrl: ExternalUrl = new ExternalUrl(this)
+
+  def getType(implicit d: Domain): Symbol = url match {
+    case _ if isInternalUrl => WrappedUrl.Internal
+    case _ if isRelativeUrl => WrappedUrl.Relative
+    case _                  => WrappedUrl.External
+  }
+}
+
+object WrappedUrl {
+  val Internal = 'internal
+  val Relative = 'relative
+  val External = 'external
+}

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/url/exception/NonInternalUrlException.scala

@@ -0,0 +1,3 @@
+package pl.writeonly.re.shared.hyde.common.url.exception
+
+class NonInternalUrlException(url: String) extends IllegalArgumentException(url)

re/shared/src/main/scala/pl/writeonly/re/shared/hyde/common/url/exception/NonRelativeUrlException.scala

@@ -0,0 +1,3 @@
+package pl.writeonly.re.shared.hyde.common.url.exception
+
+class NonRelativeUrlException(url: String) extends IllegalArgumentException(url)