Permalink
Browse files

Delete quasar's binary data type

  • Loading branch information...
alissapajer committed Aug 6, 2018
1 parent 3eae37d commit baa71d60e818fb61000eed12468666109b535f85
Showing with 74 additions and 479 deletions.
  1. +0 −1 blueeyes/src/main/scala/quasar/blueeyes/json/JValue.scala
  2. +0 −1 blueeyes/src/main/scala/quasar/precog/common/CValue.scala
  3. +0 −3 common/src/main/scala/quasar/common/PrimaryType.scala
  4. +3 −41 common/src/main/scala/quasar/common/data/Data.scala
  5. +0 −1 common/src/main/scala/quasar/common/data/QDataData.scala
  6. +1 −2 common/src/test/scala/quasar/common/data/DataGenerators.scala
  7. +0 −1 common/src/test/scala/quasar/common/data/QDataDataSpec.scala
  8. +0 −4 datagen/src/main/scala/quasar/datagen/dist.scala
  9. +2 −11 datagen/src/test/scala/quasar/datagen/GenerateSpec.scala
  10. +1 −7 ejson/src/main/scala/quasar/ejson/DecodeEJson.scala
  11. +1 −4 ejson/src/main/scala/quasar/ejson/EJson.scala
  12. +1 −7 ejson/src/main/scala/quasar/ejson/EncodeEJson.scala
  13. +7 −15 ejson/src/main/scala/quasar/ejson/Extension.scala
  14. +1 −4 ejson/src/main/scala/quasar/ejson/Fixed.scala
  15. +1 −12 ejson/src/main/scala/quasar/ejson/JsonCodec.scala
  16. +0 −1 ejson/src/main/scala/quasar/ejson/TypeTag.scala
  17. +0 −80 ejson/src/main/scala/quasar/ejson/Z85.scala
  18. +1 −4 ejson/src/main/scala/quasar/ejson/optics.scala
  19. +0 −1 ejson/src/test/scala/quasar/ejson/EJsonArbitrary.scala
  20. +0 −62 ejson/src/test/scala/quasar/ejson/Z85.scala
  21. +13 −21 frontend/src/main/scala/quasar/Type.scala
  22. +1 −10 frontend/src/main/scala/quasar/frontend/data/DataCodec.scala
  23. +8 −9 frontend/src/main/scala/quasar/tpe/package.scala
  24. +1 −3 frontend/src/test/scala/quasar/TypeGenerators.scala
  25. +1 −19 frontend/src/test/scala/quasar/frontend/data/DataCodecSpec.scala
  26. +4 −5 frontend/src/test/scala/quasar/types.scala
  27. +0 −1 impl/src/main/scala/quasar/impl/schema/package.scala
  28. +0 −34 impl/src/test/scala/quasar/impl/schema/ExtractSstSpec.scala
  29. +0 −2 qscript/src/main/scala/quasar/qscript/RenderQScriptDSL.scala
  30. +0 −2 qscript/src/test/scala/quasar/qscript/RenderQScriptDSLSpec.scala
  31. +0 −3 qsu/src/main/scala/quasar/qsu/QScriptUniform.scala
  32. +5 −6 repl/src/test/scala/quasar/repl/PrettifySpec.scala
  33. +1 −18 sst/src/main/scala/quasar/sst/TypeStat.scala
  34. +0 −20 sst/src/main/scala/quasar/sst/compression.scala
  35. +1 −19 sst/src/main/scala/quasar/sst/package.scala
  36. +18 −38 sst/src/test/scala/quasar/sst/CompressionSpec.scala
  37. +1 −3 sst/src/test/scala/quasar/sst/TypeStatArbitrary.scala
  38. +1 −1 yggdrasil/src/main/scala/quasar/yggdrasil/bytecode/JType.scala
  39. +0 −3 yggdrasil/src/test/scala/quasar/yggdrasil/table/ColumnarTableModuleSpec.scala
@@ -94,7 +94,6 @@ object JValue {
case Data.LocalDate(value) => JString(value.toString)
case Data.LocalTime(value) => JString(value.toString)
case Data.Interval(value) => JString(value.toString)
case Data.Binary(values) => JArray(values.map(JNumLong(_)): _*)
case Data.NA => JUndefined
}
@@ -302,7 +302,6 @@ object RValue extends RValueInstances {
case Data.LocalDate(v) => CLocalDate(v).some
case Data.LocalTime(v) => CLocalTime(v).some
case Data.Interval(k) => CInterval(k).some
case Data.Binary(k) => CArray[Long](k.map(l => l.toLong).toArray).some
case Data.NA => None
}
@@ -27,7 +27,6 @@ sealed abstract class PrimaryType
// NB: These are defined in order of their total ordering
final case object Null extends PrimaryType
final case object Bool extends PrimaryType
final case object Byte extends PrimaryType
final case object Char extends PrimaryType
final case object Int extends PrimaryType
final case object Dec extends PrimaryType
@@ -38,7 +37,6 @@ object PrimaryType {
def name = Prism[String, PrimaryType] {
case "null" => Null.some
case "boolean" => Bool.some
case "byte" => Byte.some
case "character" => Char.some
case "integer" => Int.some
case "decimal" => Dec.some
@@ -48,7 +46,6 @@ object PrimaryType {
} {
case Null => "null"
case Bool => "boolean"
case Byte => "byte"
case Char => "character"
case Int => "integer"
case Dec => "decimal"
@@ -46,7 +46,6 @@ import matryoshka._
import matryoshka.patterns._
import monocle.{Iso, Optional, Prism}
import scalaz.{Lens => _, Optional=>_, _}, Scalaz._
import scodec.bits.ByteVector
sealed abstract class Data extends Product with Serializable
@@ -143,28 +142,6 @@ object Data {
Prism.partial[Data, DateTimeInterval]
{ case Data.Interval(dti) => dti }(Data.Interval)
final case class Binary(value: ImmutableArray[Byte]) extends Data {
def base64: String = new sun.misc.BASE64Encoder().encode(value.toArray)
override def toString = "Binary(Array[Byte](" + value.mkString(", ") + "))"
/**
* scala equality needs to remain for Spark to work
* @see Planner.qscriptCore
*/
override def equals(that: Any): Boolean = that match {
case Binary(value2) => value ≟ value2
case _ => false
}
override def hashCode = java.util.Arrays.hashCode(value.toArray[Byte])
}
object Binary {
def fromArray(array: Array[Byte]): Binary = Binary(ImmutableArray.fromArray(array))
}
val _binary =
Prism.partial[Data, ImmutableArray[Byte]] { case Data.Binary(bs) => bs } (Data.Binary(_))
/**
An object to represent any value that might come from a backend, but that
we either don't know about or can't represent in this ADT. We represent it
@@ -271,13 +248,6 @@ object Data {
val fromExtension: Algebra[Extension, Data] = {
case ejson.Meta(value, meta) => (meta, value) match {
case (EJsonTypeSize(TypeTag.Binary, size), Str(data)) =>
if (size.isValidInt)
ejson.z85.decode(data).fold[Data](
NA)(
bv => Binary(ImmutableArray.fromArray(bv.take(size.toLong).toArray)))
else NA
case (EJsonType(TypeTag.OffsetDateTime), Obj(map)) =>
(extract(map.get(DateTimeConstants.year), _int)(_.toInt) ⊛
extract(map.get(DateTimeConstants.month), _int)(_.toInt) ⊛
@@ -353,18 +323,15 @@ object Data {
case (_, _) => value
}
case ejson.Map(value) =>
case ejson.Map(value) =>
value.traverse(Bitraverse[(?, ?)].leftTraverse.traverse(_) {
case Str(key) => key.some
case _ => None
}).fold[Data](NA)(pairs => Obj(ListMap(pairs: _*)))
case ejson.Int(value) => Int(value)
// FIXME: cheating, but it’s what we’re already doing in the SQL parser
case ejson.Byte(value) => Binary.fromArray(Array[Byte](value))
case ejson.Int(value) => Int(value)
case ejson.Char(value) => Str(value.toString)
case ejson.Char(value) => Str(value.toString)
}
// TODO: Data should be replaced with EJson. These just exist to bridge the
@@ -454,11 +421,6 @@ object Data {
DateTimeConstants.nanosecond -> Int(value.duration.getNano))),
EJsonType(TypeTag.Interval))).right
case Binary(value) =>
E.inj(ejson.Meta(
Str(ejson.z85.encode(ByteVector.view(value.toArray))),
EJsonTypeSize(TypeTag.Binary, value.size))).right
case data => data.left
})
}
@@ -49,7 +49,6 @@ object QDataData extends QData[Data] {
case Data.LocalDate(_) => QLocalDate
case Data.LocalTime(_) => QLocalTime
case Data.Interval(_) => QInterval
case Data.Binary(_) => error(s"Unable to represent `Data.Binary`.")
case Data.NA => error(s"Unable to represent `Data.NA`.")
case Data.Obj(_) => QObject
case Data.Arr(_) => QArray
@@ -107,8 +107,7 @@ object DataGenerators extends DataGenerators {
TimeGenerators.genOffsetTime ^^ Data.OffsetTime,
TimeGenerators.genLocalDateTime ^^ Data.LocalDateTime,
TimeGenerators.genLocalDate ^^ Data.LocalDate,
TimeGenerators.genLocalTime ^^ Data.LocalTime,
arrayOf(genByte) ^^ Data.Binary.fromArray)
TimeGenerators.genLocalTime ^^ Data.LocalTime)
def genKey = Gen.alphaChar ^^ (_.toString)
@@ -29,7 +29,6 @@ object QDataDataSpec extends Qspec with DataGenerators {
// we don't need to adjust for this case because Equal[Data] compares them as equal
def adjustExpected(data: Data): Option[Data] = data match {
case Data.NA => None // not supported by qdata
case Data.Binary(_) => None // not supported by qdata
case d => Some(d)
}
@@ -202,7 +202,6 @@ object dist {
private def leafDist[J](implicit J: Corecursive.Aux[J, EJson]): Dist[J] =
Dist.mix(
Dist[Boolean] map (EJson.bool(_)),
Dist[Byte] map (EJson.byte(_)),
Dist[Char] map (EJson.char(_)),
Dist.bigdecimal(BigIntMaxBytes, BigDecimalMaxScale) map (EJson.dec(_)),
Dist.bigint(BigIntMaxBytes) map (EJson.int(_)),
@@ -244,9 +243,6 @@ object dist {
((fs / total).toDouble, Dist.constant(false))
) map (EJson.bool(_))
case TypeStat.Byte(_, bn, bx) =>
Dist.intrange(bn.toInt, bx.toInt) map (i => EJson.byte(i.toByte))
case TypeStat.Char(ss, cn, cx) =>
gaussian(ss)
.cata((Dist.gaussian[A] _).tupled, Dist.constant(ss.mean))
@@ -77,11 +77,11 @@ final class GenerateSpec extends quasar.Qspec {
"weighted selection from a union" >> {
val x = sstL(TypeStat.char(SampleStats.freq(0.0, 'a'.toDouble), 'a', 'a'), SimpleType.Char)
val y = sstL(TypeStat.byte(100.0, 7.toByte, 7.toByte), SimpleType.Char)
val y = sstL(TypeStat.int(SampleStats.freq(100.0, BigInt(7).toDouble), BigInt(7), BigInt(7)), SimpleType.Int)
val z = sstL(TypeStat.bool(0.0, 0.0), SimpleType.Bool)
val u = sst(TypeStat.count(100.0), TypeF.union(x, y, IList(z)))
valuesShould(u)(_ ≟ J.byte(7.toByte))
valuesShould(u)(_ ≟ J.int(BigInt(7)))
}
"array" >> {
@@ -218,15 +218,6 @@ final class GenerateSpec extends quasar.Qspec {
valuesShould(t ⊹ f)(J.bool.nonEmpty)
}
"byte within range" >> {
val bsst =
NonEmptyList(3, 4, 5, 6, 7, 8, 9, 10, 11) foldMap1 { i =>
sstL(tsJ1(J.byte(i.toByte)), SimpleType.Byte)
}
valuesShould(bsst)(J.byte.exist(b => (b >= 3.toByte) && (b <= 17.toByte)))
}
"char within range" >> {
val csst =
NonEmptyList('e', 'f', 'g', 'h', 'i', 'j', 'k', 'l') foldMap1 { c =>
@@ -16,7 +16,7 @@
package quasar.ejson
import slamdata.Predef.{Int => SInt, Char => SChar, Byte => SByte, _}
import slamdata.Predef.{Int => SInt, Char => SChar, _}
import quasar.RenderedTree
import quasar.contrib.argonaut._
import quasar.ejson.implicits._
@@ -123,12 +123,6 @@ sealed abstract class DecodeEJsonInstances extends DecodeEJsonInstances0 {
implicit val shortDecodeEJson: DecodeEJson[Short] =
bigIntDecodeEJson.reinterpret("Short", bi => bi.isValidShort option bi.shortValue)
implicit val byteDecodeEJson: DecodeEJson[SByte] =
new DecodeEJson[SByte] {
def decode[J](j: J)(implicit JC: Corecursive.Aux[J, EJson], JR: Recursive.Aux[J, EJson]): Decoded[SByte] =
Decoded.attempt(j, Fixed[J].byte.getOption(j) \/> "Byte")
}
implicit val charDecodeEJson: DecodeEJson[SChar] =
new DecodeEJson[SChar] {
def decode[J](j: J)(implicit JC: Corecursive.Aux[J, EJson], JR: Recursive.Aux[J, EJson]): Decoded[SChar] = {
@@ -16,7 +16,7 @@
package quasar.ejson
import slamdata.Predef.{Byte => SByte, Char => SChar, Int => _, Map => _, _}
import slamdata.Predef.{Char => SChar, Int => _, Map => _, _}
import quasar.contrib.matryoshka.{project, totally}
import quasar.contrib.iota.copkTraverse
@@ -63,9 +63,6 @@ object EJson {
def bool[T](b: Boolean)(implicit T: Corecursive.Aux[T, EJson]): T =
fromCommon(Bool(b))
def byte[T](b: SByte)(implicit T: Corecursive.Aux[T, EJson]): T =
fromExt(Byte(b))
def char[T](c: SChar)(implicit T: Corecursive.Aux[T, EJson]): T =
fromExt(Char(c))
@@ -16,7 +16,7 @@
package quasar.ejson
import slamdata.Predef.{Int => SInt, Char => SChar, Byte => SByte, _}
import slamdata.Predef.{Int => SInt, Char => SChar, _}
import quasar.contrib.argonaut._
import quasar.contrib.iota.copkTraverse
@@ -66,12 +66,6 @@ sealed abstract class EncodeEJsonInstances extends EncodeEJsonInstances0 {
implicit val shortEncodeEJson: EncodeEJson[Short] =
intEncodeEJson.contramap(_.toInt)
implicit val byteEncodeEJson: EncodeEJson[SByte] =
new EncodeEJson[SByte] {
def encode[J](b: SByte)(implicit JC: Corecursive.Aux[J, EJson], JR: Recursive.Aux[J, EJson]): J =
Fixed[J].byte(b)
}
implicit val charEncodeEJson: EncodeEJson[SChar] =
new EncodeEJson[SChar] {
def encode[J](c: SChar)(implicit JC: Corecursive.Aux[J, EJson], JR: Recursive.Aux[J, EJson]): J =
@@ -16,7 +16,7 @@
package quasar.ejson
import slamdata.Predef.{Byte => SByte, Char => SChar, _}
import slamdata.Predef.{Char => SChar, _}
import quasar.{RenderTree, NonTerminal, Terminal}, RenderTree.ops._
import quasar.contrib.matryoshka._
import quasar.fp._
@@ -26,14 +26,13 @@ import monocle.{Iso, Prism}
import scalaz.{Applicative, Cord, Equal, IMap, Order, Scalaz, Show, Traverse}, Scalaz._
/** This is an extension to JSON that allows arbitrary expressions as map (née
* object) keys and adds additional primitive types, including characters,
* bytes, and distinct integers. It also adds metadata, which allows arbitrary
* object) keys and adds additional primitive types, including characters
* and distinct integers. It also adds metadata, which allows arbitrary
* annotations on values.
*/
sealed abstract class Extension[A]
final case class Meta[A](value: A, meta: A) extends Extension[A]
final case class Map[A](value: List[(A, A)]) extends Extension[A]
final case class Byte[A](value: SByte) extends Extension[A]
final case class Char[A](value: SChar) extends Extension[A]
final case class Int[A](value: BigInt) extends Extension[A]
@@ -42,9 +41,6 @@ object Extension extends ExtensionInstances {
obj => Optics.map(obj.value.toList.map(_.leftMap(f)))
object Optics {
def byte[A] =
Prism.partial[Extension[A], scala.Byte] { case Byte(b) => b } (Byte(_))
def char[A] =
Prism.partial[Extension[A], scala.Char] { case Char(c) => c } (Char(_))
@@ -83,7 +79,7 @@ sealed abstract class ExtensionInstances {
implicit val ordC: Order[SChar] = scalaz.std.anyVal.char
Order.orderBy { e =>
val g = generic(e)
g.copy(_4 = g._4 map (IMap.fromFoldable(_)))
g.copy(_3 = g._3 map (IMap.fromFoldable(_)))
}
}
}
@@ -99,7 +95,7 @@ sealed abstract class ExtensionInstances {
implicit val eqlC: Equal[SChar] = scalaz.std.anyVal.char
Equal.equalBy { e =>
val g = generic(e)
g.copy(_4 = g._4 map (EqMap.fromFoldable(_)))
g.copy(_3 = g._3 map (EqMap.fromFoldable(_)))
}
}
}
@@ -113,7 +109,6 @@ sealed abstract class ExtensionInstances {
fa match {
case Meta(value, meta) => (f(value) ⊛ f(meta))(Meta(_, _))
case Map(value) => value.traverse(_.bitraverse(f, f)).map(Map(_))
case Byte(value) => G.point(Byte(value))
case Char(value) => G.point(Char(value))
case Int(value) => G.point(Int(value))
}
@@ -124,7 +119,6 @@ sealed abstract class ExtensionInstances {
def apply[α](eq: Show[α]) = Show.show(a => a match {
case Meta(v, m) => Cord(s"Meta($v, $m)")
case Map(v) => Cord(s"Map($v)")
case Byte(v) => Cord(s"Byte($v)")
case Char(v) => Cord(s"Char($v)")
case Int(v) => Cord(s"Int($v)")
})
@@ -144,7 +138,6 @@ sealed abstract class ExtensionInstances {
case Map(v) =>
NonTerminal("Map" :: c, none, v map (_.render))
case Byte(b) => t("Byte", b)
case Char(v) => t("Char", v)
case Int(v) => t("Int", v)
}
@@ -163,10 +156,9 @@ sealed abstract class ExtensionInstances {
private def generic[A](e: Extension[A]) =
(
byte.getOption(e),
char.getOption(e),
int.getOption(e) ,
map.getOption(e) ,
int.getOption(e),
map.getOption(e),
meta.getOption(e)
)
}
@@ -16,7 +16,7 @@
package quasar.ejson
import slamdata.Predef.{Byte => SByte, Char => SChar, Int => _, Map => _, _}
import slamdata.Predef.{Char => SChar, Int => _, Map => _, _}
import quasar.contrib.matryoshka.birecursiveIso
import quasar.ejson.implicits._
import quasar.contrib.iota.copkTraverse
@@ -34,9 +34,6 @@ final class Fixed[J] private ()(implicit JC: Corecursive.Aux[J, EJson], JR: Recu
val bool: Prism[J, Boolean] =
iso composePrism optics.bool
val byte: Prism[J, SByte] =
iso composePrism optics.byte
val char: Prism[J, SChar] =
iso composePrism optics.char
Oops, something went wrong.

0 comments on commit baa71d6

Please sign in to comment.