diff --git a/src/main/scala/xerial/larray/LArray.scala b/src/main/scala/xerial/larray/LArray.scala index d7cbb80..147cab0 100644 --- a/src/main/scala/xerial/larray/LArray.scala +++ b/src/main/scala/xerial/larray/LArray.scala @@ -7,12 +7,9 @@ package xerial.larray -import scala.reflect.runtime.{universe => ru} import scala.reflect.ClassTag -import ru._ import xerial.core.log.Logger -import collection.GenIterable -import collection.mutable.ArrayBuilder + /** * Large Array (LArray) interface. The differences from Array[A] includes: @@ -21,7 +18,7 @@ import collection.mutable.ArrayBuilder * - The memory of LArray[A] resides outside of the normal garbage-collected JVM heap. So the user must release the memory via [[xerial.larray.LArray#free]]. * - LArray elements are not initialized, so explicit initialization is needed * - - * @tparam A + * @tparam A element type */ trait LArray[A] extends LIterable[A] { @@ -55,8 +52,7 @@ trait LArray[A] extends LIterable[A] { /** * Release the memory of LArray. After calling this method, the results of calling the other methods becomes undefined or might cause JVM crash. */ - def free: Unit - + def free /** * Byte size of an element. For example, if A is Int, its elementByteSize is 4 @@ -93,43 +89,43 @@ object LArray { def free { /* do nothing */ } - def write(srcOffset: Long, dest: Array[Byte], destOffset: Int, length: Int): Int = 0 - - /** - * Read the contents from a given source buffer - * @param src - * @param srcOffset - * @param destOffset - * @param length - */ - def read(src: Array[Byte], srcOffset: Int, destOffset: Long, length: Int) = 0 - } def empty = EmptyArray def apply() = EmptyArray + + import java.{lang=>jl} + + private[larray] def wrap[A:ClassTag](size:Long, m:Memory) : LArray[A] = { + val tag = implicitly[ClassTag[A]] + tag.runtimeClass match { + case jl.Integer.TYPE => new LIntArray(size / 4, m).asInstanceOf[LArray[A]] + case jl.Byte.TYPE => new LByteArray(size, m).asInstanceOf[LArray[A]] + case jl.Long.TYPE => new LLongArray(size / 8, m).asInstanceOf[LArray[A]] + // TODO Short, Char, Float, Double + case _ => sys.error(s"unsupported type: $tag") + } + } + + /** * Creates an LArray with given elements. * * @param xs the elements to put in the array * @return an array containing all elements from xs. */ - def apply[T: TypeTag : ClassTag](xs: T*): LArray[T] = { + def apply[A : ClassTag](xs: A*): LArray[A] = { val size = xs.size - val t = typeOf[T] - val arr : LArray[T] = t match { - case t if t =:= typeOf[Int] => new LIntArray(size).asInstanceOf[LArray[T]] - case t if t =:= typeOf[Byte] => new LByteArray(size).asInstanceOf[LArray[T]] - case _ => new LObjectArray32[T](size).asInstanceOf[LArray[T]] - } + val arr = new LObjectArray32[A](size) var i = 0 for(x <- xs) { arr(i) = x; i += 1 } arr } + def apply(first: Int, elems: Int*): LArray[Int] = { // elems: Int* => Seq[Int] val size = 1 + elems.size @@ -152,6 +148,14 @@ object LArray { arr } + // TODO apply(Char..) + // TODO apply(Short..) + // TODO apply(Float ..) + // TODO apply(Long ..) + // TODO apply(Double ..) + // TODO apply(AnyRef ..) + + def copy[A](src:LArray[A], srcPos:Long, dest:LArray[A], destPos:Long, length:Long) { import UnsafeUtil.unsafe val copyLen = math.min(length, math.min(src.size - srcPos, dest.size - destPos)) @@ -203,19 +207,26 @@ trait RawByteArray[A] extends LArray[A] { /** * Read the contents from a given source buffer - * @param src - * @param srcOffset - * @param destOffset - * @param length + * @param src source buffer + * @param srcOffset byte offset in the source buffer + * @param destOffset byte offset from the destination address + * @param length byte length to read from the source */ def read(src:Array[Byte], srcOffset:Int, destOffset:Long, length:Int) : Int + + + /** + * Create an input stream for reading LArray byte contents + * @return + */ + def toInputStream : java.io.InputStream = LArrayInputStream(this) } /** * Wrapping Array[Int] to support Long-type indexes - * @param size + * @param size array size */ class LIntArraySimple(val size: Long) extends LArray[Int] { @@ -253,7 +264,7 @@ class LIntArraySimple(val size: Long) extends LArray[Int] { /** * Emulate large arrays using two-diemensional matrix of Int. Array[Int](page index)(offset in page) - * @param size + * @param size array size */ class MatrixBasedLIntArray(val size:Long) extends LArray[Int] { @@ -312,13 +323,13 @@ private[larray] trait UnsafeArray[T] extends RawByteArray[T] with Logger { self: val writeLen = math.min(dest.length - destOffset, math.min(length, byteLength - srcOffset)).toInt trace("copy to array") LArray.impl.asInstanceOf[xerial.larray.impl.LArrayNativeAPI].copyToArray(m.address + srcOffset, dest, destOffset, writeLen) - writeLen.toInt + writeLen } def read(src:Array[Byte], srcOffset:Int, destOffset:Long, length:Int) : Int = { val readLen = math.min(src.length-srcOffset, math.min(byteLength - destOffset, length)).toInt LArray.impl.asInstanceOf[xerial.larray.impl.LArrayNativeAPI].copyFromArray(src, srcOffset, m.address + destOffset, readLen) - readLen.toInt + readLen } def readByte(index:Long) = m.getByte(index) @@ -471,8 +482,8 @@ object LObjectArray { /** * LArray[A] of Objects. This implementation is a simple wrapper of Array[A] and used when the array size is less than 2G - * @param size - * @tparam A + * @param size array size + * @tparam A object type */ class LObjectArray32[A : ClassTag](val size:Long) extends LArray[A] { require(size < Int.MaxValue) @@ -493,8 +504,8 @@ class LObjectArray32[A : ClassTag](val size:Long) extends LArray[A] { /** * LArray[A] of Object of more than 2G entries. - * @param size - * @tparam A + * @param size array size + * @tparam A object type */ class LObjectArrayLarge[A : ClassTag](val size:Long) extends LArray[A] { diff --git a/src/main/scala/xerial/larray/LArrayBuilder.scala b/src/main/scala/xerial/larray/LArrayBuilder.scala index 6ea8ff8..4b9190c 100644 --- a/src/main/scala/xerial/larray/LArrayBuilder.scala +++ b/src/main/scala/xerial/larray/LArrayBuilder.scala @@ -13,8 +13,8 @@ import reflect.ClassTag /** * Extention of [[scala.collection.mutable.Builder]] to Long indexes - * @tparam Elem - * @tparam To + * @tparam Elem element type + * @tparam To LArray type to generate */ trait LBuilder[-Elem, +To] { @@ -57,6 +57,63 @@ trait LBuilder[-Elem, +To] { abstract class LArrayBuilder[A] extends LBuilder[A, LArray[A]] + +class LByteArrayBuilder extends LArrayBuilder[Byte] { + private var elems : LByteArray = _ + private var capacity: Long = 0L + private[larray] var size: Long = 0L + + def append(b:Array[Byte], offset:Int, len:Int) = { + ensureSize(size + len) + elems.read(b, offset, size, len) + size += len + this + } + + private def mkArray(size:Long) : LByteArray = { + val newArray = new LByteArray(size) + if(this.size > 0L) { + LArray.copy(elems, 0L, newArray, 0L, this.size) + elems.free + } + newArray + } + + override def sizeHint(size:Long) { + if(capacity < size) resize(size) + } + + private def ensureSize(size:Long) { + if(capacity < size || capacity == 0L){ + var newsize = if(capacity == 0L) 16L else (capacity * 1.5).toLong + while(newsize < size) newsize *= 2 + resize(newsize) + } + } + + private def resize(size:Long) { + elems = mkArray(size) + capacity = size + } + + def +=(elem: Byte): this.type = { + ensureSize(size + 1) + elems(size) = elem + size += 1 + this + } + + def clear() { + size = 0 + } + + def result(): LArray[Byte] = { + if(capacity != 0L && capacity == size) elems + else mkArray(size) + } + +} + /** * @author Taro L. Saito */ @@ -70,7 +127,7 @@ object LArrayBuilder { def make[T: ClassTag](): LArrayBuilder[T] = { val tag = implicitly[ClassTag[T]] tag.runtimeClass match { - case java.lang.Byte.TYPE => ofByte.asInstanceOf[LArrayBuilder[T]] + case java.lang.Byte.TYPE => new LByteArrayBuilder().asInstanceOf[LArrayBuilder[T]] case java.lang.Short.TYPE => sys.error("Not yet implemented") case java.lang.Character.TYPE => sys.error("Not yet implemented") case java.lang.Integer.TYPE => ofInt.asInstanceOf[LArrayBuilder[T]] @@ -93,11 +150,11 @@ object LArrayBuilder { def ofInt = new LArrayBuilder[Int] { - private var elems : LArray[Int] = _ + private var elems : LIntArray = _ private var capacity: Long = 0L private var size: Long = 0L - private def mkArray(size:Long) : LArray[Int] = { + private def mkArray(size:Long) : LIntArray = { val newArray = new LIntArray(size) if(this.size > 0L) { LArray.copy(elems, 0L, newArray, 0L, this.size) @@ -140,55 +197,6 @@ object LArrayBuilder { } } - def ofByte = new LArrayBuilder[Byte] { - - private var elems : LArray[Byte] = _ - private var capacity: Long = 0L - private var size: Long = 0L - - private def mkArray(size:Long) : LArray[Byte] = { - val newArray = new LByteArray(size) - if(this.size > 0L) { - LArray.copy(elems, 0L, newArray, 0L, this.size) - elems.free - } - newArray - } - - override def sizeHint(size:Long) { - if(capacity < size) resize(size) - } - - private def ensureSize(size:Long) { - if(capacity < size || capacity == 0L){ - var newsize = if(capacity == 0L) 16L else (capacity * 1.5).toLong - while(newsize < size) newsize *= 2 - resize(newsize) - } - } - - private def resize(size:Long) { - elems = mkArray(size) - capacity = size - } - - def +=(elem: Byte): this.type = { - ensureSize(size + 1) - elems(size) = elem - size += 1 - this - } - - def clear() { - size = 0 - } - - def result(): LArray[Byte] = { - if(capacity != 0L && capacity == size) elems - else mkArray(size) - } - } - // TODO ofChar // TODO ofShort @@ -201,7 +209,7 @@ object LArrayBuilder { private var elems : LArray[A] = _ private var capacity: Long = 0L - private var size: Long = 0L + private[larray] var size: Long = 0L private def mkArray(size:Long) : LArray[A] = { val newArray = LObjectArray.ofDim[A](size) diff --git a/src/main/scala/xerial/larray/LArrayInputStream.scala b/src/main/scala/xerial/larray/LArrayInputStream.scala index b094d2e..e54027e 100644 --- a/src/main/scala/xerial/larray/LArrayInputStream.scala +++ b/src/main/scala/xerial/larray/LArrayInputStream.scala @@ -8,6 +8,7 @@ package xerial.larray import java.io.InputStream +import xerial.core.log.Logger object LArrayInputStream { @@ -33,7 +34,7 @@ object LArrayInputStream { * * @author Taro L. Saito */ -private[larray] class RawLArrayInputStream[A](array:RawByteArray[A]) extends InputStream { +private[larray] class RawLArrayInputStream[A](array:RawByteArray[A]) extends InputStream with Logger { private var cursor = 0L private var mark = 0L @@ -45,10 +46,14 @@ private[larray] class RawLArrayInputStream[A](array:RawByteArray[A]) extends Inp } override def read(b: Array[Byte], offset:Int, len:Int) : Int = { - val readLen = math.min(len, array.size - cursor).toInt - array.write(cursor, b, offset, readLen) - cursor += readLen - readLen + if(cursor >= array.size) + -1 + else { + val readLen = math.min(len, array.byteLength - cursor).toInt + array.write(cursor, b, offset, readLen) + cursor += readLen + readLen + } } diff --git a/src/main/scala/xerial/larray/LArrayOutputStream.scala b/src/main/scala/xerial/larray/LArrayOutputStream.scala index 2211f37..bdc8799 100644 --- a/src/main/scala/xerial/larray/LArrayOutputStream.scala +++ b/src/main/scala/xerial/larray/LArrayOutputStream.scala @@ -8,23 +8,28 @@ package xerial.larray import java.io.OutputStream - -object LArrayOutputStream { - - def apply[A](array:LArray[A]) : OutputStream = { - array match { - case r:RawByteArray[A] => new RawLArrayOutputStream[A](r) - case _ => sys.error(s"cannot create OutputStream from this LArray class:${array.getClass}}") - } - } -} +import reflect.ClassTag /** + * Create LArray using [[java.io.OutputStream]] interface + * * @author Taro L. Saito */ -private[larray] class RawLArrayOutputStream[A](array:RawByteArray[A]) extends OutputStream { +class LArrayOutputStream[A : ClassTag] extends OutputStream { - def write(b: Int) { - // TODO impl + private val buf = new LByteArrayBuilder + + def write(v: Int) { + buf += v.toByte } + + override def write(b: Array[Byte], off: Int, len: Int) { + buf.append(b, off, len) + } + + def result : LArray[A] = { + val arr = buf.result.asInstanceOf[LByteArray] + LArray.wrap[A](arr.size, arr.m) + } + } \ No newline at end of file diff --git a/src/test/scala/xerial/larray/LArrayInputStreamTest.scala b/src/test/scala/xerial/larray/LArrayInputStreamTest.scala new file mode 100644 index 0000000..003414a --- /dev/null +++ b/src/test/scala/xerial/larray/LArrayInputStreamTest.scala @@ -0,0 +1,35 @@ +//-------------------------------------- +// +// LArrayInputStreamTest.scala +// Since: 2013/03/21 23:38 +// +//-------------------------------------- + +package xerial.larray + +import xerial.core.io.IOUtil + + +/** + * @author Taro L. Saito + */ +class LArrayInputStreamTest extends LArraySpec { + + "LArrayInputStream" should { + "be created from LArray[A]" in { + val l = LArray(1, 3, 4, 5) + debug(s"input ${l.mkString(", ")}") + val in = LArrayInputStream(l) + IOUtil.readFully(in) { buf => + debug(s"buf length: ${buf.length}") + val out = new LArrayOutputStream[Int] + out.write(buf) + val r = out.result + debug(s"output ${r.mkString(", ")}") + l.sameElements(r) should be (true) + } + } + + + } +} \ No newline at end of file diff --git a/src/test/scala/xerial/larray/LArrayTest.scala b/src/test/scala/xerial/larray/LArrayTest.scala index 878b8c4..8e4d76b 100644 --- a/src/test/scala/xerial/larray/LArrayTest.scala +++ b/src/test/scala/xerial/larray/LArrayTest.scala @@ -86,7 +86,7 @@ class LArrayTest extends LArraySpec { val b = new Array[Byte](l.byteLength.toInt) l match { - case l:RawByteArray => + case l:RawByteArray[_] => debug(s"LArray: [${l.mkString(", ")}]") debug(s"Array[Byte]: [${b.mkString(", ")}]") l.write(0, b, 0, l.byteLength.toInt) @@ -96,7 +96,7 @@ class LArrayTest extends LArraySpec { debug(s"Array[Byte]: [${b.mkString(", ")}]") val l2 = LArray(0, 0) l2 match { - case l2:RawByteArray => + case l2:RawByteArray[_] => l2.read(b, 0, 0, b.length) debug(s"LArray2: [${l2.mkString(", ")}]") l.sameElements(l2) should be (true) @@ -125,7 +125,7 @@ class LArrayTest extends LArraySpec { val a = new Array[Int](M) var i = 0 while (i < M) { - a(i) = r.nextInt(N); + a(i) = r.nextInt(N) i += 1 } a @@ -223,6 +223,7 @@ class LArrayTest extends LArraySpec { "have constructor" in { val a = Array[Byte](1, 2, 3) + val b = LArray[Byte](1, 5, 34) b(0) should be (1.toByte)