#28 edit chart menu

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.watcherExclude": {
+        "**/target": true
+    }
+}

build.sc

@@ -1,7 +1,7 @@
 import $ivy.`com.lihaoyi::mill-contrib-bloop:$MILL_VERSION`
 import $ivy.`org.scala-js::scalajs-env-jsdom-nodejs:1.0.0`
 import $ivy.`org.slf4j:slf4j-simple:1.6.1`
-import $ivy.`io.bit3:jsass:5.10.4`
+import $ivy.`io.bit3:jsass:5.11.0`
 import mill._
 import mill.scalalib._
 import mill.scalalib.publish._
@@ -15,7 +15,7 @@ import java.util.Calendar
  *** The Vizier Backend 
  *************************************************/
 object vizier extends ScalaModule with PublishModule {
-  val VERSION       = "2.0.0-rc2"
+  val VERSION       = "2.0.0-rc3"
   val PLAY_JS       = ivy"com.typesafe.play::play-json::2.9.2"
 
   val MIMIR_CAVEATS = ivy"info.vizierdb::mimir-caveats::0.3.6"

scripts/bootstrap.sh

@@ -1,4 +1,4 @@
-ARTIFACT=info.vizierdb:vizier_2.12:2.0.0-rc2 
+ARTIFACT=info.vizierdb:vizier_2.12:2.0.0-rc3
 
 coursier bootstrap $ARTIFACT\
   -r https://s01.oss.sonatype.org/content/groups/public/ \

test_data/1.01. Simple linear regression.csv

@@ -0,0 +1,85 @@
+SAT,GPA
+1714,2.4
+1664,2.52
+1760,2.54
+1685,2.74
+1693,2.83
+1670,2.91
+1764,3
+1764,3
+1792,3.01
+1850,3.01
+1735,3.02
+1775,3.07
+1735,3.08
+1712,3.08
+1773,3.12
+1872,3.17
+1755,3.17
+1674,3.17
+1842,3.17
+1786,3.19
+1761,3.19
+1722,3.19
+1663,3.2
+1687,3.21
+1974,3.24
+1826,3.28
+1787,3.28
+1821,3.28
+2020,3.28
+1794,3.28
+1769,3.28
+1934,3.28
+1775,3.29
+1855,3.29
+1880,3.29
+1849,3.31
+1808,3.32
+1954,3.34
+1777,3.37
+1831,3.37
+1865,3.37
+1850,3.38
+1966,3.38
+1702,3.39
+1990,3.39
+1925,3.4
+1824,3.4
+1956,3.4
+1857,3.41
+1979,3.41
+1802,3.41
+1855,3.42
+1907,3.42
+1634,3.42
+1879,3.44
+1887,3.47
+1730,3.47
+1953,3.47
+1781,3.47
+1891,3.48
+1964,3.49
+1808,3.49
+1893,3.5
+2041,3.51
+1893,3.51
+1832,3.52
+1850,3.52
+1934,3.54
+1861,3.58
+1931,3.58
+1933,3.59
+1778,3.59
+1975,3.6
+1934,3.6
+2021,3.61
+2015,3.62
+1997,3.64
+2020,3.65
+1843,3.71
+1936,3.71
+1810,3.71
+1987,3.73
+1962,3.76
+2050,3.81

test_data/output_with_nulls.csv

@@ -0,0 +1,26 @@
+title,description,date,quantity
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,
+,,,

vizier/backend/src/info/vizierdb/Vizier.scala

@@ -167,14 +167,6 @@ object Vizier
     }
   }
 
-  def setWorkingDirectory(): Unit =
-  {
-    if(config.workingDirectory.isDefined){
-      val path = new File(config.workingDirectory()).getAbsolutePath()
-      System.setProperty("user.dir", path)
-    }
-  }
-
   def main(args: Array[String]) 
   {
     config = new Config(args)

vizier/backend/src/info/vizierdb/api/GetArtifact.scala

@@ -17,8 +17,9 @@ package info.vizierdb.api
 import scalikejdbc._
 import play.api.libs.json._
 import org.apache.spark.sql.DataFrame
-
+import play.api.libs.json.JsArray
 import info.vizierdb.catalog.Artifact
+import info.vizierdb.util.ExperimentalOptions
 import info.vizierdb.types.{ Identifier, ArtifactType }
 import org.mimirdb.caveats.Caveat
 import info.vizierdb.spark.caveats.CaveatFormat._
@@ -36,12 +37,13 @@ object GetArtifact
   extends LazyLogging
 {
   def getArtifact(projectId: Long, artifactId: Long, expecting: Option[ArtifactType.T]): Option[Artifact] = 
-      CatalogDB.withDBReadOnly { implicit session => 
+      CatalogDB.withDB { implicit session => 
         Artifact.getOption(artifactId, Some(projectId))
       }.filter { artifact => 
         expecting.isEmpty || expecting.get.equals(artifact.t)
       }
 
+
   def apply(
     projectId: Identifier,
     artifactId: Identifier,
@@ -58,7 +60,7 @@ object GetArtifact
     val forceProfiler = profile.map { _.equals("true") }.getOrElse(false)
     getArtifact(projectId, artifactId, expecting) match {
       case Some(artifact) => 
-        CatalogDB.withDBReadOnly { implicit s => 
+        CatalogDB.withDB{ implicit s => 
           artifact.describe(
             offset = offset, 
             limit = limit, 
@@ -77,7 +79,6 @@ object GetArtifact
         ErrorResponse.noSuchEntity
     }
   }
-
   def typed(
     expectedType: ArtifactType.T
   )(
@@ -135,7 +136,7 @@ object GetArtifact
     {
       getArtifact(projectId, artifactId, None) match {
         case Some(artifact) => 
-          CatalogDB.withDBReadOnly { implicit s => artifact.summarize() }
+          CatalogDB.withDB { implicit s => artifact.summarize() }
         case None => 
           ErrorResponse.noSuchEntity
       }

vizier/backend/src/info/vizierdb/catalog/Artifact.scala

@@ -35,6 +35,7 @@ import info.vizierdb.spark.SparkSchema.fieldFormat
 import org.apache.spark.sql.AnalysisException
 import com.typesafe.scalalogging.LazyLogging
 import info.vizierdb.api.akka.VizierServer
+import info.vizierdb.profiler.DataProfiler
 import info.vizierdb.util.ExperimentalOptions
 
 case class Artifact(
@@ -183,12 +184,20 @@ case class Artifact(
 
           val df = dataframe
 
+          logger.trace("Dataset Properties: \n"+string)
+
           { () => 
-            val data = dataConstructor()
+            val data = dataConstructor()  
             val rowCount: Long = 
-                data.properties
-                    .get("count")
-                    .map { _.as[Long] }
+              data.properties
+                .get("count")
+                .flatMap { values =>
+                  try {
+                    Some(values.as[Long])
+                  } catch {
+                    case e: Exception => None
+                  }
+                }
                     .getOrElse { df().count() }
 
             Artifact.translateDatasetContainerToVizierClassic(
@@ -251,6 +260,29 @@ case class Artifact(
                )
     val computeCaveats = 
       ExperimentalOptions.isEnabled("ENABLE-MIMIR") && includeCaveats
+
+    val df = dataframe(session)()
+    if (forceProfiler) {
+      val updatedProperties = datasetDescriptor.properties.get("is_profiled") match {
+        case Some(JsBoolean(true)) =>
+          // profiler has been generated so do nothing
+        case _ =>
+          println("Profiler has been generated")
+          // run the profiler
+          val dataProfile: Map[String, JsValue] = DataProfiler.apply(df)
+          // Transform the dataProfile into a sequence of tuples
+          // Filter out the specified keys and transform the remaining dataProfile into a sequence of tuples
+          val dynamicProperties: Seq[(String, JsValue)] = dataProfile
+            .filterKeys(key => key != "is_profiled")
+            .toSeq
+          // Combine the static properties with the dynamic ones
+          updateDatasetProperties(
+            Seq("is_profiled" -> JsBoolean(true)) ++ dynamicProperties: _*
+          )
+      }
+    }
+
+
     return { () => 
       try {
         QueryWithCaveats(
@@ -291,6 +323,7 @@ case class Artifact(
     datasetDescriptor.properties.get(name)
   }
 
+
   /**
    * Retrieve or construct the specified dataset property.
    * @param    name       The name of a dataset property.

vizier/backend/src/info/vizierdb/commands/Commands.scala

@@ -102,10 +102,10 @@ object Commands
   register(packageId = "plot", name = "Data Plotting", category = "plot")(
     "chart"        -> info.vizierdb.commands.plot.SimpleChart,
     "line-chart"   -> info.vizierdb.commands.plot.LineChart,
-    "scatterplot"  -> info.vizierdb.commands.plot.ScatterPlot,
+    "scatterplot" -> info.vizierdb.commands.plot.ScatterPlot,
     "cdf"          -> info.vizierdb.commands.plot.CDFPlot,
     "geo"          -> info.vizierdb.commands.plot.GeoPlot,
-    "bar-chart"   -> info.vizierdb.commands.plot.BarChart,
+    "barchart"    -> info.vizierdb.commands.plot.BarChart,
   )
 
   register(packageId = "sql", name = "SQL", category = "code")(