-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
7 changed files
with
195 additions
and
59 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| package ai.starlake.utils | ||
|
|
||
| import scala.util.parsing.combinator.JavaTokenParsers | ||
|
|
||
| // From https://users.scala-lang.org/t/solved-parser-combinator-removing-comments/6635 | ||
| object CommentParser extends JavaTokenParsers { | ||
|
|
||
| def singleLine: Parser[String] = "//.*".r ^^ (_ => "") | ||
| def multiLine: Parser[String] = """/\*.*\*/""".r ^^^ "" | ||
| def comments: Parser[Seq[String]] = (singleLine | multiLine).* | ||
| def commentedText: Parser[String] = comments ~> "[^\\/*]*".r <~ comments | ||
| def empty: Parser[Seq[String]] = ".*$".r ^^ { e => Seq(e) } | ||
| def expression: Parser[String] = commentedText ~ (empty | commentedText.*) ^^ { | ||
| case (a: String) ~ (b: Seq[String]) => a + b.mkString("") | ||
| } | ||
|
|
||
| def stripComments(str: String): Either[String, String] = { | ||
| parseAll(expression, str) match { | ||
| case Success(result, _) => Right(result) | ||
| case failedOrIncomplete => Left(failedOrIncomplete.toString) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 0 additions & 35 deletions
35
src/test/scala/ai/starlake/job/gcp/BigQueryNativeJobTest.scala
This file was deleted.
Oops, something went wrong.
74 changes: 74 additions & 0 deletions
74
src/test/scala/ai/starlake/job/index/bqload/BigQueryNativeJobSpec.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package ai.starlake.job.index.bqload | ||
|
|
||
| import ai.starlake.TestHelper | ||
| import ai.starlake.config.Settings | ||
| import ai.starlake.schema.handlers.{SchemaHandler, SimpleLauncher} | ||
| import ai.starlake.schema.model._ | ||
| import ai.starlake.workflow.{IngestionWorkflow, TransformConfig} | ||
| import com.google.cloud.bigquery.TableId | ||
| import org.apache.hadoop.fs.Path | ||
| import org.scalatest.BeforeAndAfterAll | ||
|
|
||
| class BigQueryNativeJobSpec extends TestHelper with BeforeAndAfterAll { | ||
| override def beforeAll(): Unit = { | ||
| BigQueryJobBase.bigquery.delete(TableId.of("bqtest", "account")) | ||
| } | ||
| override def afterAll(): Unit = { | ||
| BigQueryJobBase.bigquery.delete(TableId.of("bqtest", "account")) | ||
| } | ||
|
|
||
| new WithSettings() { | ||
| "Ingest to BigQuery" should "should be ingest and store table in BigQuery" in { | ||
| if (sys.env.getOrElse("COMET_GCP_TEST", "false").toBoolean) { | ||
| import org.slf4j.impl.StaticLoggerBinder | ||
| val binder = StaticLoggerBinder.getSingleton | ||
| logger.debug(binder.getLoggerFactory.toString) | ||
| logger.debug(binder.getLoggerFactoryClassStr) | ||
|
|
||
| new WithSettings() { | ||
| new SpecTrait( | ||
| domainOrJobFilename = "bqtest.comet.yml", | ||
| sourceDomainOrJobPathname = "/sample/position/bqtest.comet.yml", | ||
| datasetDomainName = "bqtest", | ||
| sourceDatasetPathName = "/sample/position/XPOSTBL" | ||
| ) { | ||
| cleanMetadata | ||
| cleanDatasets | ||
|
|
||
| logger.info(settings.comet.datasets) | ||
| loadPending | ||
| } | ||
| } | ||
| val tableFound = | ||
| Option(BigQueryJobBase.bigquery.getTable(TableId.of("bqtest", "account"))).isDefined | ||
| tableFound should be(true) | ||
|
|
||
| } | ||
| } | ||
| "Native BigQuery AutoJob" should "succeed" in { | ||
| val businessTask1 = AutoTaskDesc( | ||
| None, | ||
| Some("select * from bqtest.account"), | ||
| "bqtest", | ||
| "jobresult", | ||
| WriteMode.OVERWRITE, | ||
| sink = Some(BigQuerySink(name = Some("sinktest"), location = Some("EU"))), | ||
| engine = Some(Engine.BQ) | ||
| ) | ||
| val businessJob = | ||
| AutoJobDesc("user", List(businessTask1), None, None, None, engine = Some(Engine.BQ)) | ||
| val schemaHandler = new SchemaHandler(metadataStorageHandler) | ||
|
|
||
| val businessJobDef = mapper | ||
| .writer() | ||
| .withAttribute(classOf[Settings], settings) | ||
| .writeValueAsString(businessJob) | ||
| lazy val pathBusiness = new Path(cometMetadataPath + "/jobs/bqjobtest.comet.yml") | ||
|
|
||
| val workflow = | ||
| new IngestionWorkflow(storageHandler, schemaHandler, new SimpleLauncher()) | ||
| storageHandler.write(businessJobDef, pathBusiness) | ||
| workflow.autoJob(TransformConfig("bqjobtest")) should be(true) | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| package ai.starlake.utils | ||
|
|
||
| import com.typesafe.scalalogging.StrictLogging | ||
| import org.apache.spark.sql.DatasetLogging | ||
| import org.scalatest.BeforeAndAfterAll | ||
| import org.scalatest.flatspec.AnyFlatSpec | ||
| import org.scalatest.matchers.should.Matchers | ||
|
|
||
| class CommentParserSpec | ||
| extends AnyFlatSpec | ||
| with Matchers | ||
| with BeforeAndAfterAll | ||
| with StrictLogging | ||
| with DatasetLogging { | ||
| "Taxonomy" should "list files by modification_time and name" in { | ||
| val r1 = CommentParser.stripComments("/* a comment */") | ||
| Right("") should equal(r1) | ||
|
|
||
| val r2 = CommentParser.stripComments("// a comment") | ||
| Right("") should equal(r2) | ||
|
|
||
| val r3 = CommentParser.stripComments("/* level1 /* level 2 */ */") | ||
| Right("") should equal(r3) | ||
|
|
||
| val r4 = CommentParser.stripComments("Text Before./* level1 /* level 2 */ */") | ||
| Right("Text Before.") should equal(r4) | ||
|
|
||
| val r5 = CommentParser.stripComments("/* level1 /* level 2 */ */Text after.") | ||
| Right("Text after.") should equal(r5) | ||
|
|
||
| val r6 = CommentParser.stripComments("Text Before./* level1 /* level 2 */ */Text after.") | ||
| Right("Text Before.Text after.") should equal(r6) | ||
| } | ||
| } |