diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f072570f..ca78c6990 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,32 +29,54 @@ jobs: cache: 'sbt' - name: Misc. Setup run: | + git branch --show-current sudo snap install task --classic + sudo snap install aws-cli --classic sudo sysctl -w vm.max_map_count=262144 - - name: Initialize SBT - run: sbt update - - name: Compile without Cache +# - name: Initialize SBT +# run: sbt update +# - name: Compile without Cache +# run: | +# task jvm:clean +# sbt compile Test/compile +# task jvm:clean +# - name: Pull Build Cache +# env: +# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} +# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} +# run: | +# sbt pullRemoteCache +# aws s3 sync s3://elastiknn-sbt-build-cache/succeeded_tests/ . +# - name: Compile with Cache +# run: sbt compile Test/compile + - name: Run Cluster + run: task cluster:run + - name: Test and push cache + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | task jvm:clean - sbt compile Test/compile - task jvm:clean - - name: Pull Build Cache + sbt clean compile Test/compile testQuick testQuick pushRemoteCache + find . -name succeeded_tests | xargs cat + aws s3 sync --exclude '*' --include '**/succeeded_tests' . s3://elastiknn-sbt-build-cache/succeeded_tests/ + - name: Pull cache and test env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: sbt pullRemoteCache - - name: Compile with Cache - run: sbt compile Test/compile - - name: Run Cluster - run: task cluster:run - - name: Test - run: task jvm:test + run: | + task jvm:clean + aws s3 sync s3://elastiknn-sbt-build-cache/succeeded_tests/ . + find . -name succeeded_tests | xargs cat + sbt clean pullRemoteCache compile Test/compile testQuick - name: Push Build Cache if: github.event_name == 'push' env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: sbt pushRemoteCache + run: | + sbt pushRemoteCache + aws s3 sync --exclude '*' --include '**/succeeded_tests' . s3://elastiknn-sbt-build-cache/succeeded_tests/ - name: Cluster Logs if: failure() run: task cluster:logs diff --git a/elastiknn-testing/src/test/scala/com/klibisz/elastiknn/query/RecallSuite.scala b/elastiknn-testing/src/test/scala/com/klibisz/elastiknn/query/RecallSuite.scala index a99f17ea7..92f3bd03d 100644 --- a/elastiknn-testing/src/test/scala/com/klibisz/elastiknn/query/RecallSuite.scala +++ b/elastiknn-testing/src/test/scala/com/klibisz/elastiknn/query/RecallSuite.scala @@ -6,6 +6,7 @@ import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s.Response import com.sksamuel.elastic4s.requests.searches.SearchResponse import futil.Futil +import org.scalatest.DoNotDiscover import org.scalatest.concurrent.AsyncCancelAfterFailure import org.scalatest.funsuite.AsyncFunSuite import org.scalatest.matchers.should.Matchers @@ -25,6 +26,7 @@ import scala.util.hashing.MurmurHash3.orderedHash * the same query, I have seen different results at times. This seems to be an effect at the Elasticsearch level. * I've tested at the Lucene (sans ES) level and that seems to be reliably deterministic. */ +@DoNotDiscover class RecallSuite extends AsyncFunSuite with Matchers with ElasticAsyncClient with AsyncCancelAfterFailure { // Each test case consists of setting up one Mapping and then running several queries against that mapping. diff --git a/project/BuildCachePlugin.scala b/project/BuildCachePlugin.scala index 06757cee2..2b93927d3 100644 --- a/project/BuildCachePlugin.scala +++ b/project/BuildCachePlugin.scala @@ -19,11 +19,14 @@ object BuildCachePlugin extends AutoPlugin { // This build cache is currently only available to the AWS IAM user used by Elastiknn's CI. // If you don't have permissions, running pullRemoteCache will fail gracefully with warnings. + private val s3BaseUrl = "s3://elastiknn-sbt-build-cache.s3-us-east-1.amazonaws.com/remote_cache" private val s3RemoteCacheResolver = "S3 Remote Cache" at "s3://elastiknn-sbt-build-cache.s3-us-east-1.amazonaws.com/remote_cache" override lazy val projectSettings = Seq( - remoteCacheResolvers += s3RemoteCacheResolver, + remoteCacheResolvers ++= Seq( + "S3 Remote Cache" at s"$s3BaseUrl/master" + ), pushRemoteCacheTo := Some(s3RemoteCacheResolver), Compile / pushRemoteCacheConfiguration ~= (_.withOverwrite(true)), Test / pushRemoteCacheConfiguration ~= (_.withOverwrite(true))