Retrieve newest projects

core/src/main/scala/ch/epfl/scala/index/newModel/NewProject.scala

@@ -1,5 +1,7 @@
 package ch.epfl.scala.index.newModel
 
+import java.time.Instant
+
 import ch.epfl.scala.index.model.Project
 import ch.epfl.scala.index.model.misc.GithubInfo
 import ch.epfl.scala.index.model.misc.GithubRepo
@@ -11,6 +13,7 @@ case class NewProject(
     organization: Organization,
     repository: Repository,
     githubInfo: Option[GithubInfo],
+    created: Option[Instant], // equivalent to the first release date
     esId: Option[String],
     // form data
     dataForm: DataForm
@@ -63,6 +66,7 @@ object NewProject {
       Organization(org),
       repository = Repository(repo),
       githubInfo = githubInfo,
+      created = None,
       esId = None,
       dataForm = formData
     )
@@ -132,6 +136,7 @@ object NewProject {
       organization = Organization(p.organization),
       repository = Repository(p.repository),
       githubInfo = p.github,
+      created = None,
       esId = p.id,
       dataForm = DataForm.from(p)
     )

core/src/main/scala/ch/epfl/scala/index/newModel/NewRelease.scala

@@ -1,5 +1,9 @@
 package ch.epfl.scala.index.newModel
 
+import java.time.Instant
+
+import scala.util.Try
+
 import ch.epfl.scala.index.model.Env
 import ch.epfl.scala.index.model.License
 import ch.epfl.scala.index.model.Release
@@ -14,7 +18,6 @@ import ch.epfl.scala.index.newModel.NewProject.DocumentationLink
 import ch.epfl.scala.index.newModel.NewProject.Organization
 import ch.epfl.scala.index.newModel.NewProject.Repository
 import ch.epfl.scala.index.newModel.NewRelease._
-import org.joda.time.DateTime
 import org.joda.time.format.ISODateTimeFormat
 
 /**
@@ -31,7 +34,7 @@ case class NewRelease(
     artifactName: ArtifactName,
     platform: Platform,
     description: Option[String],
-    released: Option[DateTime],
+    releasedAt: Option[Instant],
     resolver: Option[Resolver],
     licenses: Set[License],
     isNonStandardLib: Boolean
@@ -215,6 +218,9 @@ object NewRelease {
   case class ArtifactName(value: String) extends AnyVal {
     override def toString: String = value
   }
+  // we used to write joda.DateTime and now we moved to Instant
+  private def parseToInstant(s: String): Option[Instant] =
+    Try(format.parseDateTime(s)).map(t => Instant.ofEpochMilli(t.getMillis)).orElse(Try(Instant.parse(s))).toOption
 
   def from(r: Release): NewRelease =
     NewRelease(
@@ -225,7 +231,7 @@ object NewRelease {
       version = r.reference.version,
       platform = r.reference.target,
       description = r.description,
-      released = r.released.map(format.parseDateTime),
+      releasedAt = r.released.flatMap(parseToInstant),
       resolver = r.resolver,
       licenses = r.licenses,
       isNonStandardLib = r.isNonStandardLib

core/src/main/scala/ch/epfl/scala/services/SchedulerDatabase.scala

@@ -0,0 +1,13 @@
+package ch.epfl.scala.services
+
+import scala.concurrent.Future
+
+import ch.epfl.scala.index.newModel.NewProject
+import ch.epfl.scala.index.newModel.ProjectDependency
+
+trait SchedulerDatabase {
+  def getAllProjectRef(): Future[Seq[NewProject.Reference]]
+  def getAllProjectDependencies(): Future[Seq[ProjectDependency]]
+  def updateCreatedInProjects(ref: NewProject.Reference): Future[Unit]
+  def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int]
+}

core/src/main/scala/ch/epfl/scala/services/DatabaseApi.scala → core/src/main/scala/ch/epfl/scala/services/WebDatabase.scala

@@ -7,38 +7,22 @@ import ch.epfl.scala.index.newModel.NewProject
 import ch.epfl.scala.index.newModel.NewProject.DataForm
 import ch.epfl.scala.index.newModel.NewRelease
 import ch.epfl.scala.index.newModel.NewRelease.ArtifactName
-import ch.epfl.scala.index.newModel.ProjectDependency
 import ch.epfl.scala.index.newModel.ReleaseDependency
 
-trait DatabaseApi {
+trait WebDatabase {
   def insertProject(p: NewProject): Future[Unit]
   def insertOrUpdateProject(p: NewProject): Future[Unit]
-  def updateProjectForm(
-      ref: NewProject.Reference,
-      dataForm: DataForm
-  ): Future[Unit]
+  def updateProjectForm(ref: NewProject.Reference, dataForm: DataForm): Future[Unit]
   def findProject(projectRef: NewProject.Reference): Future[Option[NewProject]]
   def insertReleases(r: Seq[NewRelease]): Future[Int]
   def findReleases(projectRef: NewProject.Reference): Future[Seq[NewRelease]]
-  def findReleases(
-      projectRef: NewProject.Reference,
-      artifactName: ArtifactName
-  ): Future[Seq[NewRelease]]
-
-  def findDirectDependencies(
-      release: NewRelease
-  ): Future[List[ReleaseDependency.Direct]]
-  def findReverseDependencies(
-      release: NewRelease
-  ): Future[List[ReleaseDependency.Reverse]]
+  def findReleases(projectRef: NewProject.Reference, artifactName: ArtifactName): Future[Seq[NewRelease]]
+  def getMostDependentUponProject(max: Int): Future[List[(NewProject, Long)]]
+  def findDirectDependencies(release: NewRelease): Future[List[ReleaseDependency.Direct]]
+  def findReverseDependencies(release: NewRelease): Future[List[ReleaseDependency.Reverse]]
   def getAllTopics(): Future[Seq[String]]
   def getAllPlatforms(): Future[Map[NewProject.Reference, Set[Platform]]]
-  def getAllProjectDependencies(): Future[Seq[ProjectDependency]]
-  def insertProjectDependencies(
-      projectDependencies: Seq[ProjectDependency]
-  ): Future[Int]
-  def getMostDependentUponProject(max: Int): Future[List[(NewProject, Long)]]
-
+  def getLatestProjects(limit: Int): Future[Seq[NewProject]]
   def insertDependencies(dependencies: Seq[ReleaseDependency]): Future[Int]
   def countProjects(): Future[Long]
   def countReleases(): Future[Long]

core/src/test/scala/ch/epfl/scala/index/newModel/NewReleaseTests.scala

@@ -199,7 +199,7 @@ class NewReleaseTests extends AnyFunSpec with Matchers {
       artifactName = artifactName,
       platform = platform,
       description = None,
-      released = None,
+      releasedAt = None,
       resolver = resolver,
       licenses = Set(),
       isNonStandardLib = false

data/src/main/scala/ch.epfl.scala.index.data/project/PomMeta.scala

@@ -2,6 +2,8 @@ package ch.epfl.scala.index
 package data
 package project
 
+import java.time.Instant
+
 import ch.epfl.scala.index.data.bintray._
 import ch.epfl.scala.index.data.maven.ReleaseModel
 import ch.epfl.scala.index.model.release._
@@ -120,12 +122,12 @@ object PomMeta {
 
   def from(
       pom: ReleaseModel,
-      created: DateTime,
+      created: Instant,
       localRepository: LocalRepository,
       paths: DataPaths,
       sha1: String
   ): Option[PomMeta] = {
-    val createdString = PomMeta.format.print(created) // todo: keep Datetime
+    val createdString = created.toEpochMilli.toString
 
     if (!pom.isPackagingOfInterest) None
     else {

data/src/main/scala/ch.epfl.scala.index.data/project/ProjectConvert.scala

@@ -2,6 +2,8 @@ package ch.epfl.scala.index
 package data
 package project
 
+import java.time.Instant
+
 import ch.epfl.scala.index.data.bintray._
 import ch.epfl.scala.index.data.cleanup._
 import ch.epfl.scala.index.data.github._
@@ -254,7 +256,7 @@ class ProjectConvert(paths: DataPaths, githubDownload: GithubDownload) extends B
       pom: ReleaseModel,
       localRepository: LocalRepository,
       sha1: String,
-      created: DateTime,
+      created: Instant,
       githubRepo: GithubRepo,
       existingProject: Option[NewProject]
   ): Option[(NewProject, NewRelease, Seq[ReleaseDependency])] = {

infra/src/main/resources/migrations/V1__initial_schema.sql

@@ -1,6 +1,7 @@
 CREATE TABLE projects (
     organization         VARCHAR(39)   NOT NULL,
     repository           VARCHAR(100)  NOT NULL,
+    created_at           TIMESTAMPTZ,
     esId                 VARCHAR,
     PRIMARY KEY (organization, repository)
 );
@@ -58,7 +59,7 @@ CREATE TABLE releases (
     artifact            VARCHAR NOT NULL,
     platform            VARCHAR NOT NULL,
     description         VARCHAR,
-    released            VARCHAR,
+    released_at         TIMESTAMPTZ,
     resolver            VARCHAR,
     licenses            VARCHAR NOT NULL,
     isNonStandardLib    BOOLEAN NOT NULL,

infra/src/main/scala/ch.epfl.scala.index.search/ESRepo.scala

@@ -243,15 +243,6 @@ class ESRepo(esClient: ElasticClient, indexPrefix: String)(implicit ec: Executio
       .map(_.result.to[ReleaseDocument].headOption.map(_.toRelease))
   }
 
-  def getLatestProjects(): Future[List[Project]] =
-    for {
-      projects <- getLatest[Project](
-        projectIndex,
-        "created",
-        frontPageCount
-      )
-    } yield projects.map(_.formatForDisplaying)
-
   def getLatestReleases(): Future[List[Release]] =
     getLatest[ReleaseDocument](releaseIndex, "released", frontPageCount)
       .map(_.map(_.toRelease))

infra/src/main/scala/ch/epfl/scala/services/storage/sql/SqlRepo.scala

@@ -11,7 +11,8 @@ import ch.epfl.scala.index.newModel.NewProject
 import ch.epfl.scala.index.newModel.NewRelease
 import ch.epfl.scala.index.newModel.ProjectDependency
 import ch.epfl.scala.index.newModel.ReleaseDependency
-import ch.epfl.scala.services.DatabaseApi
+import ch.epfl.scala.services.SchedulerDatabase
+import ch.epfl.scala.services.WebDatabase
 import ch.epfl.scala.services.storage.sql.tables.GithubInfoTable
 import ch.epfl.scala.services.storage.sql.tables.ProjectDependenciesTable
 import ch.epfl.scala.services.storage.sql.tables.ProjectTable
@@ -22,7 +23,7 @@ import ch.epfl.scala.utils.DoobieUtils
 import ch.epfl.scala.utils.ScalaExtensions._
 import doobie.implicits._
 
-class SqlRepo(conf: DatabaseConfig, xa: doobie.Transactor[IO]) extends DatabaseApi {
+class SqlRepo(conf: DatabaseConfig, xa: doobie.Transactor[IO]) extends WebDatabase with SchedulerDatabase {
   private[sql] val flyway = DoobieUtils.flyway(conf)
   def migrate: IO[Unit] = IO(flyway.migrate())
   def dropTables: IO[Unit] = IO(flyway.clean())
@@ -43,6 +44,9 @@ class SqlRepo(conf: DatabaseConfig, xa: doobie.Transactor[IO]) extends DatabaseA
   def insertReleasesWithFailures(releases: Seq[NewRelease]): Future[Seq[(NewRelease, Try[Unit])]] =
     releases.map(r => insertRelease(r).failWithTry.map((r, _))).sequence
 
+  override def getAllProjectRef(): Future[Seq[NewProject.Reference]] =
+    run(ProjectTable.selectAllProjectRef().to[List])
+
   override def insertOrUpdateProject(project: NewProject): Future[Unit] =
     for {
       _ <- run(ProjectTable.insertOrUpdate(project))
@@ -54,27 +58,18 @@ class SqlRepo(conf: DatabaseConfig, xa: doobie.Transactor[IO]) extends DatabaseA
 
   override def updateProjectForm(ref: NewProject.Reference, dataForm: NewProject.DataForm): Future[Unit] =
     for {
-      projectOpt <- run(
-        ProjectTable.selectOne(ref.organization, ref.repository)
-      )
+      projectOpt <- run(ProjectTable.selectOne(ref))
       _ <- projectOpt
         .map(p => run(ProjectUserFormTable.update(p)(dataForm)))
         .getOrElse(Future.successful(()))
     } yield ()
 
   override def findProject(projectRef: NewProject.Reference): Future[Option[NewProject]] =
     for {
-      project <- run(
-        ProjectTable.selectOne(projectRef.organization, projectRef.repository)
-      )
-      userForm <- run(
-        ProjectUserFormTable.selectOne(
-          projectRef.organization,
-          projectRef.repository
-        )
-      )
+      project <- run(ProjectTable.selectOne(projectRef))
+      userForm <- run(ProjectUserFormTable.selectOne(projectRef))
       githubInfoTable <- project
-        .map(p => run(GithubInfoTable.selectOne(p.organization, p.repository)))
+        .map(p => run(GithubInfoTable.selectOne(p.reference)))
         .getOrElse(Future.successful(None))
     } yield project.map(
       _.copy(
@@ -140,6 +135,15 @@ class SqlRepo(conf: DatabaseConfig, xa: doobie.Transactor[IO]) extends DatabaseA
           NewProject.Reference(org, repo)
       }(_._3).view.mapValues(_.toSet).toMap)
 
+  override def getLatestProjects(limit: Int): Future[Seq[NewProject]] =
+    for {
+      projects <- run(ProjectTable.selectLatestProjects(limit).to[List])
+      projectWithGithubInfo <- projects.map { p =>
+        run(GithubInfoTable.selectOne(p.reference))
+          .map(github => p.copy(githubInfo = github))
+      }.sequence
+    } yield projectWithGithubInfo
+
   def countGithubInfo(): Future[Long] =
     run(GithubInfoTable.indexedGithubInfo().unique)
 
@@ -165,6 +169,15 @@ class SqlRepo(conf: DatabaseConfig, xa: doobie.Transactor[IO]) extends DatabaseA
       }.sequence
     } yield res.flatten
 
+  private[sql] def findOldestRelease(ref: NewProject.Reference): Future[Option[NewRelease]] =
+    run(ReleaseTable.selectOldestRelease(ref).option)
+
+  override def updateCreatedInProjects(ref: NewProject.Reference): Future[Unit] =
+    for {
+      releaseOpt <- findOldestRelease(ref)
+      _ <- run(ProjectTable.updateCreated(ref, releaseOpt.flatMap(_.releasedAt)))
+    } yield ()
+
   // to use only when inserting one element or updating one element
   // when expecting a row to be modified
   private def strictRun(update: doobie.Update0, expectedRows: Int = 1): Future[Unit] =

infra/src/main/scala/ch/epfl/scala/services/storage/sql/tables/GithubInfoTable.scala

@@ -70,18 +70,18 @@ object GithubInfoTable {
     ).update
   }
 
-  def selectOne(org: Organization, repo: Repository): doobie.ConnectionIO[Option[GithubInfo]] =
-    selectOneQuery(org, repo).option
+  def selectOne(ref: NewProject.Reference): doobie.ConnectionIO[Option[GithubInfo]] =
+    selectOneQuery(ref).option
 
   def selectAllTopics(): doobie.Query0[Set[String]] =
     buildSelect(tableFr, fr0"topics", fr0"where topics != ''")
       .query[Set[String]]
 
-  private[tables] def selectOneQuery(org: Organization, repo: Repository): doobie.Query0[GithubInfo] =
+  private[tables] def selectOneQuery(reference: NewProject.Reference): doobie.Query0[GithubInfo] =
     buildSelect(
       tableFr,
       fieldsFr,
-      where(org, repo)
+      where(reference)
     ).query[GithubInfo](githubInfoReader)
 
   val githubInfoReader: Read[GithubInfo] =