search page

wip search page auto commit auto commit handle pages attached by the node.directoryPath no results search input breadcrumb breadcrumb breadcrumb breadcrumb remove code linked to kelkoo remove pom.xml use mysql 8
KelkooGroup · Sep 2, 2020 · 22af79f · 22af79f
1 parent c3cc476
commit 22af79f
Show file tree

Hide file tree

Showing 42 changed files with 1,457 additions and 1,000 deletions.
diff --git a/app/controllers/PageController.scala b/app/controllers/PageController.scala
@@ -30,11 +30,10 @@ class PageController @Inject()(pageService: PageService, searchService: SearchSe
     }
   }
 
-  @ApiOperation(value = "Search pages through keywords", response = classOf[PageIndexDocument], responseContainer = "list")
+  @ApiOperation(value = "Search pages through a keyword", response = classOf[SearchResult])
   @ApiResponses(Array(new ApiResponse(code = 404, message = "Page not found")))
-  def searchPage(keywords: String): Action[AnyContent] = Action {
-    implicit val itemsFormat = Json.format[PageIndexDocument]
-    Ok(Json.toJson(searchService.searchForPage(keywords)))
+  def searchPage(keyword: String): Action[AnyContent] = Action {
+    Ok(Json.toJson(searchService.searchForPage(keyword)))
   }
 
 }

diff --git a/app/repositories/HierarchyRepository.scala b/app/repositories/HierarchyRepository.scala
@@ -67,6 +67,12 @@ class HierarchyRepository @Inject()(db: Database) {
     }
   }
 
+  def findByDirectoryPath(directoryPath: String): Option[HierarchyNode] = {
+    db.withConnection { implicit connection =>
+      SQL"SELECT * FROM hierarchyNode WHERE directoryPath = $directoryPath".as(parser.*).headOption
+    }
+  }
+
   def findAllByProjectId(projectId: String): Seq[HierarchyNode] = {
     db.withConnection { implicit connection =>
       SQL"SELECT * FROM project_hierarchyNode INNER JOIN hierarchyNode on (id = hierarchyId) WHERE projectId = $projectId".as(parser.*)

diff --git a/app/services/HierarchyService.scala b/app/services/HierarchyService.scala
@@ -24,22 +24,47 @@ class HierarchyService @Inject()(hierarchyRepository: HierarchyRepository) {
     case Some(shortcut) => !hierarchy.id.startsWith(shortcut)
   }
 
-  def getHierarchyPath(pageJoinProject: PageJoinProject): String = {
-    val projectHierarchies = hierarchyRepository.findAllByProjectId(pageJoinProject.project.id)
-    if (projectHierarchies.nonEmpty) {
-      val hierarchyId = projectHierarchies.headOption.map(_.id).getOrElse("")
-      val splitedId = hierarchyId.split('.')
-      var hierarchyPath = ""
-      var currentHierarchyId = "."
-      for (id <- splitedId) {
-        if (id.nonEmpty) {
-          currentHierarchyId += id + "."
-          hierarchyPath += "/" + hierarchyRepository.findById(currentHierarchyId).map(_.name).getOrElse("Not Found")
+  def getHierarchyPath(pageJoinProject: PageJoinProject): Option[String] = {
+    val projectHierarchy = hierarchyRepository.findAllByProjectId(pageJoinProject.project.id).headOption match {
+      case Some(h) => Some(h)
+      case None => hierarchyRepository.findByDirectoryPath(pageJoinProject.directory.path)
+    }
+    projectHierarchy match {
+      case Some(h) =>
+        val hierarchyId = h.id
+        val spitedId = hierarchyId.split('.')
+        var hierarchyPath = ""
+        var currentHierarchyId = "."
+        for (id <- spitedId) {
+          if (id.nonEmpty) {
+            currentHierarchyId += id + "."
+            hierarchyPath += "/" + hierarchyRepository.findById(currentHierarchyId).map(_.slugName).getOrElse("Not Found")
+          }
+        }
+        Some(hierarchyPath)
+      case None => None
+    }
+  }
+
+  def getBreadcrumb(pageJoinProject: PageJoinProject): String = {
+    val projectHierarchy = hierarchyRepository.findAllByProjectId(pageJoinProject.project.id).headOption match {
+      case Some(h) => Some(h)
+      case None => hierarchyRepository.findByDirectoryPath(pageJoinProject.directory.path)
+    }
+    projectHierarchy match {
+      case Some(h) =>
+        val hierarchyId = h.id
+        val spitedId = hierarchyId.split('.')
+        var hierarchyPath = ""
+        var currentHierarchyId = "."
+        for (id <- spitedId) {
+          if (id.nonEmpty) {
+            currentHierarchyId += id + "."
+            hierarchyPath += "/" + hierarchyRepository.findById(currentHierarchyId).map(_.name).getOrElse("Not Found")
+          }
         }
-      }
-      hierarchyPath + "/" + pageJoinProject.branch.name + pageJoinProject.page.relativePath
-    } else {
-      pageJoinProject.page.path
+        hierarchyPath.replaceFirst("/", "") + " / " + pageJoinProject.project.name + pageJoinProject.directory.relativePath.replaceAll("/", " / ") + pageJoinProject.page.label
+      case None => ""
     }
   }
 

diff --git a/app/services/PageService.scala b/app/services/PageService.scala
@@ -245,7 +245,9 @@ class PageService @Inject()(config: Configuration, projectRepository: ProjectRep
   def computePageFromPathUsingDatabaseBis(pageJoinProjectOpt: Option[PageJoinProject], path: String, forceRefresh: Boolean = true): Future[Option[PageWithContent]] = {
     pageJoinProjectOpt match {
       case Some(pageJoinProject) =>
-        indexService.addDocument(PageIndexDocument(hierarchyService.getHierarchyPath(pageJoinProject), pageJoinProject.page.path, pageJoinProject.branch.name, pageJoinProject.page.label, pageJoinProject.page.description, pageJoinProject.page.markdown.getOrElse("")))
+        if (pageJoinProject.branch.isStable) {
+          insertOrUpdateIndex(pageJoinProject)
+        }
         val key = computePageCacheKey(path)
         if (cache.get(key).isEmpty || forceRefresh || pageJoinProject.page.dependOnOpenApi) {
           logger.debug(s"Page computed: $path")
@@ -273,6 +275,23 @@ class PageService @Inject()(config: Configuration, projectRepository: ProjectRep
     }
   }
 
+  private def insertOrUpdateIndex(pageJoinProject: PageJoinProject) = {
+
+    hierarchyService.getHierarchyPath(pageJoinProject).map{ hierarchy =>
+      val document = PageIndexDocument(id = hierarchy + "/" + pageJoinProject.page.path,
+        hierarchy = hierarchy,
+        path = pageJoinProject.page.path,
+        breadcrumb = hierarchyService.getBreadcrumb(pageJoinProject),
+        project = pageJoinProject.project.name,
+        branch = pageJoinProject.branch.name,
+        label = pageJoinProject.page.label,
+        description = pageJoinProject.page.description,
+        pageContent = pageJoinProject.page.markdown.getOrElse(""))
+      indexService.insertOrUpdateDocument(document)
+    }
+
+  }
+
   def processPage(currentDirectory: Directory, localDirectoryPath: String, name: String, order: Int): Option[Page] = {
     val pageFile = new File(localDirectoryPath + "/" + name + ".md")
     if (pageFile.exists()) {

diff --git a/app/services/SearchService.scala b/app/services/SearchService.scala
@@ -1,33 +1,93 @@
 package services
 
+import com.outr.lucene4s.query.{Sort, TermSearchTerm}
 import com.outr.lucene4s.{DirectLucene, parse}
-import com.outr.lucene4s.query.Sort
 import javax.inject.{Inject, Singleton}
+import play.api.libs.json.Json
 
+case class SearchResult(items: Seq[SearchResultItem])
+
+case class SearchResultItem(page: PageIndexDocument, highlights: Seq[HighlightedFragment])
+
+case class PageIndexDocument(id: String, hierarchy: String, path: String, breadcrumb: String, project: String, branch: String, label: String, description: String, pageContent: String)
+
+case class HighlightedFragment(fragment: String, word: String)
+
+object HighlightedFragment {
+  implicit val format = Json.format[HighlightedFragment]
+}
+
+object PageIndexDocument {
+  implicit val format = Json.format[PageIndexDocument]
+}
+
+object SearchResultItem {
+  implicit val format = Json.format[SearchResultItem]
+}
+
+object SearchResult {
+  implicit val format = Json.format[SearchResult]
+}
 
-case class PageIndexDocument(hierarchy: String, path: String, branch: String, label: String, description: String, pageContent: String)
 
 @Singleton
 class IndexService {
 
-  val luceneSearchIndex = new DirectLucene(uniqueFields = List("path"), defaultFullTextSearchable = true)
+  val luceneSearchIndex = new DirectLucene(uniqueFields = List("id"), defaultFullTextSearchable = true, appendIfExists = false, autoCommit = true)
 
+  private val id = luceneSearchIndex.create.field[String]("id")
   private val hierarchy = luceneSearchIndex.create.field[String]("hierarchy")
   private val path = luceneSearchIndex.create.field[String]("path", fullTextSearchable = false)
+  private val breadcrumb = luceneSearchIndex.create.field[String]("breadcrumb")
+  private val project = luceneSearchIndex.create.field[String]("project")
   private val branch = luceneSearchIndex.create.field[String]("branch")
   private val label = luceneSearchIndex.create.field[String]("label")
   private val description = luceneSearchIndex.create.field[String]("description")
   private val pageContent = luceneSearchIndex.create.field[String]("pageContent", sortable = false)
 
-  def addDocument(document: PageIndexDocument): Unit = {
-    luceneSearchIndex.doc().fields(hierarchy(document.hierarchy), description(document.description), label(document.label), this.path(document.path), branch(document.branch), pageContent(document.pageContent)).index()
-  }
+  def insertOrUpdateDocument(document: PageIndexDocument): Unit = {
 
-  def query(keywords: String): Seq[PageIndexDocument] = {
-    val results = luceneSearchIndex.query().sort(Sort.Score).filter(parse("label:" + keywords + "*^100 | pageContent:" + keywords + "*^30 | description:" + keywords + "*^50 | branch:" + keywords + "*^30")).highlight().search()
-    results.results.map {
-      result => PageIndexDocument(result(hierarchy), result(path), result(branch), result(label), result(description), result(pageContent))
+    var i = 0
+    while (exists(document) && i < 10) {
+      luceneSearchIndex.delete(new TermSearchTerm(Some(id), document.id))
+      luceneSearchIndex.commit()
+      i = i + 1
     }
+
+    luceneSearchIndex.doc().fields(id(document.id),
+      hierarchy(document.hierarchy),
+      path(document.path),
+      breadcrumb(document.breadcrumb),
+      project(document.project),
+      branch(document.branch),
+      label(document.label),
+      description(document.description),
+      pageContent(document.pageContent)
+    ).index()
+    luceneSearchIndex.commit()
+  }
+
+  private def exists(document: PageIndexDocument) = {
+    luceneSearchIndex.query().filter(new TermSearchTerm(Some(id), document.id)).search().results.length > 0
+  }
+
+  def query(keywords: String): SearchResult = {
+    val results = luceneSearchIndex.query().sort(Sort.Score).filter(parse("id:" + keywords + "*^20 " + " | "
+      + "breadcrumb:" + keywords + "*^50 " + " | "
+      + "project:" + keywords + "*^30 " + " | "
+      + "branch:" + keywords + "*^30" + " | "
+      + "label:" + keywords + "*^100 " + " | "
+      + "description:" + keywords + "*^100 " + " | "
+      + "pageContent:" + keywords + "*^30"
+    )).highlight().limit(50).search()
+    SearchResult(results.results.map {
+      result =>
+        val highlighting = result.highlighting(breadcrumb).map(frg => HighlightedFragment(frg.fragment, frg.word)) +:
+          result.highlighting(label).map(frg => HighlightedFragment(frg.fragment, frg.word)) +:
+          result.highlighting(description).map(frg => HighlightedFragment(frg.fragment, frg.word)) +:
+          result.highlighting(pageContent).map(frg => HighlightedFragment(frg.fragment, frg.word)) +: Nil
+        SearchResultItem(PageIndexDocument(result(id), result(hierarchy), result(path), result(breadcrumb), result(project), result(branch), result(label), result(description), result(pageContent)), highlighting.flatten)
+    })
   }
 
   def reset(): Unit = {
@@ -38,7 +98,7 @@ class IndexService {
 
 class SearchService @Inject()(pageIndex: IndexService) {
 
-  def searchForPage(keywords: String): Seq[PageIndexDocument] = {
+  def searchForPage(keywords: String): SearchResult = {
     pageIndex.query(keywords)
   }
 

diff --git a/build.sbt b/build.sbt
@@ -88,7 +88,7 @@ libraryDependencies ++= Seq(
   "org.julienrf" %% "play-json-derived-codecs" % "6.0.0",
   "io.cucumber" % "gherkin" % "5.2.0",
   "com.typesafe.play" %% "anorm" % "2.5.3",
-  "mysql" % "mysql-connector-java" % "5.1.48",
+  "mysql" % "mysql-connector-java" % "8.0.20",
   "org.eclipse.jgit" % "org.eclipse.jgit" % "5.6.1.202002131546-r",
   "io.swagger" %% "swagger-play2" % "1.7.1",
   "com.h2database" % "h2" % "1.4.199",

diff --git a/conf/routes b/conf/routes
@@ -41,7 +41,7 @@ GET        /api/config                                     controllers.ConfigCon
 GET        /api/directories                                controllers.DirectoryController.getDirectoryFromPath(path: String)
 
 GET        /api/pages                                      controllers.PageController.getPageFromPath(path: String)
-GET        /api/pages/search                               controllers.PageController.searchPage(keywords: String)
+GET        /api/pages/search                               controllers.PageController.searchPage(keyword: String)
 
 POST       /api/admin/menu/refreshFromDatabase                               controllers.AdminController.refreshMenu()
 POST       /api/admin/projects/refreshFromDatabase                           controllers.AdminController.refreshAllProjectsFromDatabase()

diff --git a/documentation/Changelog.md b/documentation/Changelog.md
@@ -8,6 +8,11 @@
 }
 ```
 
+### [2020-09][1.4.0] 
+
+The main features is:
+- [0655 - Search](https://github.com/KelkooGroup/theGardener/issues/655) 
+
 ### [2020-07][1.3.2] 
 
 The main features is: