Add general search service with support for Work Item (#395)

Why:
Allow users to search using keywords, by passing id:ID_VALUE and URL.

What:
Currently, only WorkItem linking is the target for the search.

* Support search by ID.

* Support search by Free Text.

* Support search by URL.
When a token in the search string is found to be a URL, it is mapped with existing
URL patterns in the system [only WI as of now].
input URL is matched and parsed against compiled regex.
ID is taken out from URL and postfixed with :* to make startswith()

Usage:
./bin/alm-cli show search --q "Test WI" -H localhost:8080 --pp
./bin/alm-cli show search --q "id:1001" -H localhost:8080 --pp
./bin/alm-cli show search --q "http%3A%2F%2Fdemo.almighty.io%2Fdetail%2F3" -H localhost:8080
./bin/alm-cli show search --q "demo.almighty.io%2Fdetail%2F3" -H localhost:8080

Fixes #330
Fixes #380
Fixes #381
Fixes #382
Fixes #383
Related #307

application/application.go

@@ -6,6 +6,7 @@ type Application interface {
 	WorkItemTypes() WorkItemTypeRepository
 	Trackers() TrackerRepository
 	TrackerQueries() TrackerQueryRepository
+	SearchItems() SearchRepository
 }
 
 // A Transaction abstracts a database transaction. The repositories created for the transaction object make changes inside the the transaction

application/repositories.go

@@ -37,3 +37,8 @@ type TrackerQueryRepository interface {
 	Save(ctx context.Context, tq app.TrackerQuery) (*app.TrackerQuery, error)
 	Load(ctx context.Context, ID string) (*app.TrackerQuery, error)
 }
+
+// SearchRepository encapsulates searching of woritems,users,etc
+type SearchRepository interface {
+	SearchFullText(ctx context.Context, searchStr string, start *int, length *int) ([]*app.WorkItem, uint64, error)
+}

design/media_types.go

@@ -198,3 +198,28 @@ var User = a.MediaType("application/vnd.user+json", func() {
 		a.Attribute("imageURL")
 	})
 })
+
+var searchResponse = a.MediaType("application/vnd.search+json", func() {
+	a.TypeName("SearchResponse")
+	a.Description("Holds the paginated response to a search request")
+	a.Attribute("links", pagingLinks)
+	a.Attribute("meta", meta)
+	a.Attribute("data", a.CollectionOf(workItem))
+
+	a.Required("links")
+	a.Required("meta")
+	a.Required("data")
+
+	a.View("default", func() {
+		a.Attribute("links", func() {
+			a.Attribute("prev", d.String)
+			a.Attribute("next", d.String)
+			a.Attribute("first", d.String)
+			a.Attribute("last", d.String)
+		})
+		a.Attribute("meta", func() {
+			a.Attribute("totalCount", d.Integer)
+		})
+		a.Attribute("data")
+	})
+})

design/resources.go

@@ -379,3 +379,34 @@ var _ = a.Resource("trackerquery", func() {
 	})
 
 })
+
+var _ = a.Resource("search", func() {
+	a.BasePath("/search")
+
+	a.Action("show", func() {
+		a.Routing(
+			a.GET(""),
+		)
+		a.Description("Search by ID, URL, full text capability")
+		a.Params(func() {
+			a.Param("q", d.String,
+				`Following are valid input for seach query
+				1) "id:100" :- Look for work item hainvg id 100
+				2) "url:http://demo.almighty.io/details/500" :- Search on WI having id 500 and check 
+					if this URL is mentioned in searchable columns of work item
+				3) "simple keywords seperated by space" :- Search in Work Items based on these keywords.`)
+			a.Param("page[offset]", d.String, "Paging start position") // #428
+			a.Param("page[limit]", d.Integer, "Paging size")
+			a.Required("q")
+		})
+		a.Response(d.OK, func() {
+			a.Media(searchResponse)
+		})
+
+		a.Response(d.BadRequest, func() {
+			a.Media(d.ErrorMedia)
+		})
+
+		a.Response(d.InternalServerError)
+	})
+})

gormapplication/application.go

@@ -7,6 +7,7 @@ import (
 	"github.com/almighty/almighty-core/application"
 	"github.com/almighty/almighty-core/models"
 	"github.com/almighty/almighty-core/remoteworkitem"
+	"github.com/almighty/almighty-core/search"
 	"github.com/jinzhu/gorm"
 )
 
@@ -70,6 +71,10 @@ func (g *GormBase) TrackerQueries() application.TrackerQueryRepository {
 	return remoteworkitem.NewTrackerQueryRepository(g.db)
 }
 
+func (g *GormBase) SearchItems() application.SearchRepository {
+	return search.NewGormSearchRepository(g.db)
+}
+
 func (g *GormBase) DB() *gorm.DB {
 	return g.db
 }

main.go

@@ -189,6 +189,10 @@ func main() {
 	userCtrl := NewUserController(service, identityRepository, tokenManager)
 	app.MountUserController(service, userCtrl)
 
+	// Mount "search" controller
+	searchCtrl := NewSearchController(service, appDB)
+	app.MountSearchController(service, searchCtrl)
+
 	fmt.Println("Git Commit SHA: ", Commit)
 	fmt.Println("UTC Build Time: ", BuildTime)
 	fmt.Println("UTC Start Time: ", StartTime)

migration/migration.go

@@ -91,6 +91,9 @@ func getMigrations() migrations {
 	// Version 4
 	m = append(m, steps{executeSQLFile("004-drop-tracker-query-id.sql")})
 
+	// Version 5
+	m = append(m, steps{executeSQLFile("005-add-search-index.sql")})
+
 	// Version N
 	//
 	// In order to add an upgrade, simply append an array of MigrationFunc to the

migration/sql-files/005-add-search-index.sql

@@ -0,0 +1,22 @@
+-- Add field on work_item  to store Full Text Search Vector
+ALTER TABLE work_items ADD tsv tsvector;
+
+UPDATE work_items SET tsv =
+    setweight(to_tsvector('english', id::text),'A') ||
+    setweight(to_tsvector('english', coalesce(fields->>'system.title','')),'B') ||
+    setweight(to_tsvector('english', coalesce(fields->>'system.description','')),'C');
+
+CREATE INDEX fulltext_search_index ON work_items USING GIN (tsv);
+
+CREATE FUNCTION workitem_tsv_trigger() RETURNS trigger AS $$
+begin
+  new.tsv :=
+    setweight(to_tsvector('english', new.id::text),'A') ||
+    setweight(to_tsvector('english', coalesce(new.fields->>'system.title','')),'B') ||
+    setweight(to_tsvector('english', coalesce(new.fields->>'system.description','')),'C');
+  return new;
+end
+$$ LANGUAGE plpgsql;
+
+CREATE TRIGGER upd_tsvector BEFORE INSERT OR UPDATE OF id, fields ON work_items
+FOR EACH ROW EXECUTE PROCEDURE workitem_tsv_trigger();

search.go

@@ -0,0 +1,132 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+
+	"github.com/almighty/almighty-core/app"
+	"github.com/almighty/almighty-core/application"
+	"github.com/almighty/almighty-core/models"
+	"github.com/goadesign/goa"
+)
+
+// SearchController implements the search resource.
+type SearchController struct {
+	*goa.Controller
+	db application.DB
+}
+
+// NewSearchController creates a search controller.
+func NewSearchController(service *goa.Service, db application.DB) *SearchController {
+	if db == nil {
+		panic("db must not be nil")
+	}
+	return &SearchController{Controller: service.NewController("SearchController"), db: db}
+}
+
+// Show runs the show action.
+func (c *SearchController) Show(ctx *app.ShowSearchContext) error {
+	var offset int
+	var limit int
+
+	if ctx.PageOffset == nil {
+		offset = 0
+	} else {
+		offsetValue, err := strconv.Atoi(*ctx.PageOffset)
+		if err != nil {
+			offset = 0
+		} else {
+			offset = offsetValue
+		}
+	}
+
+	if ctx.PageLimit == nil {
+		limit = 100
+	} else {
+		limit = *ctx.PageLimit
+	}
+	if offset < 0 {
+		return ctx.BadRequest(goa.ErrBadRequest(fmt.Sprintf("offset must be >= 0, but is: %d", offset)))
+	}
+
+	return application.Transactional(c.db, func(appl application.Application) error {
+		//return transaction.Do(c.ts, func() error {
+		result, c, err := appl.SearchItems().SearchFullText(ctx.Context, ctx.Q, &offset, &limit)
+		count := int(c)
+		if err != nil {
+			switch err := err.(type) {
+			case models.BadParameterError:
+				return ctx.BadRequest(goa.ErrBadRequest(fmt.Sprintf("Error listing work items: %s", err.Error())))
+			default:
+				log.Printf("Error listing work items: %s", err.Error())
+				return ctx.InternalServerError()
+			}
+		}
+
+		response := app.SearchResponse{
+			Links: &app.PagingLinks{},
+			Meta:  &app.WorkItemListResponseMeta{TotalCount: count},
+			Data:  result,
+		}
+
+		// prev link
+		if offset > 0 && count > 0 {
+			var prevStart int
+			// we do have a prev link
+			if offset <= count {
+				prevStart = offset - limit
+			} else {
+				// the first range that intersects the end of the useful range
+				prevStart = offset - (((offset-count)/limit)+1)*limit
+			}
+			realLimit := limit
+			if prevStart < 0 {
+				// need to cut the range to start at 0
+				realLimit = limit + prevStart
+				prevStart = 0
+			}
+			prev := fmt.Sprintf("%s?q=%s&page[offset]=%d&page[limit]=%d", buildAbsoluteURL(ctx.RequestData), ctx.Q, prevStart, realLimit)
+			response.Links.Prev = &prev
+		}
+
+		// next link
+		nextStart := offset + len(result)
+		if nextStart < count {
+			// we have a next link
+			next := fmt.Sprintf("%s?q=%s&page[offset]=%d&page[limit]=%d", buildAbsoluteURL(ctx.RequestData), ctx.Q, nextStart, limit)
+			response.Links.Next = &next
+		}
+
+		// first link
+		var firstEnd int
+		if offset > 0 {
+			firstEnd = offset % limit // this is where the second page starts
+		} else {
+			// offset == 0, first == current
+			firstEnd = limit
+		}
+		first := fmt.Sprintf("%s?q=%s&page[offset]=%d&page[limit]=%d", buildAbsoluteURL(ctx.RequestData), ctx.Q, 0, firstEnd)
+		response.Links.First = &first
+
+		// last link
+		var lastStart int
+		if offset < count {
+			// advance some pages until touching the end of the range
+			lastStart = offset + (((count - offset - 1) / limit) * limit)
+		} else {
+			// retreat at least one page until covering the range
+			lastStart = offset - ((((offset - count) / limit) + 1) * limit)
+		}
+		realLimit := limit
+		if lastStart < 0 {
+			// need to cut the range to start at 0
+			realLimit = limit + lastStart
+			lastStart = 0
+		}
+		last := fmt.Sprintf("%s?q=%s&page[offset]=%d&page[limit]=%d", buildAbsoluteURL(ctx.RequestData), ctx.Q, lastStart, realLimit)
+		response.Links.Last = &last
+
+		return ctx.OK(&response)
+	})
+}