Pages

content/content.go

@@ -392,9 +392,19 @@ func constructArticlesFromSearchResults( sRequest *SearchRequest, sResponse *Sea
 func Search(sRequest *SearchRequest) *SearchResponse {
     startTiming := time.Now()
 
-    queryString       := constructQueryString( sRequest )
-    jsonBody          := getSapiResponseJsonBody(queryString, sRequest.MaxArticles)
-    sResponse         := parseSapiResponseJsonBody(jsonBody, sRequest, queryString)
+    fmt.Println("content.Search: sRequest.QueryType==page")
+
+    var sResponse *SearchResponse
+    if sRequest.QueryType == "pages" {
+        webUrl    := sRequest.QueryText
+        pageId    := getPageIdByWebUrl(webUrl)
+        jsonBody  := constructMainContentJsonBodyFromId(pageId)
+        sResponse  = parseMainContentJsonBody(jsonBody, sRequest, webUrl)
+    } else {
+        queryString := constructQueryString( sRequest )
+        jsonBody    := getSapiResponseJsonBody(queryString, sRequest.MaxArticles)
+        sResponse    = parseSapiResponseJsonBody(jsonBody, sRequest, queryString)
+    }
 
     var articles *[]*Article
     if !sRequest.SearchOnly {
@@ -408,6 +418,157 @@ func Search(sRequest *SearchRequest) *SearchResponse {
     return sResponse
 }
 
+func constructGetResponseJsonBody(url string) (*[]byte) {
+    urlWithKey := url + "?apiKey=" + apiKey
+
+    req, err := http.NewRequest("GET", urlWithKey, nil)
+    req.Header.Set("Content-Type", "application/json")
+    client := &http.Client{}
+    resp, err := client.Do(req)
+    if err != nil {
+        panic(err)
+    }
+    defer resp.Body.Close()
+
+    if resp.Status != "200 OK" {
+        fmt.Println("WARNING: content: constructGetResponseJsonBody: response Status:", resp.Status, ", url=", url)
+    }
+
+    jsonBody, _ := ioutil.ReadAll(resp.Body)
+
+    return &jsonBody
+}
+
+func constructAllPagesJsonBody() (*[]byte) {
+    return constructGetResponseJsonBody("http://api.ft.com/site/v1/pages")
+}
+
+func parseAllPagesJsonBody(jsonBody *[]byte) (*map[string]string) {
+
+    mapWebUrlToId := map[string]string{}
+
+    // locate results
+    var data interface{}
+    json.Unmarshal(*jsonBody, &data)
+
+    if pages, ok := data.(map[string]interface{})["pages"].([]interface{}); ok {
+        for _, p := range pages {
+            id     := ""
+            webUrl := ""
+
+            if _,ok := p.(map[string]interface{})["id"]; ok {
+                id = p.(map[string]interface{})["id"].(string)
+            }
+
+            if _,ok := p.(map[string]interface{})["webUrl"]; ok {
+                webUrl = p.(map[string]interface{})["webUrl"].(string)
+            }
+
+            mapWebUrlToId[webUrl] = id
+        }
+    }
+    return &mapWebUrlToId
+}
+
+var allKnownPageIdsByWebUrl *map[string]string
+
+func getAllPages() (*map[string]string) {
+    if allKnownPageIdsByWebUrl == nil {
+        jsonBody := constructAllPagesJsonBody()
+        allKnownPageIdsByWebUrl = parseAllPagesJsonBody(jsonBody)
+        fmt.Println("content.getAllPages: len(allKnownPageIdsByWebUrl)=", len(*allKnownPageIdsByWebUrl))
+    }
+
+    return allKnownPageIdsByWebUrl
+}
+
+func getPageIdByWebUrl(webUrl string) string {
+    return (*getAllPages())[webUrl]
+}
+
+func constructMainContentJsonBodyFromId(id string) (*[]byte) {
+    url := "http://api.ft.com/site/v1/pages/" + id + "/main-content"
+    return constructGetResponseJsonBody(url)
+}
+
+func parseMainContentJsonBody(jsonBody *[]byte, sReq *SearchRequest, webUrl string) *SearchResponse {
+
+    siteSearchUrl := "http://search.ft.com/"
+    numPossible   := 0
+    articles      := []*Article{}
+
+    // locate results
+    var data interface{}
+    json.Unmarshal(*jsonBody, &data)
+    if pageItems, ok := data.(map[string]interface{})["pageItems"].([]interface{}); ok {
+        for _, r := range pageItems {
+            siteUrl := ""
+            uuid    := ""
+            title   := ""
+            author  := ""
+            pubDateString := ""
+            var pubDateTime *time.Time
+
+            if id, ok := r.(map[string]interface{})["id"].(string); ok {
+                uuid = id
+            }
+
+            if titleOuter, ok := r.(map[string]interface{})["title"].(map[string]interface{}); ok {
+                if titleString, ok := titleOuter["title"].(string); ok {
+                    title = titleString
+                }
+            }
+
+            if location, ok := r.(map[string]interface{})["location"].(map[string]interface{}); ok {
+                if locationUri, ok := location["uri"].(string); ok {
+                    siteUrl = locationUri
+                }
+            }
+
+            if editorial, ok := r.(map[string]interface{})["editorial"].(map[string]interface{}); ok {
+                if byline, ok := editorial["byline"].(string); ok {
+                    author = byline
+                }
+            }
+
+            if lifecycle, ok := r.(map[string]interface{})["lifecycle"].(map[string]interface{}); ok {
+                if lastPublishDateTimeString, ok := lifecycle["lastPublishDateTime"].(string); ok {
+                    pubDateString = lastPublishDateTimeString
+                    pubDateTime = parsePubDateString(pubDateString)
+                }
+            }
+
+            body := title
+            excerpt := title
+
+            article := Article{
+                SiteUrl: siteUrl,
+                Uuid:    uuid,
+                Title:   title,
+                Author:  author,
+                Excerpt: excerpt,
+                Body:    body,
+                PubDateString: pubDateString,
+                PubDate:       pubDateTime,
+            }
+
+            articles = append( articles, &article )
+        }
+    }
+
+    searchResponse := SearchResponse{
+        SiteUrl:       webUrl,
+        SiteSearchUrl: siteSearchUrl,
+        NumArticles:   len(articles),
+        NumPossible:   numPossible,
+        Articles:      &articles,
+        QueryString:   "",
+        SearchRequest: sReq,
+    }
+
+    return &searchResponse
+}
+
 func main() {
     godotenv.Load()
     uuid := "b57fee24-cb3c-11e5-be0b-b7ece4e953a0"

templates/align.html

@@ -153,6 +153,30 @@ <h2>find ontology's recent articles' phrases matching the haiku meter, 5-7-5</h2
 									</table>			
 								</form>
 
+							<h2>find page's current articles' phrases matching the haiku meter, 5-7-5</h2>
+								<p>
+									You specify a sitepage. 
+									<br>If any of the sentences within the relevant articles contain text conforming to the haiku meter, 5-7-5, they are displayed. 
+									<br>Meter, you ask? See <a href="http://examples.yourdictionary.com/examples-of-iambic-pentameter.html">the internet</a> for some examples.
+									<br>You specify the meter as a sequence of 0s and 1s, where 1 is the emphasised beat.
+									<br>You can anchor the meter to the ^start or end$ of the text, or leave it free to match anywhere.
+								</p>
+								<form action="/ontology" method="GET">
+									<table>
+										<tr>
+											<td style="text-align:right;">
+												<input type="hidden" name="ontology" value="pages"> 
+												page&nbsp;<input type="text" name="value" value="http://www.ft.com/home/uk"> 
+												<br><input type="hidden" name="meter" value="..... ....... .....">
+											</td>
+											<td>
+												<input type="hidden" name="max" value="20">
+												<input type="submit" value="search for the page's articles and align on matching meter"> (NB: there will be a bit of a delay)
+											</td>
+										</tr>
+									</table>			
+								</form>
+
 							<h2>explore phrases matching meter</h2>
 								<p>
 									You specify a phrase and a meter. 

web-server.go

@@ -106,7 +106,7 @@ func main() {
 
 	http.HandleFunc("/",        log(alignFormHandler))
     http.HandleFunc("/align",   log(alignHandler))
-    http.HandleFunc("/article", log(ontologyHandler))
+    // http.HandleFunc("/article", log(ontologyHandler))
     http.HandleFunc("/detail",  log(detailHandler))
     http.HandleFunc("/ontology", log(ontologyHandler))