Skip to content

Commit

Permalink
Some refactoring, and new features.
Browse files Browse the repository at this point in the history
  - Separating functions into files
  - Adding command-line argument for authorID
  - Implementing concurrency for additional requests
  - Adding all results to a map[int]string
  - Fixing a possible panic if AuthorID slice is empty
  • Loading branch information
dalzuga committed Jan 9, 2017
1 parent 4010d14 commit 5da28cc
Show file tree
Hide file tree
Showing 3 changed files with 177 additions and 131 deletions.
4 changes: 2 additions & 2 deletions books.xml
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,7 @@
<description>
<![CDATA[
Each book in the series is (mostly) a whole story with a beginning and an end, however the plot (and characters) of every book relies <i><b>very</b></i> heavily on the prior books in the series.
Series also known as:
<i>* Guida galattica per gli autostoppisti</i> [Italian]
<i>* Štoparski vodnik po Galaksiji</i> [Slovenian]
Expand Down Expand Up @@ -662,4 +662,4 @@
</similar_books>
</book>

</GoodreadsResponse>
</GoodreadsResponse>
181 changes: 52 additions & 129 deletions go_parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ package main

import (
"encoding/xml"
"errors"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"strconv"
)

Expand All @@ -16,18 +16,32 @@ import (
*/

func main() {
AuthorID, err := getAuthorID("books.xml")
if err != nil {
log.Fatal(err)
var AuthorID int
var err error

argc := len(os.Args)

if argc == 2 {
AuthorID, err = strconv.Atoi(os.Args[1])
if err != nil {
log.Fatal(err)
}
} else {
AuthorID, err = getAuthorID("books.xml")
if err != nil {
log.Fatal(err)
}
}

fmt.Println(AuthorID)

mapTitles, err := requestAllBookTitles(AuthorID)
if err != nil {
log.Fatal(err)
}

/* print the book titles */
mapLength := len(mapTitles)

for i := 0; i < mapLength; i++ {
fmt.Println(i+1, mapTitles[i])
}
Expand All @@ -51,6 +65,10 @@ func getAuthorID(fileName string) (int, error) {
return 0, err
}

if len(grbq.Book.Authors) < 1 {
return 0, errors.New("Author ID not found on XML file!")
}

return grbq.Book.Authors[0].ID, nil // Return the Author ID from the GRBQ struct
}

Expand All @@ -72,135 +90,40 @@ func requestAllBookTitles(AuthorID int) (map[int]string, error) {
fmt.Println("Additional requests needed:", more)
}

var moreTitles map[int]string

for more > 0 {
page++
// mapTitles, more, err := requestPage(page, AuthorID, endpointBase)
moreTitles, more, err = requestPage(page, AuthorID, endpointBase)
if err != nil {
return make(map[int]string), err
}

i := len(mapTitles)
for _, value := range moreTitles {
mapTitles[i] = value
i++
}
}

return mapTitles, nil
}

/* This function requests a page from the API. */
func requestPage(page int, AuthorID int, endpointBase string) (map[int]string, int, error) {
req, err := prepareRequest(endpointBase, page, AuthorID)
if err != nil {
return make(map[int]string), 0, err
}

/*
* resp is of type *http.Response
*/
resp, err := doRequest(req)
if err != nil {
return make(map[int]string), 0, err
}

/*
* var 'more' is an int.
* If the API needs to paginate the response, more will indicate how many
* pages need to be requested for a full list of book titles.
* If there is no need to paginate, more will default to 0.
*/
mapTitles, more, err := parseResponse(resp)
if err != nil {
return make(map[int]string), 0, err
}

return mapTitles, more, nil
}

func prepareRequest(endpointBase string, page int, AuthorID int) (*http.Request, error) {
/*
* Here, 'u' is a url object.
*/
u, err := url.Parse(endpointBase)
if err != nil {
return nil, err
}

/*
* Here, we make a query 'q' out of url object 'u'
*/
q := u.Query()
q.Set("key", `kDkKnUxiz8cRBJhVjrtSA`)
q.Set("id", strconv.Itoa(AuthorID))
q.Set("page", strconv.Itoa(page))

/*
* Here, 's' is our fully constructed URL string
*/
u.RawQuery = q.Encode()
s := u.String()
fmt.Println(s)

req, err := http.NewRequest("GET", s, nil)
if err != nil {
return nil, err
}

return req, nil
}

func doRequest(req *http.Request) (*http.Response, error) {
client := &http.Client{}

resp, err := client.Do(req)
if err != nil {
return nil, err
}

return resp, nil
}

func parseResponse(resp *http.Response) (map[int]string, int, error) {
requestBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return make(map[int]string), 0, err
/* Make a map of channels, 1 channel per page */
channels := make(map[int]chan map[int]string)
for i := 2; i <= more+1; i++ {
channels[i] = make(chan map[int]string)
}

var graq GoodReadsAuthorQuery

err = xml.Unmarshal(requestBytes, &graq)
if err != nil {
return make(map[int]string), 0, err
/* Make 'more' (number of) requests */
for i := 2; i <= more+1; i++ {
go func(i int) {
moreTitles, _, err := requestPage(i, AuthorID, endpointBase)
if err != nil {
fmt.Println("This request failed:", i)
channels[i] <- make(map[int]string)
} else {
fmt.Println("Received page:", i)
channels[i] <- moreTitles
}
}(i)
}

var requestTitles = make(map[int]string)
/* usually 30, but left variable in case API changes (untested) */
booksPerPage := len(mapTitles)

for key, bookValue := range graq.Author.Books.Book {
requestTitles[key] = bookValue.Title
}
/* Receive pages in order */
for i := 2; i <= more+1; i++ {
moreTitles := <-channels[i]

more, err := checkForMore(&graq)
if err != nil {
return make(map[int]string), 0, err
}

return requestTitles, more, nil
}

func checkForMore(graq *GoodReadsAuthorQuery) (int, error) {
var start, end, total int

start = graq.Author.Books.Start
end = graq.Author.Books.End
total = graq.Author.Books.Total

if total != end {
return (total-end+start)/(end-start) + 1, nil
/* add them to mapTitles (sequential) */
for j := 0; j <= booksPerPage; j++ {
if moreTitles[j] != "" {
mapTitles[(i-1)*booksPerPage+j] = moreTitles[j]
}
}
}

return 0, nil
return mapTitles, nil
}
123 changes: 123 additions & 0 deletions requestFunctions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package main

import (
"encoding/xml"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strconv"
)

/* This function requests a page from the API. */
func requestPage(page int, AuthorID int, endpointBase string) (map[int]string, int, error) {
req, err := prepareRequest(endpointBase, page, AuthorID)
if err != nil {
return make(map[int]string), 0, err
}

/*
* resp is of type *http.Response
*/
resp, err := doRequest(req)
if err != nil {
return make(map[int]string), 0, err
}

/*
* var 'more' is an int.
* If the API needs to paginate the response, more will indicate how many
* pages need to be requested for a full list of book titles.
* If there is no need to paginate, more will default to 0.
*/
mapTitles, more, err := parseResponse(resp)
if err != nil {
return make(map[int]string), 0, err
}

return mapTitles, more, nil
}

func prepareRequest(endpointBase string, page int, AuthorID int) (*http.Request, error) {
/*
* Here, 'u' is a url object.
*/
u, err := url.Parse(endpointBase)
if err != nil {
return nil, err
}

/*
* Here, we make a query 'q' out of url object 'u'
*/
q := u.Query()
q.Set("key", `kDkKnUxiz8cRBJhVjrtSA`)
q.Set("id", strconv.Itoa(AuthorID))
q.Set("page", strconv.Itoa(page))

/*
* Here, 's' is our fully constructed URL string
*/
u.RawQuery = q.Encode()
s := u.String()
fmt.Println(s)

req, err := http.NewRequest("GET", s, nil)
if err != nil {
return nil, err
}

return req, nil
}

func doRequest(req *http.Request) (*http.Response, error) {
client := &http.Client{}

resp, err := client.Do(req)
if err != nil {
return nil, err
}

return resp, nil
}

func parseResponse(resp *http.Response) (map[int]string, int, error) {
requestBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return make(map[int]string), 0, err
}

var graq GoodReadsAuthorQuery

err = xml.Unmarshal(requestBytes, &graq)
if err != nil {
return make(map[int]string), 0, err
}

var requestTitles = make(map[int]string)

for key, bookValue := range graq.Author.Books.Book {
requestTitles[key] = bookValue.Title
}

more, err := checkForMore(&graq)
if err != nil {
return make(map[int]string), 0, err
}

return requestTitles, more, nil
}

func checkForMore(graq *GoodReadsAuthorQuery) (int, error) {
var start, end, total int

start = graq.Author.Books.Start
end = graq.Author.Books.End
total = graq.Author.Books.Total

if total != end {
return (total-end)/(end-start+1) + 1, nil
}

return 0, nil
}

0 comments on commit 5da28cc

Please sign in to comment.