pagination: updated pagination to give deterministic and natural orde…

…ring (#1070)

do/pagination.go

@@ -29,15 +29,16 @@ var perPage = 200
 var fetchFn = fetchPage
 
 type paginatedList struct {
-	list []interface{}
-	mu   sync.Mutex
+	list  [][]interface{}
+	total int
+	mu    sync.Mutex
 }
 
-func (pl *paginatedList) append(items ...interface{}) {
+func (pl *paginatedList) set(page int, items []interface{}) {
 	pl.mu.Lock()
 	defer pl.mu.Unlock()
-
-	pl.list = append(pl.list, items...)
+	pl.total += len(items)
+	pl.list[page-1] = items
 }
 
 // Generator is a function that generates the list to be paginated.
@@ -47,7 +48,24 @@ type Generator func(*godo.ListOptions) ([]interface{}, *godo.Response, error)
 func PaginateResp(gen Generator) ([]interface{}, error) {
 	opt := &godo.ListOptions{Page: 1, PerPage: perPage}
 
-	l := paginatedList{}
+	// fetch first page to get page count (x)
+	firstPage, resp, err := gen(opt)
+	if err != nil {
+		return nil, err
+	}
+
+	// find last page
+	lp, err := lastPage(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	l := paginatedList{
+		list: make([][]interface{}, lp),
+	}
+
+	// set results from the first page
+	l.set(1, firstPage)
 
 	fetchChan := make(chan int, maxFetchPages)
 
@@ -58,27 +76,13 @@ func PaginateResp(gen Generator) ([]interface{}, error) {
 			for page := range fetchChan {
 				items, err := fetchFn(gen, page)
 				if err == nil {
-					l.append(items...)
+					l.set(page, items)
 				}
 			}
 			wg.Done()
 		}()
 	}
 
-	// fetch first page to get page count (x)
-	items, resp, err := gen(opt)
-	if err != nil {
-		return nil, err
-	}
-
-	l.append(items...)
-
-	// find last page
-	lp, err := lastPage(resp)
-	if err != nil {
-		return nil, err
-	}
-
 	// start with second page
 	opt.Page++
 	for ; opt.Page <= lp; opt.Page++ {
@@ -88,7 +92,19 @@ func PaginateResp(gen Generator) ([]interface{}, error) {
 
 	wg.Wait()
 
-	return l.list, nil
+	// flatten paginated list
+	items := make([]interface{}, l.total)[:0]
+	for _, page := range l.list {
+		if page == nil {
+			// must have been an error getting page results
+			continue
+		}
+		for _, item := range page {
+			items = append(items, item)
+		}
+	}
+
+	return items, nil
 }
 
 func fetchPage(gen Generator, page int) ([]interface{}, error) {