Looping through pages when next is available

[SantoshSrinivas79]

Hi All,

What would be the best way to keep looping the browsing while a "next" link is available and then return the result to the main process.

The below example returns the first search result for the first page. How to result all first links from each search page?

// Run using node --harmony yahoo.js

var Nightmare = require('nightmare');
var vo = require('vo');

vo(function* () {
  var nightmare = Nightmare({ show: true });
  var link = yield nightmare
    .goto('http://yahoo.com')
    .type('input[title="Search"]', 'github nightmare')
    .click('#UHSearchWeb')
    .wait('a.next')
    .evaluate(function () {
      var links = document.querySelectorAll("a.td-u");
      return links[0].href;
    });
  yield nightmare.end();
  return link;
})(function (err, result) {
  if (err) return console.log(err);
  console.log(result);
});

[rosshinkley]

A hastily thrown together sample:

var Nightmare = require('nightmare');                                                                                                                                                                      
var vo = require('vo');                                                                                                                                                                                    

vo(run)(function(err, result) {                                                                                                                                                                            
    if (err) throw err;                                                                                                                                                                                    
});                                                                                                                                                                                                        

function* run() {                                                                                                                                                                                          
    var nightmare = Nightmare(),                                                                                                                                                                           
        MAX_PAGE = 10,                                                                                                                                                                                     
        currentPage = 0,                                                                                                                                                                                   
        nextExists = true,                                                                                                                                                                                 
        links = [];                                                                                                                                                                                        


    yield nightmare                                                                                                                                                                                        
        .goto('https://www.yahoo.com')                                                                                                                                                                     
        .type('.input-query', 'github nightmare')                                                                                                                                                          
        .click('#search-submit')                                                                                                                                                                           
        .wait('body')                                                                                                                                                                                      

    nextExists = yield nightmare.visible('.next');                                                                                                                                                         
    while (nextExists && currentPage < MAX_PAGE) {                                                                                                                                                         
        links.push(yield nightmare                                                                                                                                                                         
            .evaluate(function() {                                                                                                                                                                         
                var links = document.querySelectorAll("ol.searchCenterMiddle a");                                                                                                                          
                return links[0].href;                                                                                                                                                                      
            }));                                                                                                                                                                                           

        yield nightmare                                                                                                                                                                                    
            .click('.next')                                                                                                                                                                                
            .wait('body')                                                                                                                                                                                  

        currentPage++;                                                                                                                                                                                     
        nextExists = yield nightmare.visible('.next');                                                                                                                                                     
    }                                                                                                                                                                                                      

    console.dir(links);                                                                                                                                                                                    
    yield nightmare.end();                                                                                                                                                                                 
}                                                                                                                                                                                                          

You could, of course, remove the MAX_PAGE guard, but if you're searching something more popular, the script would take much longer to complete.

[SantoshSrinivas79]

Thank you @rosshinkley

[misbach]

@rosshinkley I can't get your script to run. It gives the following error: "Cannot read property 'focus' of null". Does it still run for you?

[misbach]

Nevermind, I just had to update the script a bit.

vo(run)(function(err, result) {
  if (err) throw err;
});

function* run() {
  var MAX_PAGE = 2;
  var currentPage = 0;
  var nextExists = true;
  var links = [];

  yield nightmare
    .goto('https://www.yahoo.com')
    .type('input#uh-search-box', 'bitcoin')
    .click('button#uh-search-button')
    .wait('div#main')

  nextExists = yield nightmare.visible('.next');

  while (nextExists && currentPage < MAX_PAGE) {
    links.push(yield nightmare
      .evaluate(function() {
        return document.querySelectorAll(".lh-24")[0].href;
      }));

    yield nightmare
      .click('.next')
      .wait('body')

    currentPage++;
    nextExists = yield nightmare.visible('.next');
  }

  console.dir(links);
  yield nightmare.end();
}

[Macxim]

@misbach I'm still missing the last page, even with the updated script.

[globalkonvict]

This was helpful Thanks a lot.