waitForNavigation doesn't work after clicking a link

[zhanghaofei]

# coding:utf-8
import asyncio
from pyppeteer import launch

async def main():
    brower = await launch(headless=False)
    page = await brower.newPage()
    await page.setViewport(dict(width=1200, height=1000))
    await page.goto("https://github.com")
    await page.click('.HeaderMenu [href="/features"]')
    await page.waitForNavigation()

asyncio.get_event_loop().run_until_complete(main())

https://github.com/features The page has been loaded but the navigation timed out

Traceback (most recent call last):
  File "D:/code/py3/pyppeteer/hello.py", line 42, in <module>
    asyncio.get_event_loop().run_until_complete(main())
  File "F:\python3\Lib\asyncio\base_events.py", line 467, in run_until_complete
    return future.result()
  File "D:/code/py3/pyppeteer/hello.py", line 14, in main
    await page.waitForNavigation()
  File "D:\code\py3\.venv\lib\site-packages\pyppeteer\page.py", line 698, in waitForNavigation
    raise error
pyppeteer.errors.TimeoutError: Navigation Timeout Exceeded: 30000 ms exceeded.

puppeteer solution

https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#pageclickselector-options

const [response] = await Promise.all([
  page.waitForNavigation(waitOptions),
  page.click(selector, clickOptions),
]);

But how does python solve?

[zhanghaofei]

Seems to have found a way to achieve, but I do not understand, do not know whether the correct, I hope to get your guidance, thank you
solution:

await asyncio.wait([page.click('.HeaderMenu [href="/features"]'), page.waitForNavigation()])

[miyakogi]

Your solution is correct.

The problem of your first code is:

await page.goto("https://github.com")
await page.click('.HeaderMenu [href="/features"]')  # start navigation to the feature page here
await page.waitForNavigation()  # waiting for the "next" navigation from the feature page

To avoid this situation, you need to start click and waitForNavigation (almost) simultaneously, as your solution.

I will update document about page.click() by the next release.

[tonggang]

sorry guys,i found the solution still not work in my case. any idea for this?

async def main():
browser = await launch({"headless":False})
page = await browser.newPage()
await page.goto('http://wsjs.saic.gov.cn')
await page.screenshot({'path': 'example.png'})
await page.click('#txnS02',{"waitUntil":"networkidle0"})
title = await page.title()
if title is None or title =="":
await page.reload({"waitUntil":"networkidle0"})
print (await page.title() + str(len(await browser.pages())))
await page.screenshot({'path': 'example1.png'})
await page.focus("input[name='request:sn']")
await page.type("input[name='request:sn']","26716759",{"delay": 50})
await page.screenshot({'path': 'example2.png'})
await asyncio.wait([page.click('#_searchButton'), page.waitForNavigation()])
print (await page.title())
# await browser.close()
asyncio.get_event_loop().run_until_complete(main())

[miyakogi]

Since page.click does not accept waitUntil option, this line is incorrect.

await page.click('#txnS02',{"waitUntil":"networkidle0"})

So you need to use await asyncio.wait([page.click(...), page.waitForNavigation()]) here, too.

[miyakogi]

Updated document of Page.click()

[tonggang]

actually it's working for me at that line ,but still got the Navigation Timeout Exceeded: 30000 ms exceeded at line await asyncio.wait([page.click('#_searchButton'), page.waitForNavigation()])

[miyakogi]

Thank you for details.
I reproduced the error.
I will try to fix it.

[miyakogi]

I understand the problem.
Since page.click('#_searchButton') opens a new tab, navigation event is fired in the different page instance.
So you need to watch targetcreated event on the browser.

Try the below code:

import asyncio
from pyppeteer import launch

async def main() -> None:
    browser = await launch({"headless": False, "args": ['--no-sandbox']})
    page = await browser.newPage()
    await page.goto('http://wsjs.saic.gov.cn')
    await page.screenshot({'path': 'example.png'})
    await asyncio.wait([page.waitForNavigation(), page.click('#txnS02')])

    title = await page.title()
    if title is None or title == "":
        await page.reload({"waitUntil": "networkidle0"})

    print(await page.title() + str(len(await browser.pages())))
    await page.screenshot({'path': 'example1.png'})
    await page.focus("input[name='request:sn']")
    await page.type("input[name='request:sn']", "26716759", {"delay": 50})
    await page.screenshot({'path': 'example2.png'})

    result_page_future = asyncio.get_event_loop().create_future()
    browser.once('targetcreated',
                 lambda target: result_page_future.set_result(target))
    await page.click('#_searchButton'),
    result_page = await (await result_page_future).page()
    print(await result_page.title())
    # await browser.close()

asyncio.get_event_loop().run_until_complete(main())

output is:

商标综合检索2
商标检索结果

[tonggang]

hi bro, but seems i'm not able to click the link inside the table. Means if this page is dynamic loading how can i access the content?Thanks

import asyncio
from pyppeteer import launch

async def main() -> None:
browser = await launch({"headless": False, "args": ['--no-sandbox']})
page = await browser.newPage()
await page.goto('http://wsjs.saic.gov.cn')
await page.screenshot({'path': 'example.png'})
await asyncio.wait([page.waitForNavigation(), page.click('#txnS02')])

title = await page.title()
if title is None or title == "":
    await page.reload({"waitUntil": "networkidle0"})

print(await page.title() + str(len(await browser.pages())))
await page.screenshot({'path': 'example1.png'})
await page.focus("input[name='request:sn']")
await page.type("input[name='request:sn']", "26716759", {"delay": 50})


result_page_future = asyncio.get_event_loop().create_future()
browser.once('targetcreated',
             lambda target: result_page_future.set_result(target))
await page.click('#_searchButton')
result_page = await (await result_page_future).page()

print(await result_page.title())
await result_page.click("a.ng-binding")  #it does not work

await browser.close()

asyncio.get_event_loop().run_until_complete(main())

[miyakogi]

You need to wait for page contents loaded.
Use waitForSelector method before click.

print(await result_page.title())
await result_page.waitForSelector('a.ng-binding')
await result_page.click('a.ng-binding')

[tonggang]

Thank u bro!!

[SIMPNDEV]

Hi @miyakogi , could you give this post https://stackoverflow.com/questions/60763644/cant-go-on-clicking-on-the-next-page-button-while-scraping-certain-fields-from a shot in case there is any solution you would like to offer. The post that I created in there is about pyppeteer. Thanks.