Skip to content

Commit

Permalink
cannot use chrome driver due to permission 448 but chrome has 511
Browse files Browse the repository at this point in the history
  • Loading branch information
dcalde committed Dec 22, 2020
1 parent 3533882 commit fb18288
Showing 1 changed file with 5 additions and 14 deletions.
19 changes: 5 additions & 14 deletions scraper.py
Expand Up @@ -37,23 +37,14 @@ def clean_description(description: str) -> str:
return description.replace("\r", " ")

# can no longer use a simple request to get the page content. Need headless browser
# chrome_options = ChromeOptions()
# chrome_options.binary_location = '/usr/bin/google-chrome'
# chrome_options.add_argument('--headless')
# chrome_options.add_argument('--disable-gpu')
chrome_options = ChromeOptions()
chrome_options.binary_location = '/usr/bin/google-chrome'
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
# chrome_options.add_argument('--disable-dev-shm-usage')
# chrome_options.add_argument('--no-sandbox')

# debugging only
print("chromedriver: " + str(stat.S_IMODE(os.lstat('/usr/local/bin/chromedriver').st_mode)))
print("chrome: " + str(stat.S_IMODE(os.lstat('/usr/bin/google-chrome').st_mode)))
os.chmod('/usr/local/bin/chromedriver', 0o755)
os.chmod('/usr/bin/google-chrome', 0o755)
print("chromedriver: " + str(stat.S_IMODE(os.lstat('/usr/local/bin/chromedriver').st_mode)))
print("chrome: " + str(stat.S_IMODE(os.lstat('/usr/bin/google-chrome').st_mode)))

with Browser('chrome', executable_path='/usr/local/bin/chromedriver', headless=True) as browser:
# with Browser('chrome', options=chrome_options) as browser:
with Browser('chrome', options=chrome_options) as browser:
browser.visit(URL)
links = browser.find_by_css(".list-item > a")
for link in links:
Expand Down

0 comments on commit fb18288

Please sign in to comment.