[fix] glassdoor duplicates

Bunsly · Oct 31, 2023 · 2b7fea4 · 2b7fea4
1 parent d37f86e
commit 2b7fea4
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-jobspy"
-version = "1.1.24"
+version = "1.1.25"
 description = "Job scraper for LinkedIn, Indeed, Glassdoor & ZipRecruiter"
 authors = ["Zachary Hampton <zachary@bunsly.com>", "Cullen Watson <cullen@bunsly.com>"]
 homepage = "https://github.com/Bunsly/JobSpy"

diff --git a/src/jobspy/scrapers/glassdoor/__init__.py b/src/jobspy/scrapers/glassdoor/__init__.py
@@ -78,6 +78,9 @@ def fetch_jobs_page(
             job_url = res_json["data"]["jobListings"]["jobListingSeoLinks"][
                 "linkItems"
             ][i]["url"]
+            if job_url in self.seen_urls:
+                continue
+            self.seen_urls.add(job_url)
             job = job["jobview"]
             title = job["job"]["jobTitleText"]
             company_name = job["header"]["employerNameFromSearch"]