Skip to content

Add pagination to api_process.py #37

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 17, 2025
Merged

Add pagination to api_process.py #37

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions api_process.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@
# and produces a new file with only the keys (and associated values) that are
# used for building the website.
#
# usage: python api_process.py
# usage: python3 api_process.py

import json
import urllib.request
from datetime import datetime
from datetime import datetime, UTC

REQUIRED_KEYS = {
'description',
Expand All @@ -22,13 +22,22 @@
'watchers_count',
}

GITHUB_LINKEDIN_REPO_URL = 'https://api.github.com/orgs/LinkedIn/repos?page=1&per_page=100'
with urllib.request.urlopen(GITHUB_LINKEDIN_REPO_URL) as response:
gh_data = json.loads(response.read().decode('utf-8'))
GITHUB_LINKEDIN_REPO_URL = 'https://api.github.com/orgs/LinkedIn/repos'
GITHUB_LINKEDIN_REPO_URL_QUERY = f'{GITHUB_LINKEDIN_REPO_URL}?page={{page}}&per_page=100'
all_repos = []
page = 1
while True:
with urllib.request.urlopen(GITHUB_LINKEDIN_REPO_URL_QUERY.format(page=page)) as response:
print(f'Fetching {GITHUB_LINKEDIN_REPO_URL} page {page}...')
gh_data = json.loads(response.read().decode('utf-8'))
if not gh_data:
break
all_repos.extend(gh_data)
page += 1

filtered_repos = list()

for repo in gh_data:
for repo in all_repos:
filtered_repo = dict()
# Skip forked repos, as they are not directly owned by LinkedIn
if repo.get('fork', False):
Expand All @@ -40,6 +49,6 @@

# Write the data out in the desired format.
with open('js/cached-github-api-response.js', 'w+') as f:
f.write(f'// Generated from {GITHUB_LINKEDIN_REPO_URL} on {datetime.utcnow().replace(microsecond=0).isoformat()}\n')
f.write(f'// Generated from {GITHUB_LINKEDIN_REPO_URL} on {datetime.now(UTC).date().isoformat()}\n')
f.write('var cachedGithubApiResponse = ')
json.dump(filtered_repos, f)