diff --git a/fetch.py b/fetch.py index d8c57b8ff..3e7b17281 100644 --- a/fetch.py +++ b/fetch.py @@ -37,28 +37,9 @@ def fetch_orgs(): # Save details about each repo for an org all_repos = defaultdict(list) for organisation in organisations: - repos = defaultdict(list) print("Fetching repos for: ", organisation) - for repo in repos_for_org(organisation): - repos["nom"].append(repo["name"]) - repos["organisation_nom"].append(repo["owner"]["login"]) - repos["plateforme"].append("GitHub") - repos["repertoire_url"].append(repo["html_url"]) - repos["description"].append(repo["description"]) - repos["est_fork"].append(repo["fork"]) - repos["date_creation"].append(repo["created_at"]) - repos["derniere_mise_a_jour"].append(repo["updated_at"]) - repos["page_accueil"].append(repo["homepage"]) - repos["nombre_stars"].append(repo["stargazers_count"]) - repos["nombre_forks"].append(repo["forks_count"]) - try: - repos["licence"].append(repo["license"]["name"]) - except Exception as e: - repos["licence"].append(None) - repos["nombre_issues_ouvertes"].append(repo["open_issues"]) - repos["langage"].append(repo["language"]) - repos["topics"].append(",".join(repo["topics"])) + repos = repos_for_org(organisation) for k, v in repos.items(): all_repos[k].extend(v) diff --git a/github.py b/github.py index 2705bba4d..6f98109fd 100644 --- a/github.py +++ b/github.py @@ -1,9 +1,10 @@ import os import requests +from collections import defaultdict from requests.packages.urllib3.util.retry import Retry from requests.adapters import HTTPAdapter -from models import Organisation +from models import Organisation, Repository GITHUB_API_ROOT = "https://api.github.com/" @@ -40,7 +41,33 @@ def repos_for_org(organisation): response = requests.get(url, headers=github_headers()) data.extend(response.json()) - return data + repos = defaultdict(list) + for repository in data: + repo = {} + + repo["nom"] = repository["name"] + repo["organisation_nom"] = repository["owner"]["login"] + repo["plateforme"] = "GitHub" + repo["repertoire_url"] = repository["html_url"] + repo["description"] = repository["description"] + repo["est_fork"] = repository["fork"] + repo["date_creation"] = repository["created_at"] + repo["derniere_mise_a_jour"] = repository["updated_at"] + repo["page_accueil"] = repository["homepage"] + repo["nombre_stars"] = repository["stargazers_count"] + repo["nombre_forks"] = repository["forks_count"] + try: + repo["licence"] = repository["license"]["name"] + except Exception as e: + repo["licence"] = None + repo["nombre_issues_ouvertes"] = repository["open_issues"] + repo["langage"] = repository["language"] + repo["topics"] = ",".join(repository["topics"]) + + for k, v in Repository(**repo).to_dict_list().items(): + repos[k].extend(v) + + return repos def clean_data(value, key): diff --git a/models.py b/models.py index e3dd2cd2e..a24fa8ed5 100644 --- a/models.py +++ b/models.py @@ -2,8 +2,35 @@ from dataclasses import asdict as dataclasses_dict +class BaseModel(object): + def to_dict(self): + return dataclasses_dict(self) + + def to_dict_list(self): + return {k: [v] for k, v in self.to_dict().items()} + + +@dataclass(frozen=True) +class Repository(BaseModel): + nom: str + organisation_nom: str + plateforme: str + repertoire_url: str + description: str + est_fork: bool + date_creation: str + derniere_mise_a_jour: str + page_accueil: str + nombre_stars: int + nombre_forks: int + licence: str + nombre_issues_ouvertes: int + langage: str + topics: str + + @dataclass(frozen=True) -class Organisation: +class Organisation(BaseModel): login: str description: str nom: str @@ -15,9 +42,3 @@ class Organisation: nombre_repertoires: int date_creation: str plateforme: str - - def to_dict(self): - return dataclasses_dict(self) - - def to_dict_list(self): - return {k: [v] for k, v in self.to_dict().items()}