Add dataclass for a repository

fetch.py

@@ -37,28 +37,9 @@ def fetch_orgs():
 # Save details about each repo for an org
 all_repos = defaultdict(list)
 for organisation in organisations:
-    repos = defaultdict(list)
     print("Fetching repos for: ", organisation)
 
-    for repo in repos_for_org(organisation):
-        repos["nom"].append(repo["name"])
-        repos["organisation_nom"].append(repo["owner"]["login"])
-        repos["plateforme"].append("GitHub")
-        repos["repertoire_url"].append(repo["html_url"])
-        repos["description"].append(repo["description"])
-        repos["est_fork"].append(repo["fork"])
-        repos["date_creation"].append(repo["created_at"])
-        repos["derniere_mise_a_jour"].append(repo["updated_at"])
-        repos["page_accueil"].append(repo["homepage"])
-        repos["nombre_stars"].append(repo["stargazers_count"])
-        repos["nombre_forks"].append(repo["forks_count"])
-        try:
-            repos["licence"].append(repo["license"]["name"])
-        except Exception as e:
-            repos["licence"].append(None)
-        repos["nombre_issues_ouvertes"].append(repo["open_issues"])
-        repos["langage"].append(repo["language"])
-        repos["topics"].append(",".join(repo["topics"]))
+    repos = repos_for_org(organisation)
 
     for k, v in repos.items():
         all_repos[k].extend(v)

github.py

@@ -1,9 +1,10 @@
 import os
 import requests
+from collections import defaultdict
 from requests.packages.urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter
 
-from models import Organisation
+from models import Organisation, Repository
 
 GITHUB_API_ROOT = "https://api.github.com/"
 
@@ -40,7 +41,33 @@ def repos_for_org(organisation):
         response = requests.get(url, headers=github_headers())
         data.extend(response.json())
 
-    return data
+    repos = defaultdict(list)
+    for repository in data:
+        repo = {}
+
+        repo["nom"] = repository["name"]
+        repo["organisation_nom"] = repository["owner"]["login"]
+        repo["plateforme"] = "GitHub"
+        repo["repertoire_url"] = repository["html_url"]
+        repo["description"] = repository["description"]
+        repo["est_fork"] = repository["fork"]
+        repo["date_creation"] = repository["created_at"]
+        repo["derniere_mise_a_jour"] = repository["updated_at"]
+        repo["page_accueil"] = repository["homepage"]
+        repo["nombre_stars"] = repository["stargazers_count"]
+        repo["nombre_forks"] = repository["forks_count"]
+        try:
+            repo["licence"] = repository["license"]["name"]
+        except Exception as e:
+            repo["licence"] = None
+        repo["nombre_issues_ouvertes"] = repository["open_issues"]
+        repo["langage"] = repository["language"]
+        repo["topics"] = ",".join(repository["topics"])
+
+        for k, v in Repository(**repo).to_dict_list().items():
+            repos[k].extend(v)
+
+    return repos
 
 
 def clean_data(value, key):

models.py

@@ -2,8 +2,35 @@
 from dataclasses import asdict as dataclasses_dict
 
 
+class BaseModel(object):
+    def to_dict(self):
+        return dataclasses_dict(self)
+
+    def to_dict_list(self):
+        return {k: [v] for k, v in self.to_dict().items()}
+
+
+@dataclass(frozen=True)
+class Repository(BaseModel):
+    nom: str
+    organisation_nom: str
+    plateforme: str
+    repertoire_url: str
+    description: str
+    est_fork: bool
+    date_creation: str
+    derniere_mise_a_jour: str
+    page_accueil: str
+    nombre_stars: int
+    nombre_forks: int
+    licence: str
+    nombre_issues_ouvertes: int
+    langage: str
+    topics: str
+
+
 @dataclass(frozen=True)
-class Organisation:
+class Organisation(BaseModel):
     login: str
     description: str
     nom: str
@@ -15,9 +42,3 @@ class Organisation:
     nombre_repertoires: int
     date_creation: str
     plateforme: str
-
-    def to_dict(self):
-        return dataclasses_dict(self)
-
-    def to_dict_list(self):
-        return {k: [v] for k, v in self.to_dict().items()}