added github stats pull and requirements.txt

hindupuravinash · Sep 8, 2018 · feb0ced · feb0ced
1 parent e952476
commit feb0ced
Show file tree

Hide file tree

Showing 8 changed files with 991 additions and 466 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+    "python.pythonPath": "/Users/davidfoster/.virtualenvs/webscraper/bin/python"
+}
diff --git a/cumulative_gans.jpg b/cumulative_gans.jpg
diff --git a/gans copy.tsv b/gans copy.tsv
diff --git a/gans.tsv b/gans.tsv
diff --git a/github_scraper.py b/github_scraper.py
@@ -0,0 +1,6 @@
+
+
+
+print(watch)
+print(start)
+print(fork)
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,25 @@
+astroid==2.0.4
+beautifulsoup4==4.6.3
+bs4==0.0.1
+certifi==2018.8.24
+chardet==3.0.4
+cycler==0.10.0
+idna==2.7
+isort==4.3.4
+Jinja2==2.10
+kiwisolver==1.0.1
+lazy-object-proxy==1.3.1
+MarkupSafe==1.0
+matplotlib==2.2.3
+mccabe==0.6.1
+numpy==1.15.1
+Pillow==5.2.0
+pylint==2.1.1
+pyparsing==2.2.0
+python-dateutil==2.7.3
+pytz==2018.5
+requests==2.19.1
+six==1.11.0
+typed-ast==1.1.0
+urllib3==1.23
+wrapt==1.10.11
diff --git a/update.py b/update.py
@@ -5,11 +5,13 @@
 
 import numpy as np
 import matplotlib.pyplot as plt
+from bs4 import BeautifulSoup
+import requests
+import csv
 
 
 def load_data():
     """ Load GANs data from the gans.csv file """
-    import csv
 
     with open('gans.tsv') as fid:
         reader = csv.DictReader(fid, delimiter='\t')
@@ -45,8 +47,32 @@ def update_figure(gans):
         plt.ylabel("Total number of papers")
         plt.savefig('cumulative_gans.jpg')
 
+def update_github_stats(gans):
+    """ Update Github stats """
+    num_rows = len(gans)
+    print('Fetching Github stats...')
+    for i, gan in enumerate(gans):
+        url = gan['Official_Code']
+        if url != "-" and url != "":
+            print(str(i) + '/' + str(num_rows))
+            result = requests.get(url)
+            c = result.text
+            soup = BeautifulSoup(c, "html.parser")
+            samples = soup.select("a.social-count")
+            gan['Watches'] = samples[0].get_text().strip().replace(",", "")
+            gan['Stars'] = samples[1].get_text().strip().replace(",", "")
+            gan['Forks'] = samples[2].get_text().strip().replace(",", "")
+
+    print(str(i) + '/' + str(num_rows))
+    print('Complete.')
+
+    with open('gans.tsv', 'w') as outfile:
+        fp = csv.DictWriter(outfile, gans[0].keys(), delimiter='\t')
+        fp.writeheader()
+        fp.writerows(gans)
 
 if __name__ == '__main__':
     GANS = load_data()
     update_readme(GANS)
     update_figure(GANS)
+    update_github_stats(GANS)