Change API to SofaScore

.gitignore

@@ -129,3 +129,6 @@ dmypy.json
 
 # Local Scripts
 /local/
+
+# Media folder
+/media/

matches/goals_populator.py

@@ -1,6 +1,7 @@
 import json
 import operator
 import re
+import time
 from datetime import date, timedelta
 from functools import reduce
 
@@ -15,6 +16,8 @@
 def fetch_videogoals():
     print('Fetching new goals')
     _fetch_reddit_goals()
+    # How to get historic data
+    # _fetch_reddit_goals_from_date(days_ago=30)
 
 
 def _fetch_reddit_goals():
@@ -38,6 +41,23 @@ def _fetch_reddit_goals():
     print('Finished fetching goals')
 
 
+def _fetch_reddit_goals_from_date(days_ago=2):
+    start_date = date.today() - timedelta(days=days_ago)
+    for single_date in (start_date + timedelta(n) for n in range(days_ago + 1)):
+        response = _fetch_historic_data_from_reddit_api(single_date)
+        data = json.loads(response.content)
+        if 'data' not in data.keys():
+            print(f'No data in response: {response.content}')
+            return
+        results = len(data['data'])
+        print(f'{results} posts fetched...')
+        for post in data['data']:
+            if post['url'] is not None and 'Thread' not in post['title'] and 'reddit.com' not in post['url']:
+                title = post['title']
+                find_and_store_match(post, title)
+    print('Finished fetching goals')
+
+
 def find_and_store_match(post, title):
     home_team, away_team, minute_str = extract_names_from_title(title)
     if home_team is None or away_team is None:
@@ -114,3 +134,16 @@ def _fetch_data_from_reddit_api(after):
     response = requests.get(f'http://api.reddit.com/r/soccer/new?limit=100&after={after}',
                             headers=headers)
     return response
+
+
+def _fetch_historic_data_from_reddit_api(from_date):
+    after = int(time.mktime(from_date.timetuple()))
+    before = int(after + 86400)  # a day
+    headers = {
+        "User-agent": "Goals Populator 0.1"
+    }
+    response = requests.get(
+        f'https://api.pushshift.io/reddit/search/submission/'
+        f'?subreddit=soccer&sort=desc&sort_type=created_utc&after={after}&before={before}&size=1000',
+        headers=headers)
+    return response

matches/matches_populator.py

@@ -8,18 +8,24 @@
 from .models import Match, Team
 
 
-@background(schedule=60 * 60)
+@background(schedule=60 * 5)
 def fetch_new_matches():
     print('Fetching new matches...')
-    start_date = date.today() - timedelta(days=2)
-    for single_date in (start_date + timedelta(n) for n in range(3)):
-        response = _fetch_data_from_api(single_date)
+    fetch_matches_from_sofascore()
+    # How to get historic data
+    # fetch_matches_from_sofascore(days_ago=30)
+
+
+def fetch_matches_from_rapidapi(days_ago=2):
+    start_date = date.today() - timedelta(days=days_ago)
+    for single_date in (start_date + timedelta(n) for n in range(days_ago + 1)):
+        response = _fetch_data_from_rapidpi_api(single_date)
         data = json.loads(response.content)
         results = data['api']['results']
         print(f'{results} matches fetched...')
         for fixture in data['api']['fixtures']:
-            home_team = _get_or_create_home_team(fixture)
-            away_team = _get_or_create_away_team(fixture)
+            home_team = _get_or_create_home_team_rapidapi(fixture)
+            away_team = _get_or_create_away_team_rapidapi(fixture)
             home_goals = fixture['goalsHomeTeam']
             away_goals = fixture['goalsAwayTeam']
             score = None
@@ -36,39 +42,71 @@ def fetch_new_matches():
             _save_or_update_match(match)
         print(f'Ended processing day {single_date}')
     print('Ended processing matches')
-    pass
 
 
-def _delete_legacy_team(team):
-    legacy_teams = Team.objects.filter(name__exact=team.name, id__gte=9990000)
-    if legacy_teams.exists():
-        for legacy_team in legacy_teams:
-            matches_home = Match.objects.filter(home_team=legacy_team)
-            matches_home.update(home_team=team)
-            matches_away = Match.objects.filter(away_team=legacy_team)
-            matches_away.update(away_team=team)
-            legacy_team.delete()
+def fetch_matches_from_sofascore(days_ago=0):
+    start_date = date.today() - timedelta(days=days_ago)
+    for single_date in (start_date + timedelta(n) for n in range(days_ago + 1)):
+        response = _fetch_data_from_sofascore_api(single_date)
+        data = json.loads(response.content)
+        for tournament in data['sportItem']['tournaments']:
+            for fixture in tournament["events"]:
+                home_team = _get_or_create_home_team_sofascore(fixture)
+                away_team = _get_or_create_away_team_sofascore(fixture)
+                score = None
+                if 'display' in fixture['homeScore'] and 'display' in fixture['awayScore']:
+                    home_goals = fixture['homeScore']['display']
+                    away_goals = fixture['awayScore']['display']
+                    if home_goals is not None and away_goals is not None:
+                        score = f'{home_goals}:{away_goals}'
+                start_timestamp = fixture["startTimestamp"]
+                match_datetime = datetime.fromtimestamp(start_timestamp)
+                print(f'{home_team} - {away_team} | {score} at {match_datetime}')
+                match = Match()
+                match.home_team = home_team
+                match.away_team = away_team
+                match.score = score
+                match.datetime = match_datetime
+                _save_or_update_match(match)
+        print(f'Ended processing day {single_date}')
+    print('Ended processing matches')
 
 
-def _get_or_create_away_team(fixture):
+def _get_or_create_away_team_rapidapi(fixture):
     away_team, away_team_created = Team.objects.get_or_create(id=fixture['awayTeam']['team_id'])
     away_team.name = fixture['awayTeam']['team_name']
-    away_team.logo = fixture['awayTeam']['logo']
+    away_team.logo_url = fixture['awayTeam']['logo']
     away_team.save()
-    _delete_legacy_team(away_team)
     return away_team
 
 
-def _get_or_create_home_team(fixture):
+def _get_or_create_away_team_sofascore(fixture):
+    team_id = fixture['awayTeam']['id']
+    away_team, away_team_created = Team.objects.get_or_create(id=team_id)
+    away_team.name = fixture['awayTeam']['name']
+    away_team.logo_url = f"https://www.sofascore.com/images/team-logo/football_{team_id}.png"
+    away_team.save()
+    return away_team
+
+
+def _get_or_create_home_team_rapidapi(fixture):
     home_team, home_team_created = Team.objects.get_or_create(id=fixture['homeTeam']['team_id'])
     home_team.name = fixture['homeTeam']['team_name']
-    home_team.logo = fixture['homeTeam']['logo']
+    home_team.logo_url = fixture['homeTeam']['logo']
     home_team.save()
-    _delete_legacy_team(home_team)
     return home_team
 
 
-def _fetch_data_from_api(single_date):
+def _get_or_create_home_team_sofascore(fixture):
+    team_id = fixture['homeTeam']['id']
+    away_team, away_team_created = Team.objects.get_or_create(id=team_id)
+    away_team.name = fixture['homeTeam']['name']
+    away_team.logo_url = f"https://www.sofascore.com/images/team-logo/football_{team_id}.png"
+    away_team.save()
+    return away_team
+
+
+def _fetch_data_from_rapidpi_api(single_date):
     today_str = single_date.strftime("%Y-%m-%d")
     headers = {
         "X-RapidAPI-Host": "api-football-v1.p.rapidapi.com",
@@ -81,6 +119,14 @@ def _fetch_data_from_api(single_date):
     return response
 
 
+def _fetch_data_from_sofascore_api(single_date):
+    today_str = single_date.strftime("%Y-%m-%d")
+    response = requests.get(
+        f'https://www.sofascore.com/football//{today_str}/json'
+    )
+    return response
+
+
 def _save_or_update_match(match):
     matches = Match.objects.filter(home_team=match.home_team,
                                    away_team=match.away_team,

matches/migrations/0011_auto_20200128_1553.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.8 on 2020-01-28 15:53
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('matches', '0010_teamalias'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='team',
+            old_name='logo',
+            new_name='logo_url',
+        ),
+    ]

matches/migrations/0012_team_logo_file.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.8 on 2020-01-28 15:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('matches', '0011_auto_20200128_1553'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='team',
+            name='logo_file',
+            field=models.ImageField(default=None, upload_to='logos'),
+        ),
+    ]

matches/migrations/0013_auto_20200128_1725.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.8 on 2020-01-28 17:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('matches', '0012_team_logo_file'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='team',
+            name='logo_file',
+            field=models.ImageField(default=None, upload_to='media/logos'),
+        ),
+    ]

matches/models.py

@@ -1,3 +1,7 @@
+import os
+from urllib import request
+
+from django.core.files import File
 from django.db import models
 from django.urls import reverse
 from django.utils.text import slugify
@@ -6,11 +10,21 @@
 class Team(models.Model):
     id = models.IntegerField(unique=True, primary_key=True)
     name = models.CharField(max_length=256)
-    logo = models.CharField(max_length=256)
+    logo_url = models.CharField(max_length=256)
+    logo_file = models.ImageField(upload_to='media/logos', default=None)
 
     def __str__(self):
         return self.name
 
+    def save(self, *args, **kwargs):
+        if self.logo_url and not self.logo_file:
+            result = request.urlretrieve(self.logo_url)
+            self.logo_file.save(
+                os.path.basename(self.logo_url),
+                File(open(result[0], 'rb'))
+            )
+        super().save(*args, **kwargs)
+
 
 class TeamAlias(models.Model):
     alias = models.CharField(max_length=256)

matches/templates/matches/match_list.html

@@ -13,13 +13,13 @@ <h2>{{ date|date:"l, d F Y" }}</h2>
                     {% for match in match_list %}
                         <a class="list-group-item list-group-item-action"
                            href="{% url 'match-detail' slug=match.slug %}">
-                            {% if match.home_team.logo %}
-                                <img src="{{ match.home_team.logo }}" alt="{{ match.home_team.name }}"
+                            {% if match.home_team.logo_file %}
+                                <img src="{{ match.home_team.logo_file }}" alt="{{ match.home_team.name }}"
                                      class="img-fluid list-img-thumb"/>
                             {% endif %}
                             <b>{{ match }}</b>
-                            {% if match.home_team.logo %}
-                                <img src="{{ match.away_team.logo }}" alt="{{ match.away_team.name }}"
+                            {% if match.home_team.logo_file %}
+                                <img src="{{ match.away_team.logo_file }}" alt="{{ match.away_team.name }}"
                                      class="img-fluid list-img-thumb"/>
                             {% endif %}
                             at {{ match.datetime|date:"H:i" }}

requirements.txt

@@ -7,16 +7,23 @@ django-background-tasks==1.2.0
 django-compat==1.0.15
 django-heroku==0.3.1
 geoip2==2.9.0
+gitdb2==2.0.6
+GitPython==2.1.1
 gunicorn==20.0.0
 idna==2.8
 ipaddress==1.0.23
 ipapi==0.5.2
 maxminddb==1.5.1
+pbs==0.110
+Pillow==7.0.0
 psycopg2==2.8.4
 psycopg2-binary==2.8.4
 pytz==2019.3
 requests==2.22.0
+scandir==1.10.0
+sh==1.12.14
 six==1.13.0
+smmap2==2.0.5
 sqlparse==0.3.0
 urllib3==1.25.7
 whitenoise==4.1.4

videogoals/settings.py

@@ -125,11 +125,15 @@
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/2.2/howto/static-files/
 
-STATIC_URL = '/static/'
 PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
 STATICFILES_DIRS = [os.path.join(BASE_DIR, "static")]
+
+STATIC_URL = '/static/'
 STATIC_ROOT = os.environ.get('STATIC_ROOT')
 
+MEDIA_URL = '/media/'
+MEDIA_ROOT = os.path.join(BASE_DIR, "media")
+
 if os.environ.get('SECRET_KEY') is not None:
     STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.ManifestStaticFilesStorage'
 

videogoals/urls.py

@@ -1,18 +1,20 @@
+from django.conf.urls.static import static
 from django.contrib import admin
 from django.urls import include, path
 
 from background_task.models import Task
 
 from matches.goals_populator import fetch_videogoals
 from matches.matches_populator import fetch_new_matches
+from videogoals import settings
 
 urlpatterns = [
     path('admin/', admin.site.urls),
     path('', include('matches.urls'))
-]
+] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
 
 if not Task.objects.filter(verbose_name="fetch_new_matches").exists():
-    fetch_new_matches(repeat=60 * 60, repeat_until=None, verbose_name="fetch_new_matches")
+    fetch_new_matches(repeat=60 * 5, repeat_until=None, verbose_name="fetch_new_matches")
 
 if not Task.objects.filter(verbose_name="fetch_videogoals").exists():
     fetch_videogoals(repeat=60, repeat_until=None, verbose_name="fetch_videogoals")