add scraper.yaml

.github/workflows/cicd.yaml

@@ -2,9 +2,9 @@ name: Run CI/CD pipeline
 
 on:
   push:
-    branches: [ "main" ]
-    # branches-ignore:    
-    #   - "main"
+    # branches: [ "main" ]
+    branches-ignore:    
+      - "main"
   pull_request:
     branches: [ "main" ]
   # allow manual trigger  

.github/workflows/scraper.yaml

@@ -0,0 +1,32 @@
+name: Run scraper and update SQLite database
+
+on:
+  push:
+    branches: [ "main" ]
+  schedule:
+    - cron: "45 02 * * SAT"  # at 02:45 every Saturday
+
+permissions:
+  actions: write
+  contents: write
+
+jobs:
+  scrape:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.9"
+    - name: Install dependencies
+      run: |
+           python -m pip install --upgrade pip
+           pip install -r scraper/requirements.txt
+    - name: Scrape data
+      run: make scrape
+    - name: Commit changes
+      run: |
+           git config --local user.name ${{ secrets.USER_NAME }}
+           git config --local user.email ${{ secrets.EMAIL }}
+           make push message="Scrape baby scrape"

scraper/config/config.json

@@ -1,7 +1,8 @@
 {
     "url_base": "https://www.lzvcup.be",
     "steps": {
-        "historical_players": true
+        "historical_players": true,
+        "csv": true
     },
     "areas": {
         "ANTWERPEN": "results/1",
@@ -22,5 +23,6 @@
         "standings": [["area", "region", "competition", "team"], []],
         "palmares": [["team", "seizoen", "reeks", "positie"], ["area", "region", "competition"]]
     },
-    "database": "database/futsalfriend.db"
+    "database": "database/futsalfriend.db",
+    "dir_last_updated_date": "webapp/last_updated_date.txt"
 }

scraper/main.py

@@ -10,7 +10,13 @@
 from scraper.parsers.lzvcup import LZVCupParser
 from scraper.utils.base import DataStorage
 from scraper.utils.logger import Logger
-from scraper.utils.utils import add_coordinates, create_levels_table, postproces_df, ymd
+from scraper.utils.utils import (
+    add_coordinates,
+    create_levels_table,
+    postproces_df,
+    write_current_date_to_file,
+    ymd,
+)
 
 DIR_LOGS = f"{DIR_SCRIPT}/logs/{ymd()}/"  # subdivide logs by day of script execution
 
@@ -130,11 +136,11 @@ def process_data(config, dict_tables, log_main):
     return dict_tables
 
 
-def store(config, dict_tables, log_main, csv=True):
+def store(config, dict_tables, log_main):
     # refresh SQLite database
     refresh_database(dict_tables, path2db=config["database"], logger=log_main)
 
-    if csv:
+    if config["steps"]["csv"]:
         # additionally store all tables as csv files
         root = config["dir_output"]
         for data_name, data in dict_tables.items():
@@ -163,3 +169,6 @@ def store(config, dict_tables, log_main, csv=True):
 
     store(config, dict_tables, log_main=log)
     log.info("Data stored")
+
+    write_current_date_to_file(config["dir_last_updated_date"])
+    log.info("Refresh date updated.")

scraper/requirements.txt

@@ -0,0 +1,7 @@
+pandas==2.1.1
+requests==2.31.0
+beautifulsoup4==4.12.2
+lxml==4.9.3
+structlog==23.2.0
+sqlalchemy==2.0.22
+geopy==2.4.0

scraper/utils/utils.py

@@ -18,6 +18,12 @@ def ymdhms():
     return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
 
+def write_current_date_to_file(dir):
+    """Writes away current date as DD-MM-YYYY in a text file."""
+    with open(dir, "w") as f:
+        f.write(datetime.now().strftime("%d-%m-%Y"))
+
+
 def chunks(lst, n):
     """Yields successive n-sized chunks from input list."""
     for i in range(0, len(lst), n):

webapp/01_Home.py

@@ -1,13 +1,18 @@
 import streamlit as st
-from utils import add_socials_to_navbar
+from utils import add_socials_to_sidebar
 
 st.set_page_config(page_title="Futsal Friend", page_icon="⚽", layout="wide")
 
 ##################
 ########## UI   ##
 ##################
 
-add_socials_to_navbar()
+add_socials_to_sidebar()
+
+with st.sidebar:
+    with open("webapp/last_updated_date.txt", "r") as f:
+        last_updated = f.read()
+    st.markdown(f"**Last updated:** {last_updated}")
 
 st.markdown(
     """# ⚽ Futsal Friend <span style=color:#030080><font size=4>Beta</font></span>""",
@@ -22,7 +27,7 @@
 with st.expander("OK but what can I do with it?"):
     st.markdown(
         """
-        There are four things you can do: (1) scout a potential **opponent** for a 
+        There are four things you can do: (1) score a potential **opponent** for a 
         friendly match, (2) find a **team** to join as a player, (3) compare your
         game **stats** to those of others, and (4) get jolly **advice** for your
         team or yourself as a player. Check out the separate pages in the sidebar

webapp/last_updated_date.txt

@@ -0,0 +1 @@
+25-10-2023

webapp/pages/02_🏆_Friendly_Finder.py

@@ -15,7 +15,7 @@
 ##################
 
 st.title("Friendly Finder")
-st.markdown("### Scout an opponent for a friendly")
+st.markdown("### Score an opponent for a friendly")
 
 # ask for inputs
 levels = {"Courtois 💪💪💪": 1, "Casteels 💪💪": 2, "Mignolet 💪": 3}

webapp/utils.py

@@ -91,7 +91,7 @@ def style_table(df, drop_cols=[]):
     return df
 
 
-def add_socials_to_navbar():
+def add_socials_to_sidebar():
     """Adds social media buttons to the Streamlit sidebar."""
     st.markdown(
         """