Merge pull request #11 from carlosrsabreu/feat/plot

wrt issue #9

.github/workflows/history_plot.yml

@@ -0,0 +1,23 @@
+name: Publish history plot
+on:
+  schedule:
+    - cron: "0 0 1 * *"
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checks out the repository
+        uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup Python
+        uses: actions/setup-python@v3.1.2
+      - name: Install project requirements
+        run: pip install -r requirements.txt
+      - name: Post history plot
+        env:
+          CONSUMER_KEY: ${{ secrets.CONSUMER_KEY }}
+          CONSUMER_SECRET: ${{ secrets.CONSUMER_SECRET }}
+          ACCESS_TOKEN: ${{ secrets.ACCESS_TOKEN }}
+          ACCESS_TOKEN_SECRET: ${{ secrets.ACCESS_TOKEN_SECRET }}
+        run: python script_post_plot_history.py

.github/workflows/main.yml

@@ -33,5 +33,6 @@ jobs:
           git add gas_info.json
           git add history/gas_info_history.csv
           git add history/gas_info_history.json
+          git add history/img/plot.png
           git diff-index --quiet HEAD || git commit -m "Update gas information"
           git push

add_history.py

@@ -6,10 +6,10 @@
 def add_history(dict_prices):
     # JSON
     with open(CURRENT_GAS_HISTORY_JSON_FILE, 'r') as f:
-        curret_data = json.load(f)
+        current_data = json.load(f)
     with open(CURRENT_GAS_HISTORY_JSON_FILE, 'w') as f:
-        curret_data[dict_prices[CURRENT_WEEK][START_DATE_KEY]] = dict_prices[CURRENT_WEEK]
-        content = json.dumps(curret_data, indent=1, ensure_ascii=False)
+        current_data[dict_prices[CURRENT_WEEK][START_DATE_KEY]] = dict_prices[CURRENT_WEEK]
+        content = json.dumps(current_data, indent=1, ensure_ascii=False)
         f.write(content)
     # CSV
     with open(CURRENT_GAS_HISTORY_CSV_FILE, 'a') as f:

constants.py

@@ -27,9 +27,26 @@
 GASOLINE_95_TW = 'Gasolina 95  '
 GASOLINE_98_TW = 'Gasolina 98  '
 
-#PDF_GAS_PRICE_REGEX = r'(?<=€ )([\d,]+)(?= por litro)'
+# PDF_GAS_PRICE_REGEX = r'(?<=€ )([\d,]+)(?= por litro)'
 PDF_GAS_PRICE_REGEX = r'(%s|%s|%s)(?:[\.€\w ]+)(\d{1},\d{3})' % (
-    'Gasolina super sem chumbo IO 95',
-    'Gasóleo rodoviário',
-    'Gasóleo colorido e marcado'
+        'Gasolina super sem chumbo IO 95',
+        'Gasóleo rodoviário',
+        'Gasóleo colorido e marcado'
 )
+
+# History plot
+HISTORY_PLOT_LABEL_GASOLINA_IO95 = 'Gasolina IO95'
+HISTORY_PLOT_LABEL_GASOLINA_IO98 = 'Gasolina IO98'
+HISTORY_PLOT_LABEL_GASOLEO_RODOVIARIO = 'Gasoleo Rodoviario'
+HISTORY_PLOT_Y_LABEL = 'Preço (€)'
+HISTORY_PLOT_X_LABEL = 'Data'
+
+# CSV columns
+COLUMN_START_DATE = 'start_date'
+COLUMN_END_DATE = 'end_date'
+COLUMN_GASOLINA_IO95 = 'gasolina_IO95'
+COLUMN_GASOLINA_IO98 = 'gasolina_IO98'
+COLUMN_GASOLEO_RODOVIARIO = 'gasoleo_rodoviario'
+
+# Tweets
+TWEET_HISTORY = 'Variação dos preços dos combustíveis na Madeira, de {start_date} a {end_date}.'

post_tweet.py

@@ -23,6 +23,29 @@ def gas_prices_message(price_current, price_previous):
         return f'{price_current}€   =   {price_previous}€'
 
 
+def create_api(client):
+    auth = tweepy.OAuthHandler(
+            client.consumer_key,
+            client.consumer_secret,
+    )
+    auth.set_access_token(
+            client.access_token,
+            client.access_token_secret,
+    )
+    return tweepy.API(auth)
+
+
+def post_image(tweet, image_path):
+    # Get Twitter client
+    client = create_client_twitter()
+    # Create Twitter API
+    api = create_api(client)
+    # Upload image
+    image = api.media_upload(image_path)
+    # Post tweet
+    return api.update_status(status=tweet, media_ids=[image.media_id])
+
+
 def make_tweet(dict_prices):
     # Get Twitter client
     client = create_client_twitter()

requirements.txt

@@ -1,4 +1,6 @@
 requests~=2.27.1
 beautifulsoup4~=4.10.0
 tweepy~=4.8.0
-PyPDF2~=2.9.0
+PyPDF2~=2.9.0
+matplotlib~=3.5.3
+pandas~=1.4.3

script.py

@@ -31,7 +31,7 @@
 i = 1
 while i < len(gas_info):
     if gas_info[i].startswith(NEW_DATE_KEY):
-        date_positions.append(i+1)
+        date_positions.append(i + 1)
     i += 1
 
 # Parse the last available date
@@ -49,8 +49,7 @@
 # If we don't have the date, update
 if update:
     # Prepare the dictionaire
-    dict_prices = {PREVIOUS_WEEK: curret_data[CURRENT_WEEK]}
-    dict_prices[CURRENT_WEEK] = {START_DATE_KEY: start_date, END_DATE_KEY: end_date, GAS_KEY: {}}
+    dict_prices = {PREVIOUS_WEEK: curret_data[CURRENT_WEEK], CURRENT_WEEK: {START_DATE_KEY: start_date, END_DATE_KEY: end_date, GAS_KEY: {}}}
     # Parse the data
     i += 1
     while i < len(gas_info) - 1:

script_post_plot_history.py

@@ -0,0 +1,43 @@
+import locale
+import tempfile
+
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mtick
+import pandas as pd
+
+from constants import TWEET_HISTORY, CURRENT_GAS_HISTORY_CSV_FILE, COLUMN_START_DATE, COLUMN_GASOLINA_IO95, COLUMN_GASOLINA_IO98, COLUMN_GASOLEO_RODOVIARIO, HISTORY_PLOT_LABEL_GASOLINA_IO95, HISTORY_PLOT_LABEL_GASOLINA_IO98, HISTORY_PLOT_LABEL_GASOLEO_RODOVIARIO, HISTORY_PLOT_Y_LABEL, HISTORY_PLOT_X_LABEL
+from post_tweet import post_image
+
+locale.setlocale(locale.LC_ALL, 'pt_PT.UTF-8')
+
+
+def generate_plot_history(plot_path):
+    # Select the last 6 months of data
+    history = pd.read_csv(CURRENT_GAS_HISTORY_CSV_FILE)
+    history[COLUMN_START_DATE] = pd.to_datetime(history[COLUMN_START_DATE])
+    history.set_index(COLUMN_START_DATE, inplace=True)
+    history = history.last('6M').copy()
+
+    # Get start and end dates
+    start_date, end_date = history.index.min(), history.index.max()
+
+    # Generate plot
+    plt.figure()
+    plot = history.plot(y=[COLUMN_GASOLINA_IO95, COLUMN_GASOLINA_IO98, COLUMN_GASOLEO_RODOVIARIO],
+                        label=[HISTORY_PLOT_LABEL_GASOLINA_IO95, HISTORY_PLOT_LABEL_GASOLINA_IO98, HISTORY_PLOT_LABEL_GASOLEO_RODOVIARIO], ylabel=HISTORY_PLOT_Y_LABEL, xlabel=HISTORY_PLOT_X_LABEL)
+
+    plot.yaxis.set_major_formatter(mtick.FormatStrFormatter('%.3f'))
+
+    # Save plot
+    plt.savefig(plot_path, dpi=300, bbox_inches='tight')
+
+    return start_date, end_date, plot
+
+
+# Generate history plot
+temp_dir = tempfile.TemporaryDirectory()
+plot_path = temp_dir.name + '/plot.png'
+start_date, end_date, plot = generate_plot_history(plot_path)
+
+# Post tweet with image
+post_image(TWEET_HISTORY.format(start_date=start_date, end_date=end_date), plot_path)