Fetching weather

config.json

@@ -2,14 +2,14 @@
     "sunrise_lib": {
         "HOME": "/foo/bar",
         "FNAME": "/dev/shm/solar.txt",
+        "DIR_CFG": ".config/solar",
         "FNAME_MASTER": "/.config/solar/master",
+        "PATH_JSON_GEO": "/.config/solar/geo.json",
         "CPU_FREQ_APP": "/root/bin/cpu-freq.sh",
         "HOURS_DELTA": 2.5,
         "MIN_WEATHER": 0.4,
         "DIR_TMP": "/tmp",
         "DIR_XMRIG": "/progs/crypto/xmr/xmrig/b",
-        "LAT": 53,
-        "LON": 10,
         "TESTING": false
     },
     "generator": {

geo-template.json

@@ -0,0 +1,10 @@
+{
+  "geo":
+  	  {
+  	  	"time_zone" : "Europe/Berlin",
+  	  	"country" : "germany",
+  	  	"city" : "berlin",
+		"lat" : 52.5200,
+		"lon" : 13.4050
+  	  }
+}

requirements.txt

@@ -1,4 +1,3 @@
-asciimatics==1.13.0
 attrs==21.4.0
 certifi==2021.10.8
 charset-normalizer==2.0.12
@@ -16,7 +15,6 @@ msgpack==1.0.3
 numpy==1.22.3
 packaging==21.3
 pandas==1.4.2
-Pillow==9.1.0
 pvlib==0.9.1
 pykrakenapi==0.3.0
 pyparsing==3.0.8
@@ -29,4 +27,13 @@ scipy==1.8.0
 six==1.16.0
 urllib3==1.26.9
 python-dateutil==2.8.2
+# Curses menu:
 windows-curses==2.3.0 ; sys_platform == 'win32'
+# Ascii menu:
+asciimatics==1.13.0
+# Weather:
+beautifulsoup4==4.11.1
+wget==3.2
+cairosvg>=0.7.0
+Pillow==9.1.0
+#

src/generator.py

@@ -19,6 +19,7 @@
 from matplotlib import pyplot as plt
 
 import sunrise_lib
+import weather_lib
 import kraken
 from profitability import POW_Coin
 
@@ -91,7 +92,8 @@ def plot_sun(name, elev, bat, usage, show_plots):
         plt.show()
 
 def proc_data(pos):
-    pos = simul_weather(pos)
+    pos = add_weather(pos)
+    #pos = simul_weather(pos)
     pos = adj_losses(pos)
 
     print("Dumping data to:", path_positions_txt)
@@ -290,6 +292,12 @@ def simul_weather(pos):
 
     return pos
 
+def add_weather(pos):
+    # TODO: Limit the calls to the service via caching
+    weather = weather_lib.get_weather()
+    pos.loc[:, [ELEVATION_KEY]] *= weather
+    return pos
+
 def run_main(elev, show_plots):
     run_algo(elev, show_plots, get_usage_simple)
     run_algo(elev, show_plots, get_usage_endor_example)

src/geo-template.json

@@ -0,0 +1 @@
+../geo-template.json

src/sunrise_lib.py

@@ -17,19 +17,35 @@
 
 from pytz import timezone
 import requests
+import shutil
 
 #from tzlocal import get_localzone # $ pip install tzlocal
 
 from python_json_config import ConfigBuilder
 
+
+HOME = str(Path.home()) + "/" # TODO: move this in config.json too?
+
+def get_geo():
+    os.makedirs(HOME + config.sunrise_lib.DIR_CFG, exist_ok=True)
+    geo_path = HOME + config.sunrise_lib.PATH_JSON_GEO
+    if not os.path.isfile(geo_path):
+        geo_cfg_template = 'geo-template.json'
+        print("Not found: " + geo_path)
+        print("Copying " + geo_cfg_template + " to " + geo_path)
+        shutil.copy(geo_cfg_template, geo_path)
+
+    config_geo = config_builder.parse_config(geo_path)    
+    return config_geo
+
 config_builder = ConfigBuilder()
 config = config_builder.parse_config('config.json')
+config_geo = get_geo()
 config_batteries = config_builder.parse_config('batteries.json')
 
 PROJECT_NAME = "SolOptXMR"
 PROJECT_SUB_NAME = "Solar Optimal mining of XMR"
 
-HOME = str(Path.home()) # TODO: move this in config.json too?
 FNAME = config.sunrise_lib.FNAME
 FNAME_MASTER = HOME + config.sunrise_lib.FNAME_MASTER
 CPU_FREQ_APP = config.sunrise_lib.CPU_FREQ_APP
@@ -39,8 +55,8 @@
 DIR_XMRIG = HOME + config.sunrise_lib.DIR_XMRIG
 
 DATE_NOW = datetime.datetime.now()
-LAT = config.sunrise_lib.LAT
-LON = config.sunrise_lib.LON
+LAT = config_geo.geo.lat
+LON = config_geo.geo.lon
 
 TESTING = config.sunrise_lib.TESTING
 

src/weather_lib.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# https://stackoverflow.com/a/11236372
+# apt install gfortran libffi-dev
+# pip3 install pvlib ephem pytz beautifulsoup4 cairosvg wget requests Pillow
+
+import sunrise_lib
+
+import os
+import pvlib
+import datetime
+import pandas as pd
+import time
+import traceback
+from subprocess import PIPE, run
+from pathlib import Path
+
+
+from pytz import timezone
+import requests
+from bs4 import BeautifulSoup
+import wget
+from cairosvg import svg2png
+
+from PIL import Image
+
+#from tzlocal import get_localzone # $ pip install tzlocal
+
+config_geo = sunrise_lib.config_geo
+
+TESTING = False
+#TESTING = True
+
+# TODO: Extent the prediction to multiple days ahead
+def get_weather(horizon=3):
+    try:
+        print(config_geo.geo.country)
+        print(config_geo.geo.city)
+        url = "https://www.timeanddate.com/weather/{}/{}/ext".format(config_geo.geo.country, config_geo.geo.city)
+        page = requests.get(url)
+        print(page)
+        soup = BeautifulSoup(page.content, "html.parser")
+        results = soup.find(class_="mtt")
+        img_url = results['src']
+        #print(results.prettify())
+        print(img_url)
+
+        #img_url = '//c.tadst.com/gfx/w/svg/wt-33.svg'
+        #img_url = '//c.tadst.com/gfx/w/svg/wt-1.svg'
+        img_url = img_url.replace('//', 'https://')
+        filename = wget.download(img_url, out=sunrise_lib.DIR_TMP)
+        #filename = '/tmp/a/wt-33.svg'
+        print(filename)
+
+        svg_code = sunrise_lib.read_file(filename)
+        png_file = sunrise_lib.DIR_TMP + '/output.png'
+        print("Writing to " + png_file)
+        svg2png(bytestring=svg_code, write_to=png_file)
+
+        im = Image.open(png_file, 'r')
+        width, height = im.size
+        pixel_values = list(im.getdata())
+
+        dic = {}
+        for pix in pixel_values:
+            if pix[0] == 0 and pix[1] == 0 and pix[2] == 0:
+                continue
+            if pix[0] == 255 and pix[1] == 255 and pix[2] == 255:
+                continue
+            if pix in dic:
+                dic[pix] += 1
+            else:
+                dic[pix] = 1
+            #print(pix)
+
+        pix_yellow = 0
+
+        for pix in dic:
+            val = dic[pix]
+            if val < 20:
+                continue
+            if pix[0] == 255:
+                pix_yellow = val
+            print(pix, dic[pix])
+
+        max_yellow = 0.34 # based on '//c.tadst.com/gfx/w/svg/wt-1.svg'
+
+        wh = width * height
+        yellow_relat = pix_yellow / wh / max_yellow
+        print("Yellow = ", pix_yellow, yellow_relat * 100)
+
+        #print(page.text)
+
+        if yellow_relat == 0:
+            yellow_relat = 0.1
+
+        return round(yellow_relat, 4)
+    except Exception:
+        print(traceback.format_exc())
+        return 0
+
+def test():
+    get_weather()    
+
+if __name__ == "__main__":
+    test()
+