Day 14 - Web App with Flask, FastAPI, ngrok, and Invictify

Author: codingforentrepreneurs

tutorial-reference/Day 14/Pipfile

@@ -0,0 +1,18 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+flask = "*"
+gunicorn = "*"
+uvicorn = "*"
+fastapi = "*"
+requests = "*"
+pandas = "*"
+requests-html = "*"
+
+[requires]
+python_version = "3.8"

tutorial-reference/Day 14/Pipfile.lock

@@ -0,0 +1,34 @@
+Rank,Release Group,Worldwide,Domestic,%,Foreign,%
+1,Bad Boys for Life,"$419,074,646","$204,417,855",48.8%,"$214,656,791",51.2%
+2,Sonic the Hedgehog,"$306,766,470","$146,066,470",47.6%,"$160,700,000",52.4%
+3,Dolittle,"$223,343,452","$77,047,065",34.5%,"$146,296,387",65.5%
+4,Birds of Prey: And the Fantabulous Emancipation of One Harley Quinn,"$201,858,461","$84,158,461",41.7%,"$117,700,000",58.3%
+5,The Invisible Man,"$122,914,050","$64,914,050",52.8%,"$58,000,000",47.2%
+6,The Gentlemen,"$114,996,853","$36,296,853",31.6%,"$78,700,000",68.4%
+7,The Call of the Wild,"$107,604,626","$62,342,368",57.9%,"$45,262,258",42.1%
+8,The Grudge,"$49,511,319","$21,221,803",42.9%,"$28,289,516",57.1%
+9,Fantasy Island,"$47,315,959","$26,441,782",55.9%,"$20,874,177",44.1%
+10,Underwater,"$40,882,928","$17,291,078",42.3%,"$23,591,850",57.7%
+11,Like a Boss,"$29,753,143","$22,169,514",74.5%,"$7,583,629",25.5%
+12,Bloodshot,"$28,428,855","$10,021,787",35.3%,"$18,407,068",64.7%
+13,Emma.,"$25,155,355","$10,055,355",40%,"$15,100,000",60%
+14,Gretel & Hansel,"$21,282,967","$15,347,654",72.1%,"$5,935,313",27.9%
+15,Brahms: The Boy II,"$20,311,536","$12,611,536",62.1%,"$7,700,000",37.9%
+16,The Turning,"$18,474,062","$15,472,775",83.8%,"$3,001,287",16.2%
+17,The Way Back,"$14,690,514","$13,590,514",92.5%,"$1,100,000",7.5%
+18,La Belle Époque,"$14,298,084",-,-,"$14,298,084",100%
+19,I Still Believe,"$10,484,984","$9,868,521",94.1%,"$616,463",5.9%
+20,The Hunt,"$6,512,500","$5,812,500",89.3%,"$700,000",10.7%
+21,Las Pildoras De Mi Novio,"$6,292,780","$2,971,116",47.2%,"$3,321,664",52.8%
+22,The Rhythm Section,"$5,989,583","$5,437,971",90.8%,"$551,612",9.2%
+23,My Spy,"$4,418,501",-,-,"$4,418,501",100%
+24,The Last Full Measure,"$3,006,059","$2,949,212",98.1%,"$56,847",1.9%
+25,The Lodge,"$2,679,438","$1,666,564",62.2%,"$1,012,874",37.8%
+26,Trolls World Tour,"$1,887,828",-,-,"$1,887,828",100%
+27,Blind Eyes Opened,"$234,505","$234,505",100%,-,-
+28,Burden,"$139,270","$139,270",100%,-,-
+29,"14 Days, 12 Nights","$122,806","$122,530",99.8%,$276,0.2%
+30,First Cow,"$101,068","$101,068",100%,-,-
+31,Run,"$4,052",-,-,"$4,052",100%
+32,Mulan,$874,-,-,$874,100%
+33,A Quiet Place Part II,$159,-,-,$159,100%

tutorial-reference/Day 14/data/2020.csv

@@ -0,0 +1,3 @@
+<h1>Hello World</h1>
+<p>http://localhost:8000</p>
+<p>http://127.0.0.1:8000</p>

tutorial-reference/Day 14/index.html

@@ -0,0 +1,12 @@
+import os
+import datetime
+
+BASE_DIR = os.path.dirname(__file__)
+log_dir = os.path.join(BASE_DIR, "logs")
+os.makedirs(log_dir, exist_ok=True)
+
+def trigger_log_save():
+    filename = f"{datetime.datetime.now()}.txt"
+    filepath = os.path.join(log_dir, filename)
+    with open(filepath, 'w+') as f:
+        f.write("")

tutorial-reference/Day 14/logger.py

@@ -0,0 +1 @@
+gunicorn server1:app --bind 127.0.0.1:8888

tutorial-reference/Day 14/logs/2020-04-06 11:22:00.968551.txt

@@ -0,0 +1 @@
+gunicorn server1:app --bind 127.0.0.1:8888

tutorial-reference/Day 14/logs/2020-04-06 12:22:00.912814.txt

@@ -0,0 +1 @@
+uvicorn server2:app --port 8888

tutorial-reference/Day 14/logs/2020-04-06 13:22:00.853037.txt

@@ -0,0 +1 @@
+uvicorn server2:app --port 8888

tutorial-reference/Day 14/run1.ps1

@@ -0,0 +1,78 @@
+import os
+import sys
+import datetime
+import requests
+import pandas as pd
+from requests_html import HTML
+
+BASE_DIR = os.path.dirname(__file__)
+
+
+
+def url_to_txt(url, filename="world.html", save=False):
+    r = requests.get(url)
+    if r.status_code == 200:
+        html_text = r.text
+        if save:
+            with open(f"world-{year}.html", 'w') as f:
+                f.write(html_text)
+        return html_text
+    return None
+
+
+
+def parse_and_extract(url, name='2020'):
+    html_text = url_to_txt(url)
+    if html_text == None:
+        return False
+    r_html = HTML(html=html_text)
+    table_class = ".imdb-scroll-table"
+    # table_class = "#table"
+    r_table = r_html.find(table_class)
+
+    # print(r_table)
+    table_data = []
+    # table_data_dicts = []
+    header_names = []
+    if len(r_table) == 0:
+        return False
+    parsed_table = r_table[0]
+    rows = parsed_table.find("tr")
+    header_row = rows[0]
+    header_cols = header_row.find('th')
+    header_names = [x.text for x in header_cols]
+    for row in rows[1:]:
+        # print(row.text)
+        cols = row.find("td")
+        row_data = []
+        row_dict_data = {}
+        for i, col in enumerate(cols):
+            # print(i, col.text, '\n\n')
+            header_name = header_names[i]
+            # row_dict_data[header_name] = col.text
+            row_data.append(col.text)
+        # table_data_dicts.append(row_dict_data)
+        table_data.append(row_data)
+    df = pd.DataFrame(table_data, columns=header_names)
+    # df = pd.DataFrame(table_data_dicts)
+    path = os.path.join(BASE_DIR, 'data')
+    os.makedirs(path, exist_ok=True)
+    filepath = os.path.join('data', f'{name}.csv')
+    df.to_csv(filepath, index=False)
+    return True
+
+def run(start_year=None, years_ago=0):
+    if start_year == None:
+        now = datetime.datetime.now()
+        start_year = now.year
+    assert isinstance(start_year, int)
+    assert isinstance(years_ago, int)
+    assert len(f"{start_year}") == 4
+    for i in range(0, years_ago+1):
+        url = f"https://www.boxofficemojo.com/year/world/{start_year}/"
+        finished = parse_and_extract(url, name=start_year)
+        if finished:
+            print(f"Finished {start_year}")
+        else:
+            print(f"{start_year} not finished")
+        start_year -= 1

tutorial-reference/Day 14/run1.sh

@@ -0,0 +1,25 @@
+from flask import Flask
+
+from logger import trigger_log_save
+from scrape import run as scrape_runner
+
+app = Flask(__name__)
+
+# http://localhost:8000/
+@app.route("/", methods=['GET'])
+def hello_world():
+    # run other code here.
+    return "Hello, world. this is Flask"
+
+# http://localhost:8000/abc
+@app.route("/abc", methods=['GET'])
+def abc_view():
+    # run other code here.
+    return "Hello, world. this is abc"
+
+@app.route("/box-office-mojo-scraper", methods=['POST'])
+def box_office_scraper_view():
+    # run other code here.
+    trigger_log_save()
+    scrape_runner()
+    return {"data": [1,2,3]}

tutorial-reference/Day 14/run2.ps1

@@ -0,0 +1,21 @@
+import os
+import datetime
+from fastapi import FastAPI
+from logger import trigger_log_save
+from scrape import run as scrape_runner
+app = FastAPI()
+
+@app.get("/")
+def hello_world():
+    return {"hello": "world"}
+
+@app.get("/abc")
+def abc_view():
+    return {"data": [1,2,3]}
+
+
+@app.post("/box-office-mojo-scraper")
+def scrape_runner_view():
+    trigger_log_save()
+    scrape_runner()
+    return {"data": [1,2,3]}

tutorial-reference/Day 14/run2.sh

@@ -0,0 +1,7 @@
+import requests 
+
+ngrok_url = 'https://a5681caa.ngrok.io'
+endpoint = f'{ngrok_url}/box-office-mojo-scraper'
+
+r = requests.post(endpoint, json={})
+print(r.json()['data'])