Essaie régler les problèmes widgets JS (#231)

* teste raw html * Automated changes * Automated changes * test avec rawhtml * Automated changes * Automated changes * un autre tp folium * Automated changes * Automated changes * test intégration plotyl * Automated changes * Automated changes * try rawhtml * Automated changes * Automated changes * chart * Automated changes * Automated changes * up * Automated changes * Automated changes * try diff shortcode * Automated changes * Automated changes * try diff shortcode * head plotly * Automated changes * Automated changes * Automated changes * tente basique * Automated changes * Automated changes * try as html * Automated changes * Automated changes * test show * Automated changes * Automated changes * regex replace header * ajoute chart plotly * Automated changes * Automated changes * add content * Automated changes * Automated changes * change ordre * Automated changes * up * not _index * Automated changes * Automated changes * not _index * Automated changes * retire * Automated changes * retire * Automated changes * retire * Automated changes * Automated changes * tweak all * Automated changes * essaie diff * Automated changes * encoding * Automated changes * retire espace * Automated changes * Automated changes * plotly * Automated changes * Automated changes Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
linogaliana · Jun 3, 2022 · 7b9f27b · 7b9f27b
1 parent 48606dd
commit 7b9f27b
Show file tree

Hide file tree

Showing 10 changed files with 173 additions and 18 deletions.
diff --git a/.github/workflows/netlify-test.yaml b/.github/workflows/netlify-test.yaml
@@ -33,15 +33,13 @@ jobs:
           mv content/course/manipulation/index.md content/course/manipulation/_index.md
           mv content/course/visualisation/index.md content/course/visualisation/_index.md
           mv content/course/modelisation/index.md content/course/modelisation/_index.md
+          python build/wc_website.py
+          python build/tweak_markdown.py
           hugo mod graph
           hugo -D --themesDir themes -t github.com
-      - name: Clean files with Python function
-        run: |
-         pip install matplotlib wordcloud nltk
-         python build/wc_website.py
-         python build/post-build.py
       - name: Copy notebooks in dedicated directory
         run: |
+         python build/post-build.py
          mkdir -p notebooks
          python build/move_files.py notebooks
       - name: Install npm
@@ -61,6 +59,10 @@ jobs:
           npm install --unsafe-perm=true netlify-cli -g
           netlify init
           netlify deploy --alias=${BRANCHE_REF} --dir="public" --message "Preview deploy from ${BRANCHE_REF}"
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Content
+          path: content/
       - uses: actions/upload-artifact@v2
         with:
           name: Website

diff --git a/assets/scss/custom.scss b/assets/scss/custom.scss
@@ -386,6 +386,18 @@ h3 {
   line-height: 0;
 }
 
+
+// Plotly chart
+div.chart {
+  max-width: 100%;
+  margin-left: auto;
+  margin-right: auto;
+  margin-bottom: 1rem;
+
+  // Add horizontal scroll on mobile since Plotly
+  overflow-x: auto;
+}
+
 /*
 .docs-article-container {
     max-width: 760px;

diff --git a/build/post-build.py b/build/post-build.py
@@ -1,5 +1,5 @@
 import glob
-
+import re
 def read_file(filename):
     with open(filename, 'r', encoding='utf-8') as f:
         text = f.readlines()
@@ -13,14 +13,22 @@ def replace_shortcode_tabs(content):
     content = content.replace("\>", ">")
     return content
 
+def tweak_js_plotly(content):
+    content2 = re.sub(
+        r'<script type="text/javascript">\n([\S\s]*)</script>\n',
+        '',
+        content)
+    return content2
+
 
 def write_file(filename, content):
     with open(filename, 'w') as f:
         f.write(content)
 
 def clean_write_file(fl):
     content = read_file(fl)
-    content = replace_shortcode_tabs(content)
+    if not fl.endswith("_index.md"):
+        content = replace_shortcode_tabs(content)
     write_file(fl, content)
 
 list_files = glob.glob("./content/**/*.md", recursive=True)

diff --git a/build/tweak_markdown.py b/build/tweak_markdown.py
@@ -0,0 +1,42 @@
+import glob
+import re
+
+def read_file(filename):
+    with open(filename, 'r', encoding='utf-8') as f:
+        text = f.readlines()
+        f.close()
+    new_text = "".join([line for line in text])
+    s = new_text
+    return s
+
+def replace_shortcode_tabs(content):
+    content = content.replace("\<", "<")
+    content = content.replace("\>", ">")
+    return content
+
+def tweak_js_plotly(content):
+    content2 = re.sub(
+        r'<script type="text/javascript">\n([\S\s]*)</script>\n',
+        '',
+        content)
+    return content2
+
+
+def write_file(filename, content):
+    with open(filename, 'w', encoding="utf-8") as f:
+        f.write(content)
+
+def clean_write_file(fl):
+    content = read_file(fl)
+    if re.search(
+        r'<script type="text/javascript">\n([\S\s]*)</script>\n',
+        content) is not None:
+        print("plotly detected")
+        content = tweak_js_plotly(content)
+        write_file(fl, content)
+    else:
+        print("nothing to do")
+
+list_files = glob.glob("./content/course/**/*.md", recursive=True)
+print(list_files)
+[clean_write_file(fl) for fl in list_files if not fl.endswith("_index.md")]
diff --git a/content/course/manipulation/04c_API_TP/index.qmd b/content/course/manipulation/04c_API_TP/index.qmd
@@ -386,12 +386,22 @@ m.fit_bounds([sw, ne])
 ```
 
 ```{python}
+#| eval : false
 # Afficher la carte
 m
 ```
 
 {{% /box %}}
 
+{{< rawhtml >}}
+```{python}
+#| echo : false
+# Afficher la carte
+m
+```
+{{< /rawhtml >}}
+
+
 
 # Géocoder des données grâce aux API officielles
 
@@ -610,10 +620,13 @@ m.fit_bounds([sw, ne])
 
 Ce qui permet d'obtenir la carte:
 
+{{< rawhtml >}}
 ```{python}
+#| echo : false
 # Afficher la carte
 m
 ```
+{{< /rawhtml >}}
 
 Vous pouvez aller plus loin avec l'exercice suivant
 

diff --git a/content/course/modelisation/0_preprocessing/getdata.py b/content/course/modelisation/0_preprocessing/getdata.py
@@ -0,0 +1,65 @@
+import urllib
+import urllib.request
+import os
+import zipfile
+from urllib.request import Request, urlopen
+from pathlib import Path
+import numpy as np
+import pandas as pd
+import geopandas as gpd
+
+def download_url(url, save_path):
+    with urllib.request.urlopen(url) as dl_file:
+        with open(save_path, 'wb') as out_file:
+            out_file.write(dl_file.read())
+
+
+def create_votes_dataframes():
+
+  Path("data").mkdir(parents=True, exist_ok=True)
+
+
+  download_url("https://www2.census.gov/geo/tiger/GENZ2019/shp/cb_2019_us_county_20m.zip", "data/shapefile")
+  with zipfile.ZipFile("data/shapefile", 'r') as zip_ref:
+      zip_ref.extractall("data/counties")
+
+  shp = gpd.read_file("data/counties/cb_2019_us_county_20m.shp")
+  shp = shp[~shp["STATEFP"].isin(["02", "69", "66", "78", "60", "72", "15"])]
+  shp
+
+  df_election = pd.read_csv("https://raw.githubusercontent.com/tonmcg/US_County_Level_Election_Results_08-20/master/2020_US_County_Level_Presidential_Results.csv")
+  df_election.head(2)
+  population = pd.read_excel("https://www.ers.usda.gov/webdocs/DataFiles/48747/PopulationEstimates.xls?v=290.4", header = 2).rename(columns = {"FIPStxt": "FIPS"})
+  education = pd.read_excel("https://www.ers.usda.gov/webdocs/DataFiles/48747/Education.xls?v=290.4", header = 4).rename(columns = {"FIPS Code": "FIPS", "Area name": "Area_Name"})
+  unemployment = pd.read_excel("https://www.ers.usda.gov/webdocs/DataFiles/48747/Unemployment.xls?v=290.4", header = 4).rename(columns = {"fips_txt": "FIPS", "area_name": "Area_Name", "Stabr": "State"})
+  income = pd.read_excel("https://www.ers.usda.gov/webdocs/DataFiles/48747/PovertyEstimates.xls?v=290.4", header = 4).rename(columns = {"FIPStxt": "FIPS", "Stabr": "State", "Area_name": "Area_Name"})
+
+
+  dfs = [df.set_index(['FIPS', 'State']) for df in [population, education, unemployment, income]]
+  data_county = pd.concat(dfs, axis=1)
+  df_election = df_election.merge(data_county.reset_index(), left_on = "county_fips", right_on = "FIPS")
+  df_election['county_fips'] = df_election['county_fips'].astype(str).str.lstrip('0')
+  shp['FIPS'] = shp['GEOID'].astype(str).str.lstrip('0')
+  votes = shp.merge(df_election, left_on = "FIPS", right_on = "county_fips")
+
+  req = Request('https://dataverse.harvard.edu/api/access/datafile/3641280?gbrecs=false')
+  req.add_header('User-Agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:77.0) Gecko/20100101 Firefox/77.0')
+  content = urlopen(req)
+  df_historical = pd.read_csv(content, sep = "\t")
+  #df_historical = pd.read_csv('https://dataverse.harvard.edu/api/access/datafile/3641280?gbrecs=false', sep = "\t")
+
+  df_historical = df_historical.dropna(subset = ["FIPS"])
+  df_historical["FIPS"] = df_historical["FIPS"].astype(int)
+  df_historical['share'] = df_historical['candidatevotes']/df_historical['totalvotes']
+  df_historical = df_historical[["year", "FIPS", "party", "candidatevotes", "share"]]
+  df_historical['party'] = df_historical['party'].fillna("other")
+
+  df_historical_wide = df_historical.pivot_table(index = "FIPS", values=['candidatevotes',"share"], columns = ["year","party"])
+  df_historical_wide.columns = ["_".join(map(str, s)) for s in df_historical_wide.columns.values]
+  df_historical_wide = df_historical_wide.reset_index()
+  df_historical_wide['FIPS'] = df_historical_wide['FIPS'].astype(str).str.lstrip('0')
+  votes['FIPS'] = votes['GEOID'].astype(str).str.lstrip('0')
+  votes = votes.merge(df_historical_wide, on = "FIPS")
+  votes["winner"] =  np.where(votes['votes_gop'] > votes['votes_dem'], 'republican', 'democrats') 
+
+  return votes
diff --git a/content/course/modelisation/0_preprocessing/index.qmd b/content/course/modelisation/0_preprocessing/index.qmd
@@ -158,14 +158,14 @@ l'une des surcouches à JavaScript vue dans la partie [visualisation](#visualisa
 
 En l'occurrence, on peut utiliser `plotly` pour tenir compte de la population:
 
-{{< chart data="../course/modelisation/people_vote" >}}
+{{< chart data="people_vote" >}}
 
 La Figure a été obtenue avec le code suivant:
 
 ```{python}
 #| message: false
 #| warning: false
-#| output: false
+#| output: hide
 import plotly
 import plotly.graph_objects as go
 import pandas as pd
@@ -211,6 +211,8 @@ fig_plotly.update_layout(
         showlegend = True,
         geo = {"scope": 'usa', "landcolor": 'rgb(217, 217, 217)'}
     )
+
+fig_plotly.show()
 ```
 
 
@@ -312,10 +314,10 @@ htmlsnip2.write_json("scatter_matrix.json")
 
 ```
 
-{{< chart data="../course/modelisation/scatter_matrix" >}}
-
 {{% /box %}}
 
+{{< chart data="scatter_matrix" >}}
+
 
 
 ## Transformer les données

diff --git a/content/course/visualisation/maps/index.qmd b/content/course/visualisation/maps/index.qmd
@@ -488,11 +488,15 @@ for i in range(0,len(compteurs)):
 m.fit_bounds([sw, ne])
 ```
 
+{{% /box %}}
+
+{{< rawhtml >}}
 ```{python}
+#| echo : false
+# Afficher la carte
 m
 ```
-
-{{% /box %}}
+{{< /rawhtml >}}
 
 
 {{% box status="exercise" title="Exercice" icon="fas fa-pencil-alt" %}}
@@ -541,13 +545,14 @@ for i in range(0,len(df1)):
 m.fit_bounds([sw, ne])
 ```
 
+{{< /box >}}
+
 La carte obtenue doit ressembler à la suivante:
 
+{{< rawhtml >}}
 ```{python}
+#| echo : false
+# Afficher la carte
 m
 ```
-
-
-{{< /box >}}
-
-
+{{< /rawhtml >}}
diff --git a/content/course/visualisation/matplotlib/index.qmd b/content/course/visualisation/matplotlib/index.qmd
@@ -32,6 +32,7 @@ print_badges("content/course/visualisation/matplotlib.qmd")
 ```
 
 
+
 La pratique de la visualisation se fera, dans ce cours, en répliquant des graphiques qu'on peut trouver sur
 la page de l'*open-data* de la ville de Paris 
 [ici](https://opendata.paris.fr/explore/dataset/comptage-velo-donnees-compteurs/information/?disjunctive.id_compteur&disjunctive.nom_compteur&disjunctive.id&disjunctive.name).

diff --git a/layouts/partials/head.html b/layouts/partials/head.html
@@ -0,0 +1,5 @@
+<!-- Embed d3.js and plotly.js if plotly:true -->
+{{ if .Params.plotly }}
+<script src="https://d3js.org/d3.v7.min.js"></script>
+<script src="https://cdn.plot.ly/plotly-2.11.1.min.js"></script>
+{{ end }}