BaOIA_API_Gallica_FabLab.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"BaOIA_API_Gallica_FabLab.ipynb","provenance":[{"file_id":"1Dk-CylXp_OlpvFbJnwUIYGlt1x5np5_5","timestamp":1636102186920},{"file_id":"1D6q6SXyoK_PvA2isfUVyjwq2dQq-fEY-","timestamp":1635167649130}],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"Gd-7AMaHvS54"},"source":["# Recherche et alignement, correspondance de la basea de données Gallica via l'API:\n","\n"]},{"cell_type":"code","metadata":{"id":"u2vKBGDIvjkP","cellView":"form"},"source":["#@title Installation des bibliothèques nécessaires et connexion à un compte Google Drive.\n","\n","''' \n","Google Colab notebook.\n","Python == 3.7.11\n","\n","BaOIA - La Contemporaine - Université de Nanterre\n","'''\n","\n","\n","## Installation des bibliothèques et connexion au compte Google Drive\n","\n","!pip install utils\n","from google.colab import drive\n","import json\n","from openpyxl import load_workbook\n","import pandas as pd\n","import re\n","import requests\n","!pip install langdetect\n","from langdetect.lang_detect_exception import LangDetectException\n","import shutil\n","!pip install xmltojson\n","import xmltodict\n","import urllib\n","import urllib.request, urllib.error, urllib.parse\n","import xmltojson\n","from langdetect import detect\n","import os\n","import xml.etree.ElementTree as ET\n","import lxml\n","from collections import OrderedDict\n","from lxml import etree\n","!pip install unidecode==1.2.0\n","import unidecode\n","import folium\n","import unicodedata\n","from urllib.error import HTTPError\n","from re import match\n","from bs4 import BeautifulSoup\n","import urllib.request, urllib.error, urllib.parse\n","import string\n","\n","drive.mount('/content/drive/')\n","\n","\n","#@markdown Lors du lancement de la cellule, la connexion vers le Google Drive sera nécessaire, les étapes à suivre sont:\n","\n","#@markdown - Indiquer le chemin absolu vers le dossier de travail contenu sur le Google Drive (ici, le dossier de travail se nomme \"FabLab\")\n","chemin_absolu_vers_le_dossier_de_travail = '/content/drive/My Drive/FabLab-master/' #@param {type:\"string\"}\n","\n","\n","\n","#@markdown - En dessous de la cellule, un lien bleu va apparaître après 'Go to this URL in a browser'. Cliquer dessus, un nouvel onglet du navigateur va s'ouvrir.\n","\n","#@markdown - Sur ce nouvel onglet, cliquer sur le compte Google auquel le Drive est associé. Cliquer ensuite sur \"Connexion\" puis sur la petite icône en forme de carré pour copier le lien.\n","\n","#@markdown - De retour sur le notebook de travail, coller le lien dans la case en dessous de la cellule après \"Enter your authorization code:\" puis appuyer sur la touche \"Enter\" du clavier. La cellule va finir de l'exécuter normalement.\n","\n","os.chdir(chemin_absolu_vers_le_dossier_de_travail)\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uS01rbRdDMWR","cellView":"form"},"source":["#@title Recherche de données correspondantes à partir d'un terme dans la base de données Gallica.\n","\n","#@markdown\n","\n","#@markdown Indiquer ici le terme à chercher dans la base de données GALLICA: possibilité de ne mettre qu'un titre (d'oeuvre) ou un artiste, ou les deux! Il est aussi possiblé de ne séléctionner qu'un type d'objets en particuliers; mettre \"tout\" pour n'en séléctionner aucun en particulier.\n","\n","#@markdown Accents non reconnus ! Exemple, pour chercher : Molière, mettre : Moliere\n","\n","\n","#@ Les valeurs possibles sont monographie, carte, image, fascicule (pour les publications en série), manuscrit, partition, sonore (pour les documents son), objet et vidéo.\n","titre_a_chercher = \"guerre d'espagne\" #@param {type:\"string\"}\n","auteur_a_chercher = \"\" #@param {type:\"string\"}\n","type_a_chercher = \"tout\" #@param [\"tout\", \"monographie\", \"carte\", \"image\", \"fascicule\", \"manuscrit\", \"partition\", \"sonore\", \"objet\", \"vidéo\"]\n","\n","\n","\n","os.chdir(chemin_absolu_vers_le_dossier_de_travail)\n","if titre_a_chercher != \"\":\n","  os.makedirs(titre_a_chercher, exist_ok=True)\n","  os.chdir(titre_a_chercher)\n"," \n","elif auteur_a_chercher != \"\":\n","  os.makedirs(auteur_a_chercher, exist_ok=True)\n","  os.chdir(auteur_a_chercher)\n","\n","\n","chemin_dossier_recherche = os.path.abspath(os.getcwd())\n","lien_base_recherche = \"https://gallica.bnf.fr/SRU?version=1.2&operation=searchRetrieve%22&maximumRecords=49&query=\"\n","lien_de_recherche_gallica = None\n","\n","### RECHERCHE GALLICA\n","if titre_a_chercher != \"\":\n","  if ' ' in titre_a_chercher:\n","    titre_a_chercher = re.sub(r' ', '%20', titre_a_chercher)\n","    titre_a_chercher = ''.join(\"\\\"\"+titre_a_chercher+\"\\\"\")\n","    titre_pour_lien = \"\".join(\"dc.title%20adj%20%22\"+ titre_a_chercher+\"%22\")\n","  else:\n","    titre_a_chercher=titre_a_chercher\n","    titre_pour_lien = \"\".join(\"dc.title%20any%20\"+ titre_a_chercher)\n","  lien_de_recherche_gallica = \"\".join(lien_base_recherche+ titre_pour_lien)\n","\n","\n","if auteur_a_chercher != \"\":\n","  if ' ' in auteur_a_chercher:\n","    auteur_a_chercher = re.sub(r' ', '%20', auteur_a_chercher)\n","  else:\n","    auteur_a_chercher=auteur_a_chercher\n","  auteur_pour_lien = \"\".join(\"dc.creator%adj%20\\\"\"+ auteur_a_chercher + \"\\\"\")\n","  if lien_de_recherche_gallica is None:\n","    lien_de_recherche_gallica = \"\".join(lien_base_recherche + auteur_pour_lien)\n","  else:\n","    lien_de_recherche_gallica = \"\".join(lien_de_recherche_gallica + \"%20&%20\"+auteur_pour_lien)\n","\n","if type_a_chercher is not \"tout\":\n","  type_pour_lien = \"\".join(\"%20and%20dc.type%20any%20\"+type_a_chercher)\n","  lien_de_recherche_gallica = \"\".join(lien_de_recherche_gallica + type_pour_lien)\n","\n","print(lien_de_recherche_gallica)\n","request = urllib.request.Request(lien_de_recherche_gallica)\n","try:\n","  content = urllib.request.urlopen(request)\n","  parse1 = BeautifulSoup(content,'html.parser')\n","  dict_final = []\n","  my =xmltodict.parse(parse1.prettify())\n","  dict_final.append(my)\n","except HTTPError as he:\n","  print('Pas de téléchargement possible pour la base Gallica avec cette recherche')\n","\n","for key, value in my.items():\n","    for key2, value2 in value.items():\n","        if key2 == 'srw:numberofrecords':\n","          nbr_results = value2\n","          print(\"Le nombre de résultats poour la base Gallica est de : \" + str(nbr_results))\n","          if int(nbr_results) > 50:\n","            multiples = [n for n in range(1, int(nbr_results)) if n % 50 == 0]\n","            nbr_recherche_depart = 0\n","            for multiple in multiples:\n","              nbr_recherche_depart =nbr_recherche_depart+50\n","              lien_de_recherche_gallica2 = \"\".join(lien_de_recherche_gallica+'&startRecord='+str(nbr_recherche_depart))\n","              request = urllib.request.Request(lien_de_recherche_gallica2)\n","              try:\n","                content = urllib.request.urlopen(request)\n","                parses = BeautifulSoup(content,'html.parser')\n","                my3 =xmltodict.parse(parses.prettify())\n","                dict_final.append(my3)\n","              except HTTPError  as hto:\n","                print(\"Erreur de connexion au serveur Gallica, ré-essayer!\")\n","nom_du_fichier_json_base_gallica = 'donnees_base_gallica_guerre_espagne.json' #@param {type:\"string\"}\n","with open(nom_du_fichier_json_base_gallica, 'w') as pi:\n","  json.dump(dict_final, pi, indent=4, ensure_ascii=False)\n","print(\"Fichier de données enregistré!\")\n","\n","\n","#@markdown Il est possible de jeter un oeil au fichier créé qui contient déjà les informations \"brutes\" tirées dde la base de données\n","\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"NkuWVJyLCLe0","cellView":"form"},"source":["### Base GALLICA\n","#@markdown Indiquer ici le nom du dossier dans lesquelles les images de la base Gallica vont se télécharger:\n","dossier_base_gallica = 'images_base_gallica_guerre_espagne' #@param {type:\"string\"}\n","new_gallica = {}\n","#chemin_fichier2 = \"\".join(chemin_dossier+'/'+nom_du_fichier_json_base_gallica)\n","with open(nom_du_fichier_json_base_gallica, 'r') as ret:\n","  gallica = json.load(ret)\n","print(\"Les images sont en train de se télécharger...\")\n","for ele in gallica:\n","  for key, value in ele.items():\n","    for key2, value2 in value.items():\n","      if key2 == 'srw:records':\n","        for key3, value3 in value2.items():\n","          if key3 == 'srw:record':\n","            for ele2 in value3:\n","              for key, value in ele2.items():\n","                if key == 'srw:recorddata':\n","                  for key2, value2 in value.items():\n","                    for key3, value3 in value2.items():\n","                      if key3 == 'dc:title':\n","                        le_titre = str(value3)\n","                        le_nouveau_titre = re.sub(r\"[\\['\\]]\", '', le_titre)\n","                        new_gallica[le_nouveau_titre] = {}\n","                      if key3 == 'dc:date':\n","                        date = value3\n","                      if key3 == 'dc:creator':\n","                        auteur = value3\n","                      if key3 == 'dc:type':\n","                        types = value3\n","                      if key3 == 'dc:subject':\n","                        subject = value3\n","                      if key3 == 'dc:identifier':\n","                        if type(value3) == list:\n","                          value3 = value3[1]\n","                        identifiant = value3[27:]\n","                        lien = 'http://gallica.bnf.fr/iiif/ark:' + identifiant + '/f1/full/3000/0/native.jpg'\n","                        new_gallica[le_nouveau_titre] = {'Date': date, 'Auteur':auteur, 'Type': types, 'Sujet': subject, 'Lien': lien}\n","os.chdir(chemin_dossier_recherche)\n","#@markdown Indiquer ici le nom du fichier de données nettoyé contenant uniquement les informations essentielles, base Gallica. Le fichier est au format json.\n","print('Création du nouveau fichier nettoyé pour la base Gallica.')\n","nom_du_fichier_nettoyé_base_gallica = 'base_Gallica_venus_cartes_final.json' #@param {type:\"string\"}\n","with open(nom_du_fichier_nettoyé_base_gallica, 'w') as pl:\n","  json.dump(new_gallica, pl, indent=4, ensure_ascii=False)\n","\n","if not os.path.exists(dossier_base_gallica):\n","  os.mkdir(dossier_base_gallica)\n","os.chdir(dossier_base_gallica)\n","for rekey, revalue in new_gallica.items():\n","  rekey = re.sub(r\"[/;:']\", \"_\", rekey)\n","  nom_image = \"\".join(rekey + '.jpg')\n","  nom_imageb = re.sub(r\"[/;:' ]\", \"_\", nom_image)\n","  nom_imagec = \"\".join(nom_imageb +'_'+ark+ '.jpg')\n","  for rekey2, revalue2 in revalue.items():\n","    if rekey2 == 'Lien':\n","      if ' ' in revalue2:\n","        revalue2 = re.sub(r' ', '%20', revalue2)\n","      try:\n","        urllib.request.urlretrieve(revalue2, nom_imagec)\n","      except (OSError, HTTPError, urllib.error.ContentTooShortError, UnicodeEncodeError) as http:\n","        pass"],"execution_count":null,"outputs":[]}]}