notebooks/CreateWikidataPropertiesParallel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bm-HAIgTADwd",
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "##import the Wikidata integrator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install wikidataintegrator\n",
    "from wikidataintegrator import wdi_core, wdi_login"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get all Wikidata properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "query = \"\"\"\n",
    "SELECT DISTINCT ?property ?propertyLabel ?propertyDescription ?propType WHERE {\n",
    "   ?property wikibase:directClaim ?p ;\n",
    "             wikibase:propertyType ?propType ;\n",
    "             schema:description ?propertyDescription ;\n",
    "             rdfs:label ?propertyLabel .\n",
    "   FILTER (lang(?propertyLabel) = 'en')\n",
    "   FILTER (lang(?propertyDescription) = 'en')}\n",
    "\"\"\"\n",
    "propertiesSparql = wdi_core.WDItemEngine.execute_sparql_query(query, as_dataframe=True)\n",
    "propertiesSparql\n",
    "datatype_map = {'http://wikiba.se/ontology#CommonsMedia': 'commonsMedia',\n",
    "                'http://wikiba.se/ontology#ExternalId': 'external-id',\n",
    "                'http://wikiba.se/ontology#GeoShape': 'geo-shape',\n",
    "                'http://wikiba.se/ontology#GlobeCoordinate': 'globe-coordinate',\n",
    "                'http://wikiba.se/ontology#Math': 'math',\n",
    "                'http://wikiba.se/ontology#Monolingualtext': 'monolingualtext',\n",
    "                'http://wikiba.se/ontology#Quantity': 'quantity',\n",
    "                'http://wikiba.se/ontology#String': 'string',\n",
    "                'http://wikiba.se/ontology#TabularData': 'tabular-data',\n",
    "                'http://wikiba.se/ontology#Time': 'time',\n",
    "                'http://wikiba.se/ontology#Url': 'url',\n",
    "                'http://wikiba.se/ontology#WikibaseItem': 'wikibase-item',\n",
    "                'http://wikiba.se/ontology#WikibaseLexeme': 'lexeme',\n",
    "                'http://wikiba.se/ontology#WikibaseForm': 'form',\n",
    "                'http://wikiba.se/ontology#WikibaseSense': 'sense',\n",
    "                'http://wikiba.se/ontology#MusicalNotation': 'musical-notation',\n",
    "                'http://wikiba.se/ontology#WikibaseProperty': 'wikibase-property'}\n",
    "propertiesSparql['datatype']= \"\"\n",
    "for index, row in propertiesSparql.iterrows():\n",
    "  row[\"datatype\"] = datatype_map[row[\"propType\"]] \n",
    "propertiesSparql"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "propertiesSparql.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Login to Wikibase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "from getpass import getpass\n",
    "import pprint\n",
    "\n",
    "wikibase = \"https://demowb2.wiki.opencura.com/\"\n",
    "api = \"http://localhost:8181/w/api.php\"\n",
    "sparql = \"http://localhost:8282/proxy/wdqs/bigdata/namespace/wdq/sparql\"\n",
    "entityUri = wikibase.replace(\"https:\", \"http:\")+\"entity/\"\n",
    "\n",
    "WBUSER = \"WikibaseAdmin\"\n",
    "WBPASS = \"WikibaseDockerAdminPass\"\n",
    "login = wdi_login.WDLogin(WBUSER, WBPASS, mediawiki_api_url=api)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "#### Retrieve all properties from wikibase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def get_properties():\n",
    "    property_lookup = {}\n",
    "\n",
    "    query = \"\"\"\n",
    "    SELECT ?property ?label\n",
    "    WHERE {\n",
    "        ?property a wikibase:Property .\n",
    "        ?property rdfs:label ?label .\n",
    "        FILTER (LANG(?label) = \"en\" )\n",
    "    }\"\"\"\n",
    "\n",
    "    results = wdi_core.WDItemEngine.execute_sparql_query(query=query, endpoint=sparql)\n",
    "\n",
    "    for result in results[\"results\"][\"bindings\"]:\n",
    "        label = result[\"label\"][\"value\"].split(\"/\")[-1]\n",
    "        property_lookup[label] = result[\"property\"][\"value\"].split(\"/\")[-1]\n",
    "\n",
    "    return property_lookup\n",
    "\n",
    "# Dictionary key = name of property, value = Pxxx\n",
    "property_lookup = get_properties()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Function to create properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def createProperty(login=login, wdprop=None, label=\"\", description=\"\", property_datatype=\"\"):\n",
    "  if wdprop== None:\n",
    "    s = []\n",
    "  else:\n",
    "    s = [wdi_core.WDUrl(wdprop, prop_nr=\"P1\")]\n",
    "  localEntityEngine = wdi_core.WDItemEngine.wikibase_item_engine_factory(api,sparql)\n",
    "  item = localEntityEngine(data=s)\n",
    "  item.set_label(label)\n",
    "  item.set_description(description)\n",
    "  print(item.write(login, entity_type=\"property\", property_datatype=property_datatype))\n",
    "\n",
    "#create mapping to wikidata property\n",
    "if \"property in wikidata\" not in property_lookup:\n",
    "  createProperty(login, label=\"property in wikidata\", description=\"The same property in Wikidata\", property_datatype=\"url\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Wikidata properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "datatype_map = {'http://wikiba.se/ontology#CommonsMedia': 'commonsMedia',\n",
    "                'http://wikiba.se/ontology#ExternalId': 'external-id',\n",
    "                'http://wikiba.se/ontology#GeoShape': 'geo-shape',\n",
    "                'http://wikiba.se/ontology#GlobeCoordinate': 'globe-coordinate',\n",
    "                'http://wikiba.se/ontology#Math': 'math',\n",
    "                'http://wikiba.se/ontology#Monolingualtext': 'monolingualtext',\n",
    "                'http://wikiba.se/ontology#Quantity': 'quantity',\n",
    "                'http://wikiba.se/ontology#String': 'string',\n",
    "                'http://wikiba.se/ontology#TabularData': 'tabular-data',\n",
    "                'http://wikiba.se/ontology#Time': 'time',\n",
    "                'http://wikiba.se/ontology#Url': 'url',\n",
    "                'http://wikiba.se/ontology#WikibaseItem': 'wikibase-item',\n",
    "                'http://wikiba.se/ontology#WikibaseProperty': 'wikibase-property'}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed\n",
    "import multiprocessing\n",
    "     \n",
    "# what are your inputs, and what operation do you want to \n",
    "# perform on each input. For example...\n",
    "inputs = range(10) \n",
    "def createPropertyStage(row, property_lookup):\n",
    "    if row.propertyLabel in property_lookup:\n",
    "        pass\n",
    "    else:\n",
    "        try:\n",
    "            createProperty(login=login, wdprop=row.property, label=row.propertyLabel, description=row.propertyDescription, property_datatype=datatype_map[row.propType])\n",
    "        except:\n",
    "            print(\"Failed\")\n",
    " \n",
    "num_cores = multiprocessing.cpu_count()\n",
    "     \n",
    "results = Parallel(n_jobs=num_cores)(delayed(createPropertyStage)(i, property_lookup) for i in propertiesSparql.itertuples())"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "CreateWikidataProperties.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}