02alignmentsSpark.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compute alignments using Wikidata\n",
    "\n",
    "This notebook create alignment matrices to be applied for align two different word embeddings.\n",
    "The matrices are created using [this approach](https://github.com/Babylonpartners/fastText_multilingual/blob/master/align_your_own.ipynb), but instead of using bilingual dictionaries we use the Wikidata labels.\n",
    "Here we use a parquet dump clo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os \n",
    "from pyspark.sql.functions import regexp_replace\n",
    "\n",
    "\n",
    "from fastText_multilingual.fasttext import FastVector\n",
    "\n",
    "# from https://stackoverflow.com/questions/21030391/how-to-normalize-array-numpy\n",
    "def normalized(a, axis=-1, order=2):\n",
    "    \"\"\"Utility function to normalize the rows of a numpy array.\"\"\"\n",
    "    l2 = np.atleast_1d(np.linalg.norm(a, order, axis))\n",
    "    l2[l2==0] = 1\n",
    "    return a / np.expand_dims(l2, axis)\n",
    "\n",
    "def make_training_matrices(source_dictionary, target_dictionary, bilingual_dictionary):\n",
    "    \"\"\"\n",
    "    Source and target dictionaries are the FastVector objects of\n",
    "    source/target languages. bilingual_dictionary is a list of \n",
    "    translation pair tuples [(source_word, target_word), ...].\n",
    "    \"\"\"\n",
    "    source_matrix = []\n",
    "    target_matrix = []\n",
    "\n",
    "    for (source, target) in bilingual_dictionary:\n",
    "        try:\n",
    "            source = source.lower().split()\n",
    "            sourceVector = np.zeros(300) + sum([source_dictionary[word] for word in source  if word in source_dictionary])/len(source)\n",
    "            target = target.lower().split()\n",
    "            targetVector = np.zeros(300) + sum([target_dictionary[word] for word in target  if word in target_dictionary])/len(target)\n",
    "            if (sourceVector.all() !=0) and (targetVector.all() != 0):\n",
    "                    source_matrix.append(sourceVector)\n",
    "                    target_matrix.append(targetVector)\n",
    "        except:\n",
    "            pass\n",
    "    # return training matrices\n",
    "    return np.array(source_matrix), np.array(target_matrix)\n",
    "\n",
    "def learn_transformation(source_matrix, target_matrix, normalize_vectors=True):\n",
    "    \"\"\"\n",
    "    Source and target matrices are numpy arrays, shape\n",
    "    (dictionary_length, embedding_dimension). These contain paired\n",
    "    word vectors from the bilingual dictionary.\n",
    "    \"\"\"\n",
    "    # optionally normalize the training vectors\n",
    "    if normalize_vectors:\n",
    "        source_matrix = normalized(source_matrix)\n",
    "        target_matrix = normalized(target_matrix)\n",
    "\n",
    "    # perform the SVD\n",
    "    product = np.matmul(source_matrix.transpose(), target_matrix)\n",
    "    U, s, V = np.linalg.svd(product)\n",
    "\n",
    "    # return orthogonal transformation which aligns source language to the target\n",
    "    return np.matmul(U, V)\n",
    "\n",
    "\n",
    "## Prepare folder\n",
    "outputFolder = 'my_alingments'\n",
    "if not os.path.exists(outputFolder):\n",
    "    os.makedirs(outputFolder)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are using wmfdata v1.2, but v1.3 is available.\n",
      "\n",
      "To update, run `pip install --upgrade git+https://github.com/wikimedia/wmfdata-python.git@release`.\n",
      "\n",
      "To see the changes, refer to https://github.com/wikimedia/wmfdata-python/blob/release/CHANGELOG.md\n",
      "PySpark executors will use /usr/lib/anaconda-wmf/bin/python3.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "##### replaced by the code below\n",
    "# get wikidata data\n",
    "\n",
    "\n",
    "#df = spark.read.parquet('/user/joal/wmf/data/wmf/wikidata/item_page_link/20190204')\n",
    "#df = df[df['page_namespace'] == 0]\n",
    "#df = df.withColumn('page', regexp_replace('page_title', '_', ' '))\n",
    "#df = df.select('wiki_db','item_id','page')\n",
    "\n",
    "import wmfdata\n",
    "spark = wmfdata.spark.create_session('regular')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+\n",
      "|          partition|\n",
      "+-------------------+\n",
      "|snapshot=2021-11-15|\n",
      "|snapshot=2021-11-22|\n",
      "|snapshot=2021-11-29|\n",
      "|snapshot=2021-12-06|\n",
      "|snapshot=2021-12-13|\n",
      "|snapshot=2021-12-20|\n",
      "+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Check the last partition of table wmf.wikidata_item_page_link\n",
    "spark.sql('SHOW PARTITIONS wmf.wikidata_item_page_link').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.functions import regexp_replace\n",
    "\n",
    "last_partition = '2021-12-20'\n",
    "df = spark.sql('''SELECT wiki_db,page_title,item_id \n",
    "                FROM wmf.wikidata_item_page_link\n",
    "                WHERE snapshot = \"{0}\"\n",
    "'''.format(last_partition))\n",
    "df = df.withColumn('page', regexp_replace('page_title', '_', ' '))\n",
    "df = df.select('wiki_db','item_id','page')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "en\n",
      "reading word vectors from vectors/wiki.en.vec\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "reading word vectors from vectors/wiki.he.vec\n",
      "== he\n",
      "reading word vectors from vectors/wiki.ca.vec\n",
      "== ca\n",
      "reading word vectors from vectors/wiki.pt.vec\n",
      "== pt\n",
      "reading word vectors from vectors/wiki.ar.vec\n",
      "== ar\n",
      "reading word vectors from vectors/wiki.it.vec\n",
      "== it\n",
      "reading word vectors from vectors/wiki.uk.vec\n",
      "== uk\n",
      "reading word vectors from vectors/wiki.es.vec\n",
      "== es\n",
      "reading word vectors from vectors/wiki.fr.vec\n",
      "== fr\n",
      "reading word vectors from vectors/wiki.ru.vec\n",
      "== ru\n",
      "ru\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "reading word vectors from vectors/wiki.he.vec\n",
      "== he\n",
      "reading word vectors from vectors/wiki.ca.vec\n",
      "== ca\n",
      "reading word vectors from vectors/wiki.pt.vec\n",
      "== pt\n",
      "reading word vectors from vectors/wiki.ar.vec\n",
      "== ar\n",
      "reading word vectors from vectors/wiki.it.vec\n",
      "== it\n",
      "reading word vectors from vectors/wiki.uk.vec\n",
      "== uk\n",
      "reading word vectors from vectors/wiki.es.vec\n",
      "== es\n",
      "reading word vectors from vectors/wiki.fr.vec\n",
      "== fr\n",
      "fr\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "reading word vectors from vectors/wiki.he.vec\n",
      "== he\n",
      "reading word vectors from vectors/wiki.ca.vec\n",
      "== ca\n",
      "reading word vectors from vectors/wiki.pt.vec\n",
      "== pt\n",
      "reading word vectors from vectors/wiki.ar.vec\n",
      "== ar\n",
      "reading word vectors from vectors/wiki.it.vec\n",
      "== it\n",
      "reading word vectors from vectors/wiki.uk.vec\n",
      "== uk\n",
      "reading word vectors from vectors/wiki.es.vec\n",
      "== es\n",
      "es\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "reading word vectors from vectors/wiki.he.vec\n",
      "== he\n",
      "reading word vectors from vectors/wiki.ca.vec\n",
      "== ca\n",
      "reading word vectors from vectors/wiki.pt.vec\n",
      "== pt\n",
      "reading word vectors from vectors/wiki.ar.vec\n",
      "== ar\n",
      "reading word vectors from vectors/wiki.it.vec\n",
      "== it\n",
      "reading word vectors from vectors/wiki.uk.vec\n",
      "== uk\n",
      "uk\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "reading word vectors from vectors/wiki.he.vec\n",
      "== he\n",
      "reading word vectors from vectors/wiki.ca.vec\n",
      "== ca\n",
      "reading word vectors from vectors/wiki.pt.vec\n",
      "== pt\n",
      "reading word vectors from vectors/wiki.ar.vec\n",
      "== ar\n",
      "reading word vectors from vectors/wiki.it.vec\n",
      "== it\n",
      "it\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "reading word vectors from vectors/wiki.he.vec\n",
      "== he\n",
      "reading word vectors from vectors/wiki.ca.vec\n",
      "== ca\n",
      "reading word vectors from vectors/wiki.pt.vec\n",
      "== pt\n",
      "reading word vectors from vectors/wiki.ar.vec\n",
      "== ar\n",
      "ar\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "reading word vectors from vectors/wiki.he.vec\n",
      "== he\n",
      "reading word vectors from vectors/wiki.ca.vec\n",
      "== ca\n",
      "reading word vectors from vectors/wiki.pt.vec\n",
      "== pt\n",
      "pt\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "reading word vectors from vectors/wiki.he.vec\n",
      "== he\n",
      "reading word vectors from vectors/wiki.ca.vec\n",
      "== ca\n",
      "ca\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "reading word vectors from vectors/wiki.he.vec\n",
      "== he\n",
      "he\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "reading word vectors from vectors/wiki.fa.vec\n",
      "== fa\n",
      "fa\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "reading word vectors from vectors/wiki.zh.vec\n",
      "== zh\n",
      "zh\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "reading word vectors from vectors/wiki.id.vec\n",
      "== id\n",
      "id\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "reading word vectors from vectors/wiki.vi.vec\n",
      "== vi\n",
      "vi\n",
      "reading word vectors from vectors/wiki.ta.vec\n",
      "== ta\n",
      "ta\n"
     ]
    }
   ],
   "source": [
    "import glob,os\n",
    "vectors = sorted(glob.glob('vectors/wiki.*.vec'), key=os.path.getsize) #sorted by size to load the largest files just once\n",
    "lang2 = ''\n",
    "while vectors:   \n",
    "    lang1 = vectors.pop()\n",
    "    lang1_code = lang1.split('.')[1]\n",
    "    print(lang1_code)\n",
    "    if lang1 == lang2:\n",
    "        lang1_dictionary = lang2_dictionary\n",
    "    else:\n",
    "        lang1_dictionary = FastVector(vector_file=lang1)\n",
    "    for lang2 in vectors:\n",
    "        lang2_dictionary = FastVector(vector_file=lang2)\n",
    "        lang2_code = lang2.split('.')[1]\n",
    "        print('==',lang2_code)\n",
    "        df2 = df[df.wiki_db == '%swiki' % lang1_code].join(df[df.wiki_db == '%swiki' % lang2_code].withColumnRenamed(\"page\", \"page2\").withColumnRenamed('wiki_db','wiki_db2'),on='item_id')\n",
    "        pairs = df2.toPandas()       \n",
    "        bilingual_dictionary = list(zip(pairs['page'],pairs['page2']))\n",
    "        ##common words\n",
    "        #lang1_words = set(lang1_dictionary.word2id.keys()lang1_dictionary.word2id.keys())\n",
    "        #lang2_words = set(lang2_dictionary.word2id.keys())\n",
    "        #overlap = list(lang1_words & lang2_words)\n",
    "        #bilingual_dictionary.extend([(entry, entry) for entry in overlap])\n",
    "        # form the training matrices\n",
    "        source_matrix, target_matrix = make_training_matrices(lang1_dictionary, lang2_dictionary, bilingual_dictionary)\n",
    "        # learn and apply the transformation\n",
    "        transform = learn_transformation(source_matrix, target_matrix)\n",
    "        with open('%s/apply_in_%s_to_%s.txt' % (outputFolder,lang1_code,lang2_code),'w') as f:\n",
    "            np.savetxt(f, transform)\n",
    "        bilingual_dictionary = [(y,x) for x,y in bilingual_dictionary] #reverse pairs\n",
    "        # form the training matrices\n",
    "        source_matrix, target_matrix = make_training_matrices(lang2_dictionary, lang1_dictionary, bilingual_dictionary)\n",
    "        # learn and apply the transformation\n",
    "        transform = learn_transformation(source_matrix, target_matrix)\n",
    "        with open('%s/apply_in_%s_to_%s.txt' % (outputFolder,lang2_code,lang1_code),'w') as f:\n",
    "            np.savetxt(f, transform)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}