webis_import.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Webis Abstractive Snippet Corpus 2020\n",
    "\n",
    "https://zenodo.org/record/3653834#.Y4fccDPMIc4\n",
    "\n",
    "- prepared by MaxM\n",
    "- paper: [Abstractive Snippet Generation](https://arxiv.org/abs/2002.10782)\n",
    "\n",
    "Unfortunately the data quality is very bad, so we will probably not use it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/media/koepf/data2/laion/laion_sum_data\n",
      "reading:  ../webis/webis_grouped.parquet\n",
      "num_row_groups: 359\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "import pyarrow.parquet as pq\n",
    "import pyarrow as pa\n",
    "\n",
    "print(Path.cwd())\n",
    "\n",
    "fn = Path('../webis/webis_grouped.parquet')\n",
    "\n",
    "print('reading: ', fn)\n",
    "pf = pq.ParquetFile(fn)\n",
    "print('num_row_groups:', pf.num_row_groups)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "writing part 0: 400000 rows\n",
      "writing part 1: 400000 rows\n",
      "writing part 2: 400000 rows\n",
      "writing part 3: 400000 rows\n",
      "writing part 4: 400000 rows\n",
      "writing part 5: 400000 rows\n",
      "writing part 6: 400000 rows\n",
      "writing part 7: 400000 rows\n",
      "writing part 8: 389701 rows\n"
     ]
    }
   ],
   "source": [
    "chunk_size = 400000\n",
    "prefix = 'webis'\n",
    "compression = 'snappy'\n",
    "\n",
    "output_dir = Path('data/webis')\n",
    "output_dir.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "def write_part(df, output_dir, prefix, part, compression):\n",
    "    fn = output_dir / f'{prefix}_{part:05d}.{compression}.parquet'\n",
    "    print(f'writing part {part}: {len(df)} rows')\n",
    "    df.to_parquet(fn, compression=compression, engine=\"pyarrow\", row_group_size=1000)\n",
    "\n",
    "frames = []\n",
    "row_count = 0\n",
    "\n",
    "part = 0\n",
    "for x in pf.iter_batches(batch_size=10000):\n",
    "    df_part = x.to_pandas()\n",
    "    provenance = pd.array([json.dumps({ 'src': prefix, 'doi': '10.5281/zenodo.3653834' })] * len(df_part), dtype=\"string\")\n",
    "    df_part[\"provenance\"] = provenance\n",
    "    row_count += len(df_part)\n",
    "    frames.append(df_part)\n",
    "\n",
    "    if row_count >= chunk_size:\n",
    "        df = pd.concat(frames, ignore_index=True)\n",
    "        write_part(df, output_dir, prefix, part, compression)\n",
    "        frames = []\n",
    "        row_count = 0\n",
    "        part += 1\n",
    "\n",
    "if row_count > 0:\n",
    "    df = pd.concat(frames, ignore_index=True)\n",
    "    write_part(df, output_dir, prefix, part, compression)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400000\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_parquet(path=str(output_dir / 'webis_00003.snappy.parquet'), columns=['text', 'summary'])\n",
    "print(len(df))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text: signature our commitment to our associates goes beyond just growing their professional skills . for many of our team members , working in one of our restaurants is often their first job . so we strive to develop their life skills foundational abilities such as teamwork , responsibility , problem solving , positive energy , and a relentless drive to serve and satisfy customers so they can grow to their full capability , be successful in whatever they do , and achieve their dreams . world hunger relief is our signature global program to address hunger and hopelessness . our partnership with the world food programme has succeeded in ways we never imagined possible . through it we have gone beyond making a contribution to relieving hunger and we have built momentum for ongoing volunteerism and community engagement globally by giving our associates the tools , opportunities and encouragement to place themselves at the heart of pressing societal issues . since world hunger relief movement launched in <NUM> , more than <NUM> million of the company employees , franchisees and their families have volunteered more than <NUM> million hours to aid hunger relief efforts in communities worldwide . the movement has raised nearly <NUM> million for the wfp and other hunger relief organizations and is helping to provide over <NUM> million meals and save the lives of millions of people in more than <NUM> countries where hunger is most prevalent . at the clinton global initiative in <NUM> , yum ! brands pledged to do the following over the next few years raise and donate at least <NUM> million to help wfp and others provide <NUM> million meals for hungry school children in developing countries donate <NUM> million hours of hunger relief volunteer service in the communities in which it operates donate <NUM> million worth of its prepared food to hunger agencies in the united states and use the company marketing clout to generate awareness of the hunger problem , and convince others to become part of the solution .\n",
      "summary: their . hunger on focus also , hut pizza and , chicken fried kentucky , bell taco oversees who , brand yum thesignatureworld hunger relief program wfp in partnership with un world program was launched in <NUM> . company employees have volunteered <NUM> million hours to aid hunger relief worldwide .\n"
     ]
    }
   ],
   "source": [
    "print('text:', df['text'][1500])\n",
    "print('summary:', df['summary'][1500])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.10 ('news_venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "52837886e5e45ddaef7e1045db27270fbeb0fd633b908ac870aa34fbd32295eb"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}