1. Download and collect data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!wget https://www2.informatik.uni-hamburg.de/nats/pub/SWC/SWC_Dutch_no_audio.tar.xz -O data/raw/nl_wiki_text.tar.xz\n",
    "!tar -xf data/raw/nl_wiki_text.tar.xz -C data/raw/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_path = []\n",
    "\n",
    "for dirname, _, filenames in os.walk('./data/raw/dutch'):\n",
    "    for filename in filenames:\n",
    "        if filename == 'audiometa.txt':\n",
    "            interested_file = os.path.join(dirname, filename)\n",
    "            text_path.append(interested_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"There are {len(text_path)} files with interesting documents.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_list = []\n",
    "for file in text_path:\n",
    "    with open(file, 'r') as myfile:\n",
    "        data = myfile.read()\n",
    "        text_list.append(data)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'text' : text_list})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(\"./data/processed/df_wiki_text.pickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.4 64-bit ('datascience': conda)",
   "language": "python",
   "name": "python37464bitdatasciencecondaa6d3a21c00574213ab53e74f2ef82285"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}