stats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: datasets in /home/zaid/.local/lib/python3.9/site-packages (2.14.4)\n",
      "Requirement already satisfied: pandas in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (2.0.3)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/zaid/anaconda3/envs/evals/lib/python3.9/site-packages (from datasets) (2.30.0)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (2023.6.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (0.16.4)\n",
      "Requirement already satisfied: packaging in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (21.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (1.25.2)\n",
      "Requirement already satisfied: xxhash in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (3.3.0)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/zaid/anaconda3/envs/evals/lib/python3.9/site-packages (from datasets) (6.0)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (4.65.0)\n",
      "Requirement already satisfied: multiprocess in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (0.70.15)\n",
      "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (0.3.7)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (12.0.1)\n",
      "Requirement already satisfied: aiohttp in /home/zaid/.local/lib/python3.9/site-packages (from datasets) (3.8.5)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/zaid/.local/lib/python3.9/site-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/zaid/.local/lib/python3.9/site-packages (from aiohttp->datasets) (1.9.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/zaid/anaconda3/envs/evals/lib/python3.9/site-packages (from aiohttp->datasets) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/zaid/.local/lib/python3.9/site-packages (from aiohttp->datasets) (1.4.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/zaid/.local/lib/python3.9/site-packages (from aiohttp->datasets) (4.0.2)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /home/zaid/.local/lib/python3.9/site-packages (from aiohttp->datasets) (3.2.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/zaid/.local/lib/python3.9/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: filelock in /home/zaid/.local/lib/python3.9/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.12.2)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/zaid/.local/lib/python3.9/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.4.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/zaid/.local/lib/python3.9/site-packages (from packaging->datasets) (3.0.9)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/zaid/anaconda3/envs/evals/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (2023.5.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/zaid/anaconda3/envs/evals/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (1.26.15)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/zaid/anaconda3/envs/evals/lib/python3.9/site-packages (from requests>=2.19.0->datasets) (3.4)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /home/zaid/.local/lib/python3.9/site-packages (from pandas->datasets) (2023.3)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/zaid/.local/lib/python3.9/site-packages (from pandas->datasets) (2023.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/zaid/.local/lib/python3.9/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /home/zaid/anaconda3/envs/evals/lib/python3.9/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "cidar = load_dataset(\"arbml/cidar\")\n",
    "alpagasus = load_dataset(\"arbml/alpagasus_cleaned_ar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_dataset(data, keyword, pre = 'cidar'):\n",
    "    open(f'keywords/{pre}_{keyword}.txt', 'w').write('\\n*********\\n'.join([x['instruction']+x['output'] for x in data]))\n",
    "\n",
    "def extract_by_keyword(keyword):\n",
    "    global cidar, alpagasus\n",
    "    c = cidar.filter(lambda x: keyword in x['instruction']+x['output'])['train']\n",
    "    a = alpagasus.filter(lambda x: keyword in x['instruction']+x['output'])['train']\n",
    "    print(len(c))\n",
    "    print(len(a))\n",
    "    save_dataset(c, keyword, pre = 'cidar')\n",
    "    save_dataset(a, keyword, pre = 'alpagasus')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a151f5301294349a5b4f2a054348003",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filter:   0%|          | 0/10000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "227a2c8c019b4effb702f5f323863225",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Filter:   0%|          | 0/9229 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "118\n",
      "71\n"
     ]
    }
   ],
   "source": [
    "extract_by_keyword('تشكيل')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "تشكيل\n",
      "26\n",
      "0\n",
      "إعراب\n",
      "246\n",
      "0\n",
      "ترجم\n",
      "137\n",
      "121\n"
     ]
    }
   ],
   "source": [
    "keywords = ['تشكيل', 'إعراب', 'ترجم']\n",
    "for keyword in keywords:\n",
    "    print(keyword)\n",
    "    data = open(f'keywords/cidar_{keyword}.txt').read().split('\\n*********\\n')\n",
    "    print(len(data) - 1)\n",
    "    data = open(f'keywords/alpagasus_{keyword}.txt').read().split('\\n*********\\n')\n",
    "    print(len(data) - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "evals",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}