wandb_setup.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Usage\n",
    "\n",
    "```python\n",
    "api = wandb.Api() \\\n",
    "artifact = api.artifact('ire-shshsh/mdes/multi_news:v0', type='dataset') \\\n",
    "path_to_file = artifact.get_path('train.csv').download('test')\n",
    "```\n",
    "\n",
    "- `train.csv` is the file you want to use\n",
    "- `test` is the directory you want to download the file to\n",
    "- You do not need to give a download directory. It will create one on its own"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wandb\n",
    "wandb.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "run = wandb.init(entity='ire-shshsh', job_type='upload-dataset')\n",
    "artifact = wandb.Artifact(name='multi_news', type='dataset')\n",
    "artifact.add_dir('./dataset_handling/multi_news/')\n",
    "run.log_artifact(artifact)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "api = wandb.Api()\n",
    "artifact = api.artifact('ire-shshsh/mdes/multi_news:v0', type='dataset')\n",
    "artifact_dir = artifact.get_path('train.csv').download('test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "artifact = wandb.Artifact(name='multi_x_science', type='dataset')\n",
    "artifact.add_dir('./dataset_handling/multi_x_science/')\n",
    "run.log_artifact(artifact)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "for file in os.listdir('./clustering/tf_idf_method_2'):\n",
    "    if file.endswith('.csv'):\n",
    "        print(f'multi_news_modified_{file[16:-4]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#go through all the .csv files in directory mmr\n",
    "import os\n",
    "\n",
    "for file in os.listdir('./clustering/tf_idf_method_2/'):\n",
    "    if file.endswith('.csv'):\n",
    "        artifact = wandb.Artifact(name=f'multi_news_modified_{file[16:-4]}', type='dataset')\n",
    "        artifact.add_file('./clustering/tf_idf_method_2/' + file)\n",
    "        run.log_artifact(artifact)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "main",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}