test_teensy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/michaelnath/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from Modules.Code2Code.Extracontent.code_snippet_dataset import CodeSnippetDataset\n",
    "from Modules.Code2Explanation.code2doc import Code2DocModule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration michaelnath--annotated-code-functions-base-109e1212a74aeb76\n",
      "Found cached dataset parquet (/Users/michaelnath/.cache/huggingface/datasets/michaelnath___parquet/michaelnath--annotated-code-functions-base-109e1212a74aeb76/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n",
      "/Users/michaelnath/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/models/auto/modeling_auto.py:1248: FutureWarning: The class `AutoModelWithLMHead` is deprecated and will be removed in a future version. Please use `AutoModelForCausalLM` for causal language models, `AutoModelForMaskedLM` for masked language models and `AutoModelForSeq2SeqLM` for encoder-decoder models.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Got snippets!\n",
      "Begin processing 100 functions!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Your max_length is set to 20, but you input_length is only 15. You might consider decreasing max_length manually, e.g. summarizer('...', max_length=7)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch 1 done!\n",
      "Batch 2 done!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Your max_length is set to 20, but you input_length is only 14. You might consider decreasing max_length manually, e.g. summarizer('...', max_length=7)\n",
      "Your max_length is set to 20, but you input_length is only 16. You might consider decreasing max_length manually, e.g. summarizer('...', max_length=8)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch 3 done!\n",
      "Batch 4 done!\n",
      "Batch 5 done!\n",
      "Batch 6 done!\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 10\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[39m# Get documentation from dataset\u001b[39;00m\n\u001b[1;32m      9\u001b[0m code2doc \u001b[39m=\u001b[39m Code2DocModule(code_snippets)\n\u001b[0;32m---> 10\u001b[0m data_with_docs \u001b[39m=\u001b[39m code2doc\u001b[39m.\u001b[39;49mget_docs()\n\u001b[1;32m     12\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mGot documentations!\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "File \u001b[0;32m~/program/stanford/cs224n/CodeSage/Modules/Code2Explanation/code2doc.py:28\u001b[0m, in \u001b[0;36mCode2DocModule.get_docs\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mBegin processing \u001b[39m\u001b[39m{}\u001b[39;00m\u001b[39m functions!\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39mformat(\u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msnippets[\u001b[39m\"\u001b[39m\u001b[39mfunction\u001b[39m\u001b[39m\"\u001b[39m])))\n\u001b[1;32m     27\u001b[0m \u001b[39mfor\u001b[39;00m i \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(\u001b[39m1\u001b[39m, math\u001b[39m.\u001b[39mceil(\u001b[39mlen\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msnippets[\u001b[39m\"\u001b[39m\u001b[39mfunction\u001b[39m\u001b[39m\"\u001b[39m]) \u001b[39m/\u001b[39m batch_size)):\n\u001b[0;32m---> 28\u001b[0m   \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmodel(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msnippets[\u001b[39m\"\u001b[39;49m\u001b[39mfunction\u001b[39;49m\u001b[39m\"\u001b[39;49m][(i \u001b[39m-\u001b[39;49m \u001b[39m1\u001b[39;49m) \u001b[39m*\u001b[39;49m batch_size: i \u001b[39m*\u001b[39;49m batch_size])\n\u001b[1;32m     29\u001b[0m   \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mBatch \u001b[39m\u001b[39m{\u001b[39;00mi\u001b[39m}\u001b[39;00m\u001b[39m done!\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     30\u001b[0m \u001b[39m# responses = self.model(self.snippets[\"function\"])\u001b[39;00m\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/pipelines/text2text_generation.py:265\u001b[0m, in \u001b[0;36mSummarizationPipeline.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    241\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__call__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    242\u001b[0m \u001b[39m    \u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    243\u001b[0m \u001b[39m    Summarize the text(s) given as inputs.\u001b[39;00m\n\u001b[1;32m    244\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    263\u001b[0m \u001b[39m          ids of the summary.\u001b[39;00m\n\u001b[1;32m    264\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 265\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49m\u001b[39m__call__\u001b[39;49m(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/pipelines/text2text_generation.py:165\u001b[0m, in \u001b[0;36mText2TextGenerationPipeline.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    136\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__call__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m    137\u001b[0m \u001b[39m    \u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m    138\u001b[0m \u001b[39m    Generate the output text(s) using text(s) given as inputs.\u001b[39;00m\n\u001b[1;32m    139\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    162\u001b[0m \u001b[39m          ids of the generated text.\u001b[39;00m\n\u001b[1;32m    163\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 165\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39;49m()\u001b[39m.\u001b[39;49m\u001b[39m__call__\u001b[39;49m(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m    166\u001b[0m     \u001b[39mif\u001b[39;00m (\n\u001b[1;32m    167\u001b[0m         \u001b[39misinstance\u001b[39m(args[\u001b[39m0\u001b[39m], \u001b[39mlist\u001b[39m)\n\u001b[1;32m    168\u001b[0m         \u001b[39mand\u001b[39;00m \u001b[39mall\u001b[39m(\u001b[39misinstance\u001b[39m(el, \u001b[39mstr\u001b[39m) \u001b[39mfor\u001b[39;00m el \u001b[39min\u001b[39;00m args[\u001b[39m0\u001b[39m])\n\u001b[1;32m    169\u001b[0m         \u001b[39mand\u001b[39;00m \u001b[39mall\u001b[39m(\u001b[39mlen\u001b[39m(res) \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m \u001b[39mfor\u001b[39;00m res \u001b[39min\u001b[39;00m result)\n\u001b[1;32m    170\u001b[0m     ):\n\u001b[1;32m    171\u001b[0m         \u001b[39mreturn\u001b[39;00m [res[\u001b[39m0\u001b[39m] \u001b[39mfor\u001b[39;00m res \u001b[39min\u001b[39;00m result]\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/pipelines/base.py:1065\u001b[0m, in \u001b[0;36mPipeline.__call__\u001b[0;34m(self, inputs, num_workers, batch_size, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1061\u001b[0m \u001b[39mif\u001b[39;00m can_use_iterator:\n\u001b[1;32m   1062\u001b[0m     final_iterator \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_iterator(\n\u001b[1;32m   1063\u001b[0m         inputs, num_workers, batch_size, preprocess_params, forward_params, postprocess_params\n\u001b[1;32m   1064\u001b[0m     )\n\u001b[0;32m-> 1065\u001b[0m     outputs \u001b[39m=\u001b[39m [output \u001b[39mfor\u001b[39;00m output \u001b[39min\u001b[39;00m final_iterator]\n\u001b[1;32m   1066\u001b[0m     \u001b[39mreturn\u001b[39;00m outputs\n\u001b[1;32m   1067\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/pipelines/base.py:1065\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   1061\u001b[0m \u001b[39mif\u001b[39;00m can_use_iterator:\n\u001b[1;32m   1062\u001b[0m     final_iterator \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_iterator(\n\u001b[1;32m   1063\u001b[0m         inputs, num_workers, batch_size, preprocess_params, forward_params, postprocess_params\n\u001b[1;32m   1064\u001b[0m     )\n\u001b[0;32m-> 1065\u001b[0m     outputs \u001b[39m=\u001b[39m [output \u001b[39mfor\u001b[39;00m output \u001b[39min\u001b[39;00m final_iterator]\n\u001b[1;32m   1066\u001b[0m     \u001b[39mreturn\u001b[39;00m outputs\n\u001b[1;32m   1067\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/pipelines/pt_utils.py:124\u001b[0m, in \u001b[0;36mPipelineIterator.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    121\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mloader_batch_item()\n\u001b[1;32m    123\u001b[0m \u001b[39m# We're out of items within a batch\u001b[39;00m\n\u001b[0;32m--> 124\u001b[0m item \u001b[39m=\u001b[39m \u001b[39mnext\u001b[39;49m(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49miterator)\n\u001b[1;32m    125\u001b[0m processed \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minfer(item, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mparams)\n\u001b[1;32m    126\u001b[0m \u001b[39m# We now have a batch of \"inferred things\".\u001b[39;00m\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/pipelines/pt_utils.py:125\u001b[0m, in \u001b[0;36mPipelineIterator.__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    123\u001b[0m \u001b[39m# We're out of items within a batch\u001b[39;00m\n\u001b[1;32m    124\u001b[0m item \u001b[39m=\u001b[39m \u001b[39mnext\u001b[39m(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39miterator)\n\u001b[0;32m--> 125\u001b[0m processed \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49minfer(item, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mparams)\n\u001b[1;32m    126\u001b[0m \u001b[39m# We now have a batch of \"inferred things\".\u001b[39;00m\n\u001b[1;32m    127\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mloader_batch_size \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    128\u001b[0m     \u001b[39m# Try to infer the size of the batch\u001b[39;00m\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/pipelines/base.py:992\u001b[0m, in \u001b[0;36mPipeline.forward\u001b[0;34m(self, model_inputs, **forward_params)\u001b[0m\n\u001b[1;32m    990\u001b[0m     \u001b[39mwith\u001b[39;00m inference_context():\n\u001b[1;32m    991\u001b[0m         model_inputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ensure_tensor_on_device(model_inputs, device\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdevice)\n\u001b[0;32m--> 992\u001b[0m         model_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_forward(model_inputs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mforward_params)\n\u001b[1;32m    993\u001b[0m         model_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ensure_tensor_on_device(model_outputs, device\u001b[39m=\u001b[39mtorch\u001b[39m.\u001b[39mdevice(\u001b[39m\"\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m\"\u001b[39m))\n\u001b[1;32m    994\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/pipelines/text2text_generation.py:187\u001b[0m, in \u001b[0;36mText2TextGenerationPipeline._forward\u001b[0;34m(self, model_inputs, **generate_kwargs)\u001b[0m\n\u001b[1;32m    185\u001b[0m generate_kwargs[\u001b[39m\"\u001b[39m\u001b[39mmax_length\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m generate_kwargs\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mmax_length\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39mmax_length)\n\u001b[1;32m    186\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcheck_inputs(input_length, generate_kwargs[\u001b[39m\"\u001b[39m\u001b[39mmin_length\u001b[39m\u001b[39m\"\u001b[39m], generate_kwargs[\u001b[39m\"\u001b[39m\u001b[39mmax_length\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[0;32m--> 187\u001b[0m output_ids \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmodel\u001b[39m.\u001b[39;49mgenerate(\u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mmodel_inputs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mgenerate_kwargs)\n\u001b[1;32m    188\u001b[0m out_b \u001b[39m=\u001b[39m output_ids\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m]\n\u001b[1;32m    189\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mframework \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mpt\u001b[39m\u001b[39m\"\u001b[39m:\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/torch/autograd/grad_mode.py:27\u001b[0m, in \u001b[0;36m_DecoratorContextManager.__call__.<locals>.decorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[39m@functools\u001b[39m\u001b[39m.\u001b[39mwraps(func)\n\u001b[1;32m     25\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdecorate_context\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[1;32m     26\u001b[0m     \u001b[39mwith\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mclone():\n\u001b[0;32m---> 27\u001b[0m         \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39;49margs, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/generation/utils.py:1391\u001b[0m, in \u001b[0;36mGenerationMixin.generate\u001b[0;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, **kwargs)\u001b[0m\n\u001b[1;32m   1385\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[1;32m   1386\u001b[0m             \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mnum_return_sequences has to be 1, but is \u001b[39m\u001b[39m{\u001b[39;00mgeneration_config\u001b[39m.\u001b[39mnum_return_sequences\u001b[39m}\u001b[39;00m\u001b[39m when doing\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m   1387\u001b[0m             \u001b[39m\"\u001b[39m\u001b[39m greedy search.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m   1388\u001b[0m         )\n\u001b[1;32m   1390\u001b[0m     \u001b[39m# 11. run greedy search\u001b[39;00m\n\u001b[0;32m-> 1391\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgreedy_search(\n\u001b[1;32m   1392\u001b[0m         input_ids,\n\u001b[1;32m   1393\u001b[0m         logits_processor\u001b[39m=\u001b[39;49mlogits_processor,\n\u001b[1;32m   1394\u001b[0m         stopping_criteria\u001b[39m=\u001b[39;49mstopping_criteria,\n\u001b[1;32m   1395\u001b[0m         pad_token_id\u001b[39m=\u001b[39;49mgeneration_config\u001b[39m.\u001b[39;49mpad_token_id,\n\u001b[1;32m   1396\u001b[0m         eos_token_id\u001b[39m=\u001b[39;49mgeneration_config\u001b[39m.\u001b[39;49meos_token_id,\n\u001b[1;32m   1397\u001b[0m         output_scores\u001b[39m=\u001b[39;49mgeneration_config\u001b[39m.\u001b[39;49moutput_scores,\n\u001b[1;32m   1398\u001b[0m         return_dict_in_generate\u001b[39m=\u001b[39;49mgeneration_config\u001b[39m.\u001b[39;49mreturn_dict_in_generate,\n\u001b[1;32m   1399\u001b[0m         synced_gpus\u001b[39m=\u001b[39;49msynced_gpus,\n\u001b[1;32m   1400\u001b[0m         \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mmodel_kwargs,\n\u001b[1;32m   1401\u001b[0m     )\n\u001b[1;32m   1403\u001b[0m \u001b[39melif\u001b[39;00m is_contrastive_search_gen_mode:\n\u001b[1;32m   1404\u001b[0m     \u001b[39mif\u001b[39;00m generation_config\u001b[39m.\u001b[39mnum_return_sequences \u001b[39m>\u001b[39m \u001b[39m1\u001b[39m:\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/generation/utils.py:2179\u001b[0m, in \u001b[0;36mGenerationMixin.greedy_search\u001b[0;34m(self, input_ids, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, **model_kwargs)\u001b[0m\n\u001b[1;32m   2176\u001b[0m model_inputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mprepare_inputs_for_generation(input_ids, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mmodel_kwargs)\n\u001b[1;32m   2178\u001b[0m \u001b[39m# forward pass to get next token\u001b[39;00m\n\u001b[0;32m-> 2179\u001b[0m outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m(\n\u001b[1;32m   2180\u001b[0m     \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mmodel_inputs,\n\u001b[1;32m   2181\u001b[0m     return_dict\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[1;32m   2182\u001b[0m     output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m   2183\u001b[0m     output_hidden_states\u001b[39m=\u001b[39;49moutput_hidden_states,\n\u001b[1;32m   2184\u001b[0m )\n\u001b[1;32m   2186\u001b[0m \u001b[39mif\u001b[39;00m synced_gpus \u001b[39mand\u001b[39;00m this_peer_finished:\n\u001b[1;32m   2187\u001b[0m     \u001b[39mcontinue\u001b[39;00m  \u001b[39m# don't waste resources running the code we don't need\u001b[39;00m\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py:1663\u001b[0m, in \u001b[0;36mT5ForConditionalGeneration.forward\u001b[0;34m(self, input_ids, attention_mask, decoder_input_ids, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, inputs_embeds, decoder_inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1660\u001b[0m         decoder_attention_mask \u001b[39m=\u001b[39m decoder_attention_mask\u001b[39m.\u001b[39mto(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdecoder\u001b[39m.\u001b[39mfirst_device)\n\u001b[1;32m   1662\u001b[0m \u001b[39m# Decode\u001b[39;00m\n\u001b[0;32m-> 1663\u001b[0m decoder_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdecoder(\n\u001b[1;32m   1664\u001b[0m     input_ids\u001b[39m=\u001b[39;49mdecoder_input_ids,\n\u001b[1;32m   1665\u001b[0m     attention_mask\u001b[39m=\u001b[39;49mdecoder_attention_mask,\n\u001b[1;32m   1666\u001b[0m     inputs_embeds\u001b[39m=\u001b[39;49mdecoder_inputs_embeds,\n\u001b[1;32m   1667\u001b[0m     past_key_values\u001b[39m=\u001b[39;49mpast_key_values,\n\u001b[1;32m   1668\u001b[0m     encoder_hidden_states\u001b[39m=\u001b[39;49mhidden_states,\n\u001b[1;32m   1669\u001b[0m     encoder_attention_mask\u001b[39m=\u001b[39;49mattention_mask,\n\u001b[1;32m   1670\u001b[0m     head_mask\u001b[39m=\u001b[39;49mdecoder_head_mask,\n\u001b[1;32m   1671\u001b[0m     cross_attn_head_mask\u001b[39m=\u001b[39;49mcross_attn_head_mask,\n\u001b[1;32m   1672\u001b[0m     use_cache\u001b[39m=\u001b[39;49muse_cache,\n\u001b[1;32m   1673\u001b[0m     output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m   1674\u001b[0m     output_hidden_states\u001b[39m=\u001b[39;49moutput_hidden_states,\n\u001b[1;32m   1675\u001b[0m     return_dict\u001b[39m=\u001b[39;49mreturn_dict,\n\u001b[1;32m   1676\u001b[0m )\n\u001b[1;32m   1678\u001b[0m sequence_output \u001b[39m=\u001b[39m decoder_outputs[\u001b[39m0\u001b[39m]\n\u001b[1;32m   1680\u001b[0m \u001b[39m# Set device for model parallelism\u001b[39;00m\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py:1055\u001b[0m, in \u001b[0;36mT5Stack.forward\u001b[0;34m(self, input_ids, attention_mask, encoder_hidden_states, encoder_attention_mask, inputs_embeds, head_mask, cross_attn_head_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1042\u001b[0m     layer_outputs \u001b[39m=\u001b[39m checkpoint(\n\u001b[1;32m   1043\u001b[0m         create_custom_forward(layer_module),\n\u001b[1;32m   1044\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1052\u001b[0m         \u001b[39mNone\u001b[39;00m,  \u001b[39m# past_key_value is always None with gradient checkpointing\u001b[39;00m\n\u001b[1;32m   1053\u001b[0m     )\n\u001b[1;32m   1054\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m-> 1055\u001b[0m     layer_outputs \u001b[39m=\u001b[39m layer_module(\n\u001b[1;32m   1056\u001b[0m         hidden_states,\n\u001b[1;32m   1057\u001b[0m         attention_mask\u001b[39m=\u001b[39;49mextended_attention_mask,\n\u001b[1;32m   1058\u001b[0m         position_bias\u001b[39m=\u001b[39;49mposition_bias,\n\u001b[1;32m   1059\u001b[0m         encoder_hidden_states\u001b[39m=\u001b[39;49mencoder_hidden_states,\n\u001b[1;32m   1060\u001b[0m         encoder_attention_mask\u001b[39m=\u001b[39;49mencoder_extended_attention_mask,\n\u001b[1;32m   1061\u001b[0m         encoder_decoder_position_bias\u001b[39m=\u001b[39;49mencoder_decoder_position_bias,\n\u001b[1;32m   1062\u001b[0m         layer_head_mask\u001b[39m=\u001b[39;49mlayer_head_mask,\n\u001b[1;32m   1063\u001b[0m         cross_attn_layer_head_mask\u001b[39m=\u001b[39;49mcross_attn_layer_head_mask,\n\u001b[1;32m   1064\u001b[0m         past_key_value\u001b[39m=\u001b[39;49mpast_key_value,\n\u001b[1;32m   1065\u001b[0m         use_cache\u001b[39m=\u001b[39;49muse_cache,\n\u001b[1;32m   1066\u001b[0m         output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m   1067\u001b[0m     )\n\u001b[1;32m   1069\u001b[0m \u001b[39m# layer_outputs is a tuple with:\u001b[39;00m\n\u001b[1;32m   1070\u001b[0m \u001b[39m# hidden-states, key-value-states, (self-attention position bias), (self-attention weights), (cross-attention position bias), (cross-attention weights)\u001b[39;00m\n\u001b[1;32m   1071\u001b[0m \u001b[39mif\u001b[39;00m use_cache \u001b[39mis\u001b[39;00m \u001b[39mFalse\u001b[39;00m:\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py:713\u001b[0m, in \u001b[0;36mT5Block.forward\u001b[0;34m(self, hidden_states, attention_mask, position_bias, encoder_hidden_states, encoder_attention_mask, encoder_decoder_position_bias, layer_head_mask, cross_attn_layer_head_mask, past_key_value, use_cache, output_attentions, return_dict)\u001b[0m\n\u001b[1;32m    710\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    711\u001b[0m     query_length \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m--> 713\u001b[0m cross_attention_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mlayer[\u001b[39m1\u001b[39;49m](\n\u001b[1;32m    714\u001b[0m     hidden_states,\n\u001b[1;32m    715\u001b[0m     key_value_states\u001b[39m=\u001b[39;49mencoder_hidden_states,\n\u001b[1;32m    716\u001b[0m     attention_mask\u001b[39m=\u001b[39;49mencoder_attention_mask,\n\u001b[1;32m    717\u001b[0m     position_bias\u001b[39m=\u001b[39;49mencoder_decoder_position_bias,\n\u001b[1;32m    718\u001b[0m     layer_head_mask\u001b[39m=\u001b[39;49mcross_attn_layer_head_mask,\n\u001b[1;32m    719\u001b[0m     past_key_value\u001b[39m=\u001b[39;49mcross_attn_past_key_value,\n\u001b[1;32m    720\u001b[0m     query_length\u001b[39m=\u001b[39;49mquery_length,\n\u001b[1;32m    721\u001b[0m     use_cache\u001b[39m=\u001b[39;49muse_cache,\n\u001b[1;32m    722\u001b[0m     output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m    723\u001b[0m )\n\u001b[1;32m    724\u001b[0m hidden_states \u001b[39m=\u001b[39m cross_attention_outputs[\u001b[39m0\u001b[39m]\n\u001b[1;32m    726\u001b[0m \u001b[39m# clamp inf values to enable fp16 training\u001b[39;00m\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py:627\u001b[0m, in \u001b[0;36mT5LayerCrossAttention.forward\u001b[0;34m(self, hidden_states, key_value_states, attention_mask, position_bias, layer_head_mask, past_key_value, use_cache, query_length, output_attentions)\u001b[0m\n\u001b[1;32m    614\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\n\u001b[1;32m    615\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[1;32m    616\u001b[0m     hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    624\u001b[0m     output_attentions\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m    625\u001b[0m ):\n\u001b[1;32m    626\u001b[0m     normed_hidden_states \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlayer_norm(hidden_states)\n\u001b[0;32m--> 627\u001b[0m     attention_output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mEncDecAttention(\n\u001b[1;32m    628\u001b[0m         normed_hidden_states,\n\u001b[1;32m    629\u001b[0m         mask\u001b[39m=\u001b[39;49mattention_mask,\n\u001b[1;32m    630\u001b[0m         key_value_states\u001b[39m=\u001b[39;49mkey_value_states,\n\u001b[1;32m    631\u001b[0m         position_bias\u001b[39m=\u001b[39;49mposition_bias,\n\u001b[1;32m    632\u001b[0m         layer_head_mask\u001b[39m=\u001b[39;49mlayer_head_mask,\n\u001b[1;32m    633\u001b[0m         past_key_value\u001b[39m=\u001b[39;49mpast_key_value,\n\u001b[1;32m    634\u001b[0m         use_cache\u001b[39m=\u001b[39;49muse_cache,\n\u001b[1;32m    635\u001b[0m         query_length\u001b[39m=\u001b[39;49mquery_length,\n\u001b[1;32m    636\u001b[0m         output_attentions\u001b[39m=\u001b[39;49moutput_attentions,\n\u001b[1;32m    637\u001b[0m     )\n\u001b[1;32m    638\u001b[0m     layer_output \u001b[39m=\u001b[39m hidden_states \u001b[39m+\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdropout(attention_output[\u001b[39m0\u001b[39m])\n\u001b[1;32m    639\u001b[0m     outputs \u001b[39m=\u001b[39m (layer_output,) \u001b[39m+\u001b[39m attention_output[\u001b[39m1\u001b[39m:]  \u001b[39m# add attentions if we output them\u001b[39;00m\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py:518\u001b[0m, in \u001b[0;36mT5Attention.forward\u001b[0;34m(self, hidden_states, mask, key_value_states, position_bias, past_key_value, layer_head_mask, query_length, use_cache, output_attentions)\u001b[0m\n\u001b[1;32m    514\u001b[0m \u001b[39m# get key/value states\u001b[39;00m\n\u001b[1;32m    515\u001b[0m key_states \u001b[39m=\u001b[39m project(\n\u001b[1;32m    516\u001b[0m     hidden_states, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mk, key_value_states, past_key_value[\u001b[39m0\u001b[39m] \u001b[39mif\u001b[39;00m past_key_value \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m    517\u001b[0m )\n\u001b[0;32m--> 518\u001b[0m value_states \u001b[39m=\u001b[39m project(\n\u001b[1;32m    519\u001b[0m     hidden_states, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mv, key_value_states, past_key_value[\u001b[39m1\u001b[39;49m] \u001b[39mif\u001b[39;49;00m past_key_value \u001b[39mis\u001b[39;49;00m \u001b[39mnot\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m \u001b[39melse\u001b[39;49;00m \u001b[39mNone\u001b[39;49;00m\n\u001b[1;32m    520\u001b[0m )\n\u001b[1;32m    522\u001b[0m \u001b[39m# compute scores\u001b[39;00m\n\u001b[1;32m    523\u001b[0m scores \u001b[39m=\u001b[39m torch\u001b[39m.\u001b[39mmatmul(\n\u001b[1;32m    524\u001b[0m     query_states, key_states\u001b[39m.\u001b[39mtranspose(\u001b[39m3\u001b[39m, \u001b[39m2\u001b[39m)\n\u001b[1;32m    525\u001b[0m )  \u001b[39m# equivalent of torch.einsum(\"bnqd,bnkd->bnqk\", query_states, key_states), compatible with onnx op>9\u001b[39;00m\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/models/t5/modeling_t5.py:493\u001b[0m, in \u001b[0;36mT5Attention.forward.<locals>.project\u001b[0;34m(hidden_states, proj_layer, key_value_states, past_key_value)\u001b[0m\n\u001b[1;32m    489\u001b[0m     hidden_states \u001b[39m=\u001b[39m shape(proj_layer(hidden_states))\n\u001b[1;32m    490\u001b[0m \u001b[39melif\u001b[39;00m past_key_value \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    491\u001b[0m     \u001b[39m# cross-attn\u001b[39;00m\n\u001b[1;32m    492\u001b[0m     \u001b[39m# (batch_size, n_heads, seq_length, dim_per_head)\u001b[39;00m\n\u001b[0;32m--> 493\u001b[0m     hidden_states \u001b[39m=\u001b[39m shape(proj_layer(key_value_states))\n\u001b[1;32m    495\u001b[0m \u001b[39mif\u001b[39;00m past_key_value \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    496\u001b[0m     \u001b[39mif\u001b[39;00m key_value_states \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m    497\u001b[0m         \u001b[39m# self-attn\u001b[39;00m\n\u001b[1;32m    498\u001b[0m         \u001b[39m# (batch_size, n_heads, key_length, dim_per_head)\u001b[39;00m\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/torch/nn/modules/module.py:1194\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1190\u001b[0m \u001b[39m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1191\u001b[0m \u001b[39m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1192\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m (\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_backward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_hooks \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward_pre_hooks \u001b[39mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m         \u001b[39mor\u001b[39;00m _global_forward_hooks \u001b[39mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0m     \u001b[39mreturn\u001b[39;00m forward_call(\u001b[39m*\u001b[39;49m\u001b[39minput\u001b[39;49m, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m   1195\u001b[0m \u001b[39m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1196\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[39m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/torch/nn/modules/linear.py:114\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mforward\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39minput\u001b[39m: Tensor) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tensor:\n\u001b[0;32m--> 114\u001b[0m     \u001b[39mreturn\u001b[39;00m F\u001b[39m.\u001b[39;49mlinear(\u001b[39minput\u001b[39;49m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mweight, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mbias)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Get Dataset\n",
    "# N_SNIPPETS = 10000\n",
    "dataset = CodeSnippetDataset(github=False, languages=[\"Python\"])\n",
    "N_SNIPPETS = len(dataset.dataset)\n",
    "code_snippets = dataset.get_n_snippets(N_SNIPPETS, max_length=512)\n",
    "\n",
    "print(\"Got snippets!\")\n",
    "\n",
    "\n",
    "# Get documentation from dataset\n",
    "code2doc = Code2DocModule(code_snippets)\n",
    "data_with_docs = code2doc.get_docs()\n",
    "    \n",
    "print(\"Got documentations!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sacrebleu.metrics import CHRF\n",
    "\n",
    "refs = [\n",
    "    [\"def hello_world(): return 'hello world!'\"],\n",
    "    [\"def print_hello(): print('hello')\"],\n",
    "    [\"def greet(msg): print('msg')\"],\n",
    "]\n",
    "\n",
    "sys = [\"def print_hello(): print('hello')\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chrF2 = 100.00\n"
     ]
    }
   ],
   "source": [
    "chrf = CHRF()\n",
    "score = chrf.corpus_score(sys, refs)\n",
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/michaelnath/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Using custom data configuration Python-all-4b2efe4a27feed92\n"
     ]
    }
   ],
   "source": [
    "from Modules.Code2Code.Extracontent.code_snippet_dataset import CodeSnippetDataset\n",
    "\n",
    "dataset = CodeSnippetDataset(languages=[\"Python\"], github=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "snippets = dataset.dataset.take(30)\n",
    "functions = []\n",
    "for snippet in snippets:\n",
    "    functions += dataset.construct_list_of_functions(snippet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def __init__(self, allow=None, disallow=None, secure=True, *args, **kwargs):\n",
      "\t\tsuper(TemplateField, self).__init__(*args, **kwargs)\n",
      "\t\tself.validators.append(TemplateValidator(allow, disallow, secure))\n",
      "\n",
      "def __init__(self, field):\n",
      "\t\tself.field = field\n",
      "\n",
      "def __get__(self, instance, owner):\n",
      "\t\tif instance is None:\n",
      "\t\t\traise AttributeError # ?\n",
      "\n",
      "def __set__(self, instance, value):\n",
      "\t\tinstance.__dict__[self.field.name] = value\n",
      "\t\tsetattr(instance, self.field.attname, json.dumps(value))\n",
      "\n",
      "def __delete__(self, instance):\n",
      "\t\tdel(instance.__dict__[self.field.name])\n",
      "\t\tsetattr(instance, self.field.attname, json.dumps(None))\n",
      "\n",
      "def get_attname(self):\n",
      "\t\treturn \"%s_json\" % self.name\n",
      "\n",
      "def contribute_to_class(self, cls, name):\n",
      "\t\tsuper(JSONField, self).contribute_to_class(cls, name)\n",
      "\t\tsetattr(cls, name, JSONDescriptor(self))\n",
      "\t\tmodels.signals.pre_init.connect(self.fix_init_kwarg, sender=cls)\n",
      "\n",
      "def fix_init_kwarg(self, sender, args, kwargs, **signal_kwargs):\n",
      "\t\t# Anything passed in as self.name is assumed to come from a serializer and\n",
      "\t\t# will be treated as a json string.\n",
      "\t\tif self.name in kwargs:\n",
      "\t\t\tvalue = kwargs.pop(self.name)\n",
      "\n",
      "def formfield(self, *args, **kwargs):\n",
      "\t\tkwargs[\"form_class\"] = JSONFormField\n",
      "\t\treturn super(JSONField, self).formfield(*args, **kwargs)\n",
      "\n",
      "def get_internal_type(self):\n",
      "\t\treturn \"TextField\"\n",
      "\n",
      "def to_python(self, value):\n",
      "\t\tif not value:\n",
      "\t\t\treturn []\n",
      "\n",
      "def get_prep_value(self, value):\n",
      "\t\treturn ','.join(value)\n",
      "\n",
      "def formfield(self, **kwargs):\n",
      "\t\t# This is necessary because django hard-codes TypedChoiceField for things with choices.\n",
      "\t\tdefaults = {\n",
      "\t\t\t'widget': forms.CheckboxSelectMultiple,\n",
      "\t\t\t'choices': self.get_choices(include_blank=False),\n",
      "\t\t\t'label': capfirst(self.verbose_name),\n",
      "\t\t\t'required': not self.blank,\n",
      "\t\t\t'help_text': self.help_text\n",
      "\t\t}\n",
      "\t\tif self.has_default():\n",
      "\t\t\tif callable(self.default):\n",
      "\t\t\t\tdefaults['initial'] = self.default\n",
      "\t\t\t\tdefaults['show_hidden_initial'] = True\n",
      "\t\t\telse:\n",
      "\t\t\t\tdefaults['initial'] = self.get_default()\n",
      "\n",
      "def validate(self, value, model_instance):\n",
      "\t\tinvalid_values = []\n",
      "\t\tfor val in value:\n",
      "\t\t\ttry:\n",
      "\t\t\t\tvalidate_slug(val)\n",
      "\t\t\texcept ValidationError:\n",
      "\t\t\t\tinvalid_values.append(val)\n",
      "\n",
      "def _get_choices(self):\n",
      "\t\tif isinstance(self._choices, RegistryIterator):\n",
      "\t\t\treturn self._choices.copy()\n",
      "\t\telif hasattr(self._choices, 'next'):\n",
      "\t\t\tchoices, self._choices = itertools.tee(self._choices)\n",
      "\t\t\treturn choices\n",
      "\t\telse:\n",
      "\t\t\treturn self._choices\n",
      "\n",
      "def load_source(module_name, module_path):\n",
      "    \"\"\"Loads a python module from the path of the corresponding file.\"\"\"\n",
      "\n",
      "    if sys.version_info[0] == 3 and sys.version_info[1] >= 5:\n",
      "        import importlib.util\n",
      "        spec = importlib.util.spec_from_file_location(module_name, module_path)\n",
      "        module = importlib.util.module_from_spec(spec)\n",
      "        spec.loader.exec_module(module)\n",
      "    elif sys.version_info[0] == 3 and sys.version_info[1] < 5:\n",
      "        import importlib.machinery\n",
      "        loader = importlib.machinery.SourceFileLoader(module_name, module_path)\n",
      "        module = loader.load_module()\n",
      "    return module\n",
      "\n",
      "def cleanup_old_versions(\n",
      "    src, keep_last_versions, config_file=\"config.yaml\", profile_name=None,\n",
      "):\n",
      "    \"\"\"Deletes old deployed versions of the function in AWS Lambda.\n",
      "\n",
      "    Won't delete $Latest and any aliased version\n",
      "\n",
      "    :param str src:\n",
      "        The path to your Lambda ready project (folder must contain a valid\n",
      "        config.yaml and handler module (e.g.: service.py).\n",
      "    :param int keep_last_versions:\n",
      "        The number of recent versions to keep and not delete\n",
      "    \"\"\"\n",
      "    if keep_last_versions <= 0:\n",
      "        print(\"Won't delete all versions. Please do this manually\")\n",
      "    else:\n",
      "        path_to_config_file = os.path.join(src, config_file)\n",
      "        cfg = read_cfg(path_to_config_file, profile_name)\n",
      "\n",
      "        profile_name = cfg.get(\"profile\")\n",
      "        aws_access_key_id = cfg.get(\"aws_access_key_id\")\n",
      "        aws_secret_access_key = cfg.get(\"aws_secret_access_key\")\n",
      "\n",
      "        client = get_client(\n",
      "            \"lambda\",\n",
      "            profile_name,\n",
      "            aws_access_key_id,\n",
      "            aws_secret_access_key,\n",
      "            cfg.get(\"region\"),\n",
      "        )\n",
      "\n",
      "        response = client.list_versions_by_function(\n",
      "            FunctionName=cfg.get(\"function_name\"),\n",
      "        )\n",
      "        versions = response.get(\"Versions\")\n",
      "        if len(response.get(\"Versions\")) < keep_last_versions:\n",
      "            print(\"Nothing to delete. (Too few versions published)\")\n",
      "        else:\n",
      "            version_numbers = [\n",
      "                elem.get(\"Version\") for elem in versions[1:-keep_last_versions]\n",
      "            ]\n",
      "            for version_number in version_numbers:\n",
      "                try:\n",
      "                    client.delete_function(\n",
      "                        FunctionName=cfg.get(\"function_name\"),\n",
      "                        Qualifier=version_number,\n",
      "                    )\n",
      "                except botocore.exceptions.ClientError as e:\n",
      "                    print(f\"Skipping Version {version_number}: {e}\")\n",
      "\n",
      "def deploy(\n",
      "    src,\n",
      "    requirements=None,\n",
      "    local_package=None,\n",
      "    config_file=\"config.yaml\",\n",
      "    profile_name=None,\n",
      "    preserve_vpc=False,\n",
      "):\n",
      "    \"\"\"Deploys a new function to AWS Lambda.\n",
      "\n",
      "    :param str src:\n",
      "        The path to your Lambda ready project (folder must contain a valid\n",
      "        config.yaml and handler module (e.g.: service.py).\n",
      "    :param str local_package:\n",
      "        The path to a local package with should be included in the deploy as\n",
      "        well (and/or is not available on PyPi)\n",
      "    \"\"\"\n",
      "    # Load and parse the config file.\n",
      "    path_to_config_file = os.path.join(src, config_file)\n",
      "    cfg = read_cfg(path_to_config_file, profile_name)\n",
      "\n",
      "    # Copy all the pip dependencies required to run your code into a temporary\n",
      "    # folder then add the handler file in the root of this directory.\n",
      "    # Zip the contents of this folder into a single file and output to the dist\n",
      "    # directory.\n",
      "    path_to_zip_file = build(\n",
      "        src,\n",
      "        config_file=config_file,\n",
      "        requirements=requirements,\n",
      "        local_package=local_package,\n",
      "    )\n",
      "\n",
      "    existing_config = get_function_config(cfg)\n",
      "    if existing_config:\n",
      "        update_function(\n",
      "            cfg, path_to_zip_file, existing_config, preserve_vpc=preserve_vpc\n",
      "        )\n",
      "    else:\n",
      "        create_function(cfg, path_to_zip_file)\n",
      "\n",
      "def deploy_s3(\n",
      "    src,\n",
      "    requirements=None,\n",
      "    local_package=None,\n",
      "    config_file=\"config.yaml\",\n",
      "    profile_name=None,\n",
      "    preserve_vpc=False,\n",
      "):\n",
      "    \"\"\"Deploys a new function via AWS S3.\n",
      "\n",
      "    :param str src:\n",
      "        The path to your Lambda ready project (folder must contain a valid\n",
      "        config.yaml and handler module (e.g.: service.py).\n",
      "    :param str local_package:\n",
      "        The path to a local package with should be included in the deploy as\n",
      "        well (and/or is not available on PyPi)\n",
      "    \"\"\"\n",
      "    # Load and parse the config file.\n",
      "    path_to_config_file = os.path.join(src, config_file)\n",
      "    cfg = read_cfg(path_to_config_file, profile_name)\n",
      "\n",
      "    # Copy all the pip dependencies required to run your code into a temporary\n",
      "    # folder then add the handler file in the root of this directory.\n",
      "    # Zip the contents of this folder into a single file and output to the dist\n",
      "    # directory.\n",
      "    path_to_zip_file = build(\n",
      "        src,\n",
      "        config_file=config_file,\n",
      "        requirements=requirements,\n",
      "        local_package=local_package,\n",
      "    )\n",
      "\n",
      "    use_s3 = True\n",
      "    s3_file = upload_s3(cfg, path_to_zip_file, use_s3)\n",
      "    existing_config = get_function_config(cfg)\n",
      "    if existing_config:\n",
      "        update_function(\n",
      "            cfg,\n",
      "            path_to_zip_file,\n",
      "            existing_config,\n",
      "            use_s3=use_s3,\n",
      "            s3_file=s3_file,\n",
      "            preserve_vpc=preserve_vpc,\n",
      "        )\n",
      "    else:\n",
      "        create_function(cfg, path_to_zip_file, use_s3=use_s3, s3_file=s3_file)\n",
      "\n",
      "def upload(\n",
      "    src,\n",
      "    requirements=None,\n",
      "    local_package=None,\n",
      "    config_file=\"config.yaml\",\n",
      "    profile_name=None,\n",
      "):\n",
      "    \"\"\"Uploads a new function to AWS S3.\n",
      "\n",
      "    :param str src:\n",
      "        The path to your Lambda ready project (folder must contain a valid\n",
      "        config.yaml and handler module (e.g.: service.py).\n",
      "    :param str local_package:\n",
      "        The path to a local package with should be included in the deploy as\n",
      "        well (and/or is not available on PyPi)\n",
      "    \"\"\"\n",
      "    # Load and parse the config file.\n",
      "    path_to_config_file = os.path.join(src, config_file)\n",
      "    cfg = read_cfg(path_to_config_file, profile_name)\n",
      "\n",
      "    # Copy all the pip dependencies required to run your code into a temporary\n",
      "    # folder then add the handler file in the root of this directory.\n",
      "    # Zip the contents of this folder into a single file and output to the dist\n",
      "    # directory.\n",
      "    path_to_zip_file = build(\n",
      "        src,\n",
      "        config_file=config_file,\n",
      "        requirements=requirements,\n",
      "        local_package=local_package,\n",
      "    )\n",
      "\n",
      "    upload_s3(cfg, path_to_zip_file)\n",
      "\n",
      "def invoke(\n",
      "    src,\n",
      "    event_file=\"event.json\",\n",
      "    config_file=\"config.yaml\",\n",
      "    profile_name=None,\n",
      "    verbose=False,\n",
      "):\n",
      "    \"\"\"Simulates a call to your function.\n",
      "\n",
      "    :param str src:\n",
      "        The path to your Lambda ready project (folder must contain a valid\n",
      "        config.yaml and handler module (e.g.: service.py).\n",
      "    :param str alt_event:\n",
      "        An optional argument to override which event file to use.\n",
      "    :param bool verbose:\n",
      "        Whether to print out verbose details.\n",
      "    \"\"\"\n",
      "    # Load and parse the config file.\n",
      "    path_to_config_file = os.path.join(src, config_file)\n",
      "    cfg = read_cfg(path_to_config_file, profile_name)\n",
      "\n",
      "    # Set AWS_PROFILE environment variable based on `--profile` option.\n",
      "    if profile_name:\n",
      "        os.environ[\"AWS_PROFILE\"] = profile_name\n",
      "\n",
      "    # Load environment variables from the config file into the actual\n",
      "    # environment.\n",
      "    env_vars = cfg.get(\"environment_variables\")\n",
      "    if env_vars:\n",
      "        for key, value in env_vars.items():\n",
      "            os.environ[key] = get_environment_variable_value(value)\n",
      "\n",
      "    # Load and parse event file.\n",
      "    path_to_event_file = os.path.join(src, event_file)\n",
      "    event = read(path_to_event_file, loader=json.loads)\n",
      "\n",
      "    # Tweak to allow module to import local modules\n",
      "    try:\n",
      "        sys.path.index(src)\n",
      "    except ValueError:\n",
      "        sys.path.append(src)\n",
      "\n",
      "    handler = cfg.get(\"handler\")\n",
      "    # Inspect the handler string (<module>.<function name>) and translate it\n",
      "    # into a function we can execute.\n",
      "    fn = get_callable_handler_function(src, handler)\n",
      "\n",
      "    timeout = cfg.get(\"timeout\")\n",
      "    if timeout:\n",
      "        context = LambdaContext(cfg.get(\"function_name\"), timeout)\n",
      "    else:\n",
      "        context = LambdaContext(cfg.get(\"function_name\"))\n",
      "\n",
      "    start = time.time()\n",
      "    results = fn(event, context)\n",
      "    end = time.time()\n",
      "\n",
      "    print(\"{0}\".format(results))\n",
      "    if verbose:\n",
      "        print(\n",
      "            \"\\nexecution time: {:.8f}s\\nfunction execution \"\n",
      "            \"timeout: {:2}s\".format(end - start, cfg.get(\"timeout\", 15))\n",
      "        )\n",
      "\n",
      "def init(src, minimal=False):\n",
      "    \"\"\"Copies template files to a given directory.\n",
      "\n",
      "    :param str src:\n",
      "        The path to output the template lambda project files.\n",
      "    :param bool minimal:\n",
      "        Minimal possible template files (excludes event.json).\n",
      "    \"\"\"\n",
      "\n",
      "    templates_path = os.path.join(\n",
      "        os.path.dirname(os.path.abspath(__file__)), \"project_templates\",\n",
      "    )\n",
      "    for filename in os.listdir(templates_path):\n",
      "        if (minimal and filename == \"event.json\") or filename.endswith(\".pyc\"):\n",
      "            continue\n",
      "        dest_path = os.path.join(templates_path, filename)\n",
      "\n",
      "        if not os.path.isdir(dest_path):\n",
      "            copy(dest_path, src)\n",
      "\n",
      "def build(\n",
      "    src,\n",
      "    requirements=None,\n",
      "    local_package=None,\n",
      "    config_file=\"config.yaml\",\n",
      "    profile_name=None,\n",
      "):\n",
      "    \"\"\"Builds the file bundle.\n",
      "\n",
      "    :param str src:\n",
      "       The path to your Lambda ready project (folder must contain a valid\n",
      "        config.yaml and handler module (e.g.: service.py).\n",
      "    :param str local_package:\n",
      "        The path to a local package with should be included in the deploy as\n",
      "        well (and/or is not available on PyPi)\n",
      "    \"\"\"\n",
      "    # Load and parse the config file.\n",
      "    path_to_config_file = os.path.join(src, config_file)\n",
      "    cfg = read_cfg(path_to_config_file, profile_name)\n",
      "\n",
      "    # Get the absolute path to the output directory and create it if it doesn't\n",
      "    # already exist.\n",
      "    dist_directory = cfg.get(\"dist_directory\", \"dist\")\n",
      "    path_to_dist = os.path.join(src, dist_directory)\n",
      "    mkdir(path_to_dist)\n",
      "\n",
      "    # Combine the name of the Lambda function with the current timestamp to use\n",
      "    # for the output filename.\n",
      "    function_name = cfg.get(\"function_name\")\n",
      "    output_filename = \"{0}-{1}.zip\".format(timestamp(), function_name)\n",
      "\n",
      "    path_to_temp = mkdtemp(prefix=\"aws-lambda\")\n",
      "    pip_install_to_target(\n",
      "        path_to_temp, requirements=requirements, local_package=local_package,\n",
      "    )\n",
      "\n",
      "    # Hack for Zope.\n",
      "    if \"zope\" in os.listdir(path_to_temp):\n",
      "        print(\n",
      "            \"Zope packages detected; fixing Zope package paths to \"\n",
      "            \"make them importable.\",\n",
      "        )\n",
      "        # Touch.\n",
      "        with open(os.path.join(path_to_temp, \"zope/__init__.py\"), \"wb\"):\n",
      "            pass\n",
      "\n",
      "    # Gracefully handle whether \".zip\" was included in the filename or not.\n",
      "    output_filename = (\n",
      "        \"{0}.zip\".format(output_filename)\n",
      "        if not output_filename.endswith(\".zip\")\n",
      "        else output_filename\n",
      "    )\n",
      "\n",
      "    # Allow definition of source code directories we want to build into our\n",
      "    # zipped package.\n",
      "    build_config = defaultdict(**cfg.get(\"build\", {}))\n",
      "    build_source_directories = build_config.get(\"source_directories\", \"\")\n",
      "    build_source_directories = (\n",
      "        build_source_directories\n",
      "        if build_source_directories is not None\n",
      "        else \"\"\n",
      "    )\n",
      "    source_directories = [\n",
      "        d.strip() for d in build_source_directories.split(\",\")\n",
      "    ]\n",
      "\n",
      "    files = []\n",
      "    for filename in os.listdir(src):\n",
      "        if os.path.isfile(filename):\n",
      "            if filename == \".DS_Store\":\n",
      "                continue\n",
      "            if filename == config_file:\n",
      "                continue\n",
      "            print(\"Bundling: %r\" % filename)\n",
      "            files.append(os.path.join(src, filename))\n",
      "        elif os.path.isdir(filename) and filename in source_directories:\n",
      "            print(\"Bundling directory: %r\" % filename)\n",
      "            files.append(os.path.join(src, filename))\n",
      "\n",
      "    # \"cd\" into `temp_path` directory.\n",
      "    os.chdir(path_to_temp)\n",
      "    for f in files:\n",
      "        if os.path.isfile(f):\n",
      "            _, filename = os.path.split(f)\n",
      "\n",
      "            # Copy handler file into root of the packages folder.\n",
      "            copyfile(f, os.path.join(path_to_temp, filename))\n",
      "            copystat(f, os.path.join(path_to_temp, filename))\n",
      "        elif os.path.isdir(f):\n",
      "            src_path_length = len(src) + 1\n",
      "            destination_folder = os.path.join(\n",
      "                path_to_temp, f[src_path_length:]\n",
      "            )\n",
      "            copytree(f, destination_folder)\n",
      "\n",
      "    # Zip them together into a single file.\n",
      "    # TODO: Delete temp directory created once the archive has been compiled.\n",
      "    path_to_zip_file = archive(\"./\", path_to_dist, output_filename)\n",
      "    return path_to_zip_file\n",
      "\n",
      "def get_callable_handler_function(src, handler):\n",
      "    \"\"\"Translate a string of the form \"module.function\" into a callable\n",
      "    function.\n",
      "\n",
      "    :param str src:\n",
      "      The path to your Lambda project containing a valid handler file.\n",
      "    :param str handler:\n",
      "      A dot delimited string representing the `<module>.<function name>`.\n",
      "    \"\"\"\n",
      "\n",
      "    # \"cd\" into `src` directory.\n",
      "    os.chdir(src)\n",
      "\n",
      "    module_name, function_name = handler.split(\".\")\n",
      "    filename = get_handler_filename(handler)\n",
      "\n",
      "    path_to_module_file = os.path.join(src, filename)\n",
      "    module = load_source(module_name, path_to_module_file)\n",
      "    return getattr(module, function_name)\n",
      "\n",
      "def get_handler_filename(handler):\n",
      "    \"\"\"Shortcut to get the filename from the handler string.\n",
      "\n",
      "    :param str handler:\n",
      "      A dot delimited string representing the `<module>.<function name>`.\n",
      "    \"\"\"\n",
      "    module_name, _ = handler.split(\".\")\n",
      "    return \"{0}.py\".format(module_name)\n",
      "\n",
      "def _filter_blacklist(package):\n",
      "        blacklist = [\"-i\", \"#\", \"Python==\", \"python-lambda==\"]\n",
      "        return all(package.startswith(entry) is False for entry in blacklist)\n",
      "\n",
      "def pip_install_to_target(path, requirements=None, local_package=None):\n",
      "    \"\"\"For a given active virtualenv, gather all installed pip packages then\n",
      "    copy (re-install) them to the path provided.\n",
      "\n",
      "    :param str path:\n",
      "        Path to copy installed pip packages to.\n",
      "    :param str requirements:\n",
      "        If set, only the packages in the supplied requirements file are\n",
      "        installed.\n",
      "        If not set then installs all packages found via pip freeze.\n",
      "    :param str local_package:\n",
      "        The path to a local package with should be included in the deploy as\n",
      "        well (and/or is not available on PyPi)\n",
      "    \"\"\"\n",
      "    packages = []\n",
      "    if not requirements:\n",
      "        print(\"Gathering pip packages\")\n",
      "        pkgStr = subprocess.check_output(\n",
      "            [sys.executable, \"-m\", \"pip\", \"freeze\"]\n",
      "        )\n",
      "        packages.extend(pkgStr.decode(\"utf-8\").splitlines())\n",
      "    else:\n",
      "        if os.path.exists(requirements):\n",
      "            print(\"Gathering requirement packages\")\n",
      "            data = read(requirements)\n",
      "            packages.extend(data.splitlines())\n",
      "\n",
      "    if not packages:\n",
      "        print(\"No dependency packages installed!\")\n",
      "\n",
      "    if local_package is not None:\n",
      "        if not isinstance(local_package, (list, tuple)):\n",
      "            local_package = [local_package]\n",
      "        for l_package in local_package:\n",
      "            packages.append(l_package)\n",
      "    _install_packages(path, packages)\n",
      "\n",
      "def get_role_name(region, account_id, role):\n",
      "    \"\"\"Shortcut to insert the `account_id` and `role` into the iam string.\"\"\"\n",
      "    prefix = ARN_PREFIXES.get(region, \"aws\")\n",
      "    return \"arn:{0}:iam::{1}:role/{2}\".format(prefix, account_id, role)\n",
      "\n",
      "def get_account_id(\n",
      "    profile_name, aws_access_key_id, aws_secret_access_key, region=None,\n",
      "):\n",
      "    \"\"\"Query STS for a users' account_id\"\"\"\n",
      "    client = get_client(\n",
      "        \"sts\", profile_name, aws_access_key_id, aws_secret_access_key, region,\n",
      "    )\n",
      "    return client.get_caller_identity().get(\"Account\")\n",
      "\n",
      "def get_client(\n",
      "    client,\n",
      "    profile_name,\n",
      "    aws_access_key_id,\n",
      "    aws_secret_access_key,\n",
      "    region=None,\n",
      "):\n",
      "    \"\"\"Shortcut for getting an initialized instance of the boto3 client.\"\"\"\n",
      "\n",
      "    boto3.setup_default_session(\n",
      "        profile_name=profile_name,\n",
      "        aws_access_key_id=aws_access_key_id,\n",
      "        aws_secret_access_key=aws_secret_access_key,\n",
      "        region_name=region,\n",
      "    )\n",
      "    return boto3.client(client)\n",
      "\n",
      "def create_function(cfg, path_to_zip_file, use_s3=False, s3_file=None):\n",
      "    \"\"\"Register and upload a function to AWS Lambda.\"\"\"\n",
      "\n",
      "    print(\"Creating your new Lambda function\")\n",
      "    byte_stream = read(path_to_zip_file, binary_file=True)\n",
      "    profile_name = cfg.get(\"profile\")\n",
      "    aws_access_key_id = cfg.get(\"aws_access_key_id\")\n",
      "    aws_secret_access_key = cfg.get(\"aws_secret_access_key\")\n",
      "\n",
      "    account_id = get_account_id(\n",
      "        profile_name,\n",
      "        aws_access_key_id,\n",
      "        aws_secret_access_key,\n",
      "        cfg.get(\"region\",),\n",
      "    )\n",
      "    role = get_role_name(\n",
      "        cfg.get(\"region\"),\n",
      "        account_id,\n",
      "        cfg.get(\"role\", \"lambda_basic_execution\"),\n",
      "    )\n",
      "\n",
      "    client = get_client(\n",
      "        \"lambda\",\n",
      "        profile_name,\n",
      "        aws_access_key_id,\n",
      "        aws_secret_access_key,\n",
      "        cfg.get(\"region\"),\n",
      "    )\n",
      "\n",
      "    # Do we prefer development variable over config?\n",
      "    buck_name = os.environ.get(\"S3_BUCKET_NAME\") or cfg.get(\"bucket_name\")\n",
      "    func_name = os.environ.get(\"LAMBDA_FUNCTION_NAME\") or cfg.get(\n",
      "        \"function_name\"\n",
      "    )\n",
      "    print(\"Creating lambda function with name: {}\".format(func_name))\n",
      "\n",
      "    if use_s3:\n",
      "        kwargs = {\n",
      "            \"FunctionName\": func_name,\n",
      "            \"Runtime\": cfg.get(\"runtime\", \"python2.7\"),\n",
      "            \"Role\": role,\n",
      "            \"Handler\": cfg.get(\"handler\"),\n",
      "            \"Code\": {\n",
      "                \"S3Bucket\": \"{}\".format(buck_name),\n",
      "                \"S3Key\": \"{}\".format(s3_file),\n",
      "            },\n",
      "            \"Description\": cfg.get(\"description\", \"\"),\n",
      "            \"Timeout\": cfg.get(\"timeout\", 15),\n",
      "            \"MemorySize\": cfg.get(\"memory_size\", 512),\n",
      "            \"VpcConfig\": {\n",
      "                \"SubnetIds\": cfg.get(\"subnet_ids\", []),\n",
      "                \"SecurityGroupIds\": cfg.get(\"security_group_ids\", []),\n",
      "            },\n",
      "            \"Publish\": True,\n",
      "        }\n",
      "    else:\n",
      "        kwargs = {\n",
      "            \"FunctionName\": func_name,\n",
      "            \"Runtime\": cfg.get(\"runtime\", \"python2.7\"),\n",
      "            \"Role\": role,\n",
      "            \"Handler\": cfg.get(\"handler\"),\n",
      "            \"Code\": {\"ZipFile\": byte_stream},\n",
      "            \"Description\": cfg.get(\"description\", \"\"),\n",
      "            \"Timeout\": cfg.get(\"timeout\", 15),\n",
      "            \"MemorySize\": cfg.get(\"memory_size\", 512),\n",
      "            \"VpcConfig\": {\n",
      "                \"SubnetIds\": cfg.get(\"subnet_ids\", []),\n",
      "                \"SecurityGroupIds\": cfg.get(\"security_group_ids\", []),\n",
      "            },\n",
      "            \"Publish\": True,\n",
      "        }\n",
      "\n",
      "    if \"tags\" in cfg:\n",
      "        kwargs.update(\n",
      "            Tags={key: str(value) for key, value in cfg.get(\"tags\").items()}\n",
      "        )\n",
      "\n",
      "    if \"environment_variables\" in cfg:\n",
      "        kwargs.update(\n",
      "            Environment={\n",
      "                \"Variables\": {\n",
      "                    key: get_environment_variable_value(value)\n",
      "                    for key, value in cfg.get(\"environment_variables\").items()\n",
      "                },\n",
      "            },\n",
      "        )\n",
      "\n",
      "    client.create_function(**kwargs)\n",
      "\n",
      "    concurrency = get_concurrency(cfg)\n",
      "    if concurrency > 0:\n",
      "        client.put_function_concurrency(\n",
      "            FunctionName=func_name, ReservedConcurrentExecutions=concurrency\n",
      "        )\n",
      "\n",
      "def update_function(\n",
      "    cfg,\n",
      "    path_to_zip_file,\n",
      "    existing_cfg,\n",
      "    use_s3=False,\n",
      "    s3_file=None,\n",
      "    preserve_vpc=False,\n",
      "):\n",
      "    \"\"\"Updates the code of an existing Lambda function\"\"\"\n",
      "\n",
      "    print(\"Updating your Lambda function\")\n",
      "    byte_stream = read(path_to_zip_file, binary_file=True)\n",
      "    profile_name = cfg.get(\"profile\")\n",
      "    aws_access_key_id = cfg.get(\"aws_access_key_id\")\n",
      "    aws_secret_access_key = cfg.get(\"aws_secret_access_key\")\n",
      "\n",
      "    account_id = get_account_id(\n",
      "        profile_name,\n",
      "        aws_access_key_id,\n",
      "        aws_secret_access_key,\n",
      "        cfg.get(\"region\",),\n",
      "    )\n",
      "    role = get_role_name(\n",
      "        cfg.get(\"region\"),\n",
      "        account_id,\n",
      "        cfg.get(\"role\", \"lambda_basic_execution\"),\n",
      "    )\n",
      "\n",
      "    client = get_client(\n",
      "        \"lambda\",\n",
      "        profile_name,\n",
      "        aws_access_key_id,\n",
      "        aws_secret_access_key,\n",
      "        cfg.get(\"region\"),\n",
      "    )\n",
      "\n",
      "    # Do we prefer development variable over config?\n",
      "    buck_name = os.environ.get(\"S3_BUCKET_NAME\") or cfg.get(\"bucket_name\")\n",
      "\n",
      "    if use_s3:\n",
      "        client.update_function_code(\n",
      "            FunctionName=cfg.get(\"function_name\"),\n",
      "            S3Bucket=\"{}\".format(buck_name),\n",
      "            S3Key=\"{}\".format(s3_file),\n",
      "            Publish=True,\n",
      "        )\n",
      "    else:\n",
      "        client.update_function_code(\n",
      "            FunctionName=cfg.get(\"function_name\"),\n",
      "            ZipFile=byte_stream,\n",
      "            Publish=True,\n",
      "        )\n",
      "\n",
      "    kwargs = {\n",
      "        \"FunctionName\": cfg.get(\"function_name\"),\n",
      "        \"Role\": role,\n",
      "        \"Runtime\": cfg.get(\"runtime\"),\n",
      "        \"Handler\": cfg.get(\"handler\"),\n",
      "        \"Description\": cfg.get(\"description\", \"\"),\n",
      "        \"Timeout\": cfg.get(\"timeout\", 15),\n",
      "        \"MemorySize\": cfg.get(\"memory_size\", 512),\n",
      "    }\n",
      "\n",
      "    if preserve_vpc:\n",
      "        kwargs[\"VpcConfig\"] = existing_cfg.get(\"Configuration\", {}).get(\n",
      "            \"VpcConfig\"\n",
      "        )\n",
      "        if kwargs[\"VpcConfig\"] is None:\n",
      "            kwargs[\"VpcConfig\"] = {\n",
      "                \"SubnetIds\": cfg.get(\"subnet_ids\", []),\n",
      "                \"SecurityGroupIds\": cfg.get(\"security_group_ids\", []),\n",
      "            }\n",
      "        else:\n",
      "            del kwargs[\"VpcConfig\"][\"VpcId\"]\n",
      "    else:\n",
      "        kwargs[\"VpcConfig\"] = {\n",
      "            \"SubnetIds\": cfg.get(\"subnet_ids\", []),\n",
      "            \"SecurityGroupIds\": cfg.get(\"security_group_ids\", []),\n",
      "        }\n",
      "\n",
      "    if \"environment_variables\" in cfg:\n",
      "        kwargs.update(\n",
      "            Environment={\n",
      "                \"Variables\": {\n",
      "                    key: str(get_environment_variable_value(value))\n",
      "                    for key, value in cfg.get(\"environment_variables\").items()\n",
      "                },\n",
      "            },\n",
      "        )\n",
      "\n",
      "    ret = client.update_function_configuration(**kwargs)\n",
      "\n",
      "    concurrency = get_concurrency(cfg)\n",
      "    if concurrency > 0:\n",
      "        client.put_function_concurrency(\n",
      "            FunctionName=cfg.get(\"function_name\"),\n",
      "            ReservedConcurrentExecutions=concurrency,\n",
      "        )\n",
      "    elif \"Concurrency\" in existing_cfg:\n",
      "        client.delete_function_concurrency(\n",
      "            FunctionName=cfg.get(\"function_name\")\n",
      "        )\n",
      "\n",
      "    if \"tags\" in cfg:\n",
      "        tags = {key: str(value) for key, value in cfg.get(\"tags\").items()}\n",
      "        if tags != existing_cfg.get(\"Tags\"):\n",
      "            if existing_cfg.get(\"Tags\"):\n",
      "                client.untag_resource(\n",
      "                    Resource=ret[\"FunctionArn\"],\n",
      "                    TagKeys=list(existing_cfg[\"Tags\"].keys()),\n",
      "                )\n",
      "            client.tag_resource(Resource=ret[\"FunctionArn\"], Tags=tags)\n",
      "\n",
      "def upload_s3(cfg, path_to_zip_file, *use_s3):\n",
      "    \"\"\"Upload a function to AWS S3.\"\"\"\n",
      "\n",
      "    print(\"Uploading your new Lambda function\")\n",
      "    profile_name = cfg.get(\"profile\")\n",
      "    aws_access_key_id = cfg.get(\"aws_access_key_id\")\n",
      "    aws_secret_access_key = cfg.get(\"aws_secret_access_key\")\n",
      "    client = get_client(\n",
      "        \"s3\",\n",
      "        profile_name,\n",
      "        aws_access_key_id,\n",
      "        aws_secret_access_key,\n",
      "        cfg.get(\"region\"),\n",
      "    )\n",
      "    byte_stream = b\"\"\n",
      "    with open(path_to_zip_file, mode=\"rb\") as fh:\n",
      "        byte_stream = fh.read()\n",
      "    s3_key_prefix = cfg.get(\"s3_key_prefix\", \"/dist\")\n",
      "    checksum = hashlib.new(\"md5\", byte_stream).hexdigest()\n",
      "    timestamp = str(time.time())\n",
      "    filename = \"{prefix}{checksum}-{ts}.zip\".format(\n",
      "        prefix=s3_key_prefix, checksum=checksum, ts=timestamp,\n",
      "    )\n",
      "\n",
      "    # Do we prefer development variable over config?\n",
      "    buck_name = os.environ.get(\"S3_BUCKET_NAME\") or cfg.get(\"bucket_name\")\n",
      "    func_name = os.environ.get(\"LAMBDA_FUNCTION_NAME\") or cfg.get(\n",
      "        \"function_name\"\n",
      "    )\n",
      "    kwargs = {\n",
      "        \"Bucket\": \"{}\".format(buck_name),\n",
      "        \"Key\": \"{}\".format(filename),\n",
      "        \"Body\": byte_stream,\n",
      "    }\n",
      "\n",
      "    client.put_object(**kwargs)\n",
      "    print(\"Finished uploading {} to S3 bucket {}\".format(func_name, buck_name))\n",
      "    if use_s3:\n",
      "        return filename\n",
      "\n",
      "def get_function_config(cfg):\n",
      "    \"\"\"Check whether a function exists or not and return its config\"\"\"\n",
      "\n",
      "    function_name = cfg.get(\"function_name\")\n",
      "    profile_name = cfg.get(\"profile\")\n",
      "    aws_access_key_id = cfg.get(\"aws_access_key_id\")\n",
      "    aws_secret_access_key = cfg.get(\"aws_secret_access_key\")\n",
      "    client = get_client(\n",
      "        \"lambda\",\n",
      "        profile_name,\n",
      "        aws_access_key_id,\n",
      "        aws_secret_access_key,\n",
      "        cfg.get(\"region\"),\n",
      "    )\n",
      "\n",
      "    try:\n",
      "        return client.get_function(FunctionName=function_name)\n",
      "    except client.exceptions.ResourceNotFoundException as e:\n",
      "        if \"Function not found\" in str(e):\n",
      "            return False\n",
      "\n",
      "def get_concurrency(cfg):\n",
      "    \"\"\"Return the Reserved Concurrent Executions if present in the config\"\"\"\n",
      "    concurrency = int(cfg.get(\"concurrency\", 0))\n",
      "    return max(0, concurrency)\n",
      "\n",
      "def _normalize(name):\n",
      "    '''Transform \"Firstname [Middlenames] Lastname\" into\n",
      "    \"Lastname, Firstname [Middlenames]\".'''\n",
      "    split = name.split()\n",
      "    if len(split) == 1:\n",
      "        return name\n",
      "    return split[-1] + ', ' + ' '.join(name[0:-1])\n",
      "\n",
      "def _make_starttag(tag, attrs):\n",
      "    'Write a starttag.'\n",
      "    out = '<' + tag\n",
      "    for key in attrs:\n",
      "        out += ' {}=\"{}\"'.format(key, html.escape(attrs[key]))\n",
      "    out += '>'\n",
      "    return out\n",
      "\n",
      "def _make_xml_elem(tag, text, attr = []):\n",
      "    'Write a flat xml element.'\n",
      "    out = '    <' + tag\n",
      "    for (key, val) in attr:\n",
      "        out += ' {}=\"{}\"'.format(key, val)\n",
      "    if text:\n",
      "        out += '>{}</{}>\\n'.format(text, tag)\n",
      "    else:\n",
      "        out += ' />\\n'\n",
      "    return out\n",
      "\n",
      "def _navp_xml(self, entry, indent_lvl):\n",
      "        'Write xml for an entry and all its subentries.'\n",
      "        xml = self._navp.format('  '*indent_lvl, str(entry.no), entry.text,\n",
      "          entry.target)\n",
      "        for sub in entry.entries:\n",
      "            xml += self._navp_xml(sub, indent_lvl+1)\n",
      "        xml += '  '*indent_lvl + '</navPoint>\\n'\n",
      "        return xml\n",
      "\n",
      "def write_xml(self, uid, title, authors):\n",
      "        'Write the xml code for the table of contents.'\n",
      "        xml = self._head.format(uid, self.max_depth, title)\n",
      "        for aut in authors:\n",
      "            xml += self._doc_author.format(aut)\n",
      "        xml += '  <navMap>\\n'\n",
      "        for entry in self.entries:\n",
      "            xml += self._navp_xml(entry, 2)\n",
      "        xml += '  </navMap>\\n</ncx>'\n",
      "        return xml\n",
      "\n",
      "def __init__(self, name, in_spine = True, guide_title = None,\n",
      "                 guide_type = None):\n",
      "        '''Initialize the object. If the file does not belong in the\n",
      "        reading order, in_spine should be set to False. If it should\n",
      "        appear in the guide, set guide_title and guide_type.'''\n",
      "        self.name = name\n",
      "        (self.ident, ext) = os.path.splitext(name)\n",
      "        name_split = name.rsplit('.', 1)\n",
      "        self.ident = name_split[0]\n",
      "        self.in_spine = in_spine\n",
      "        self.guide_title = guide_title\n",
      "        self.guide_type = guide_type\n",
      "        # Infer media-type from file extension\n",
      "        ext = ext.lower()\n",
      "        if ext in ('.htm', '.html', '.xhtml'):\n",
      "            self.media_type = 'application/xhtml+xml'\n",
      "        elif ext in ('.png', '.gif', '.jpeg'):\n",
      "            self.media_type = 'image/' + ext\n",
      "        elif ext == '.jpg':\n",
      "            self.media_type = 'image/jpeg'\n",
      "        elif ext == '.css':\n",
      "            self.media_type = 'text/css'\n",
      "        elif ext == '.ncx':\n",
      "            self.media_type = 'application/x-dtbncx+xml'\n",
      "        else:\n",
      "            raise ValueError('Can\\'t infer media-type from extension: %s' % ext)\n",
      "\n",
      "def manifest_entry(self):\n",
      "        'Write the XML element for the manifest.'\n",
      "        return _make_xml_elem('item', '',\n",
      "          [\n",
      "            ('href', self.name),\n",
      "            ('id', self.ident),\n",
      "            ('media-type', self.media_type)\n",
      "          ])\n",
      "\n",
      "def spine_entry(self):\n",
      "        '''Write the XML element for the spine.\n",
      "        (Empty string if in_spine is False.)'''\n",
      "        if self.in_spine:\n",
      "            return _make_xml_elem('itemref', '', [('idref', self.ident)])\n",
      "        else:\n",
      "            return ''\n",
      "\n",
      "def guide_entry(self):\n",
      "        '''Write the XML element for the guide.\n",
      "        (Empty string if no guide title and type are given.)'''\n",
      "        if self.guide_title and self.guide_type:\n",
      "            return _make_xml_elem('reference', '',\n",
      "              [\n",
      "                ('title', self.guide_title),\n",
      "                ('type', self.guide_type),\n",
      "                ('href', self.name)\n",
      "              ])\n",
      "        else:\n",
      "            return ''\n",
      "\n",
      "def __init__(self, tag, text, *args):\n",
      "        '''The metadata entry is an XML element. *args is used for\n",
      "        supplying the XML element's attributes as (key, value) pairs.'''\n",
      "        self.tag = tag\n",
      "        self.text = text\n",
      "        self.attr = args\n",
      "\n",
      "def write_xml(self):\n",
      "        'Write the XML element.'\n",
      "        return _make_xml_elem(self.tag, self.text, self.attr)\n",
      "\n",
      "def __repr__(self):\n",
      "        'Returns the text.'\n",
      "        return self.text\n",
      "\n",
      "def __str__(self):\n",
      "        'Returns the text.'\n",
      "        return self.text\n",
      "\n",
      "def __init__(self, date):\n",
      "        '''date must be a string of the form \"YYYY[-MM[-DD]]\". If it is\n",
      "        not of this form, or if the date is invalid, ValueError is\n",
      "        raised.'''\n",
      "        m = self._date_re.match(date) \n",
      "        if not m:\n",
      "            raise ValueError('invalid date format')\n",
      "        year = int(m.group(1))\n",
      "        try:\n",
      "            mon = int(m.group(2)[1:])\n",
      "            if mon < 0 or mon > 12:\n",
      "                raise ValueError('month must be in 1..12')\n",
      "        except IndexError:\n",
      "            pass\n",
      "        try:\n",
      "            day = int(m.group(3)[1:])\n",
      "            datetime.date(year, mon, day) # raises ValueError if invalid\n",
      "        except IndexError:\n",
      "            pass\n",
      "        self.tag = 'dc:date'\n",
      "        self.text = date\n",
      "        self.attr = ()\n",
      "\n",
      "def __init__(self, lang):\n",
      "        '''lang must be a lower-case two-letter language code,\n",
      "        optionally followed by a \"-\" and a upper-case two-letter country\n",
      "        code. (e.g., \"en\", \"en-US\", \"en-UK\", \"de\", \"de-DE\", \"de-AT\")'''\n",
      "        if self._lang_re.match(lang):\n",
      "            self.tag = 'dc:language'\n",
      "            self.text = lang\n",
      "            self.attr = ()\n",
      "        else:\n",
      "            raise ValueError('invalid language format')\n",
      "\n",
      "def __init__(self, name, fileas = None, role = 'aut'):\n",
      "        '''Initialize the object. If the argument \"fileas\" is not given,\n",
      "        \"Last-name, First-name\" is used for the file-as attribute. If\n",
      "        the argument \"role\" is not given, \"aut\" is used for the role\n",
      "        attribute.'''\n",
      "        if not fileas:\n",
      "            fileas = _normalize(name)\n",
      "        self.tag = 'dc:creator'\n",
      "        self.text = name\n",
      "        self.attr = (('opf:file-as', fileas), ('opf:role', role))\n",
      "\n",
      "def __init__(self):\n",
      "        'Initialize.'\n",
      "        self.meta = []\n",
      "        self.filelist = []\n",
      "\n",
      "def write_xml(self):\n",
      "        'Write the XML code for the OPF file.'\n",
      "        metadata = ''\n",
      "        for elem in self.meta:\n",
      "            metadata += elem.write_xml()\n",
      "        manif = ''\n",
      "        spine = ''\n",
      "        guide = ''\n",
      "        for finfo in self.filelist:\n",
      "            manif += finfo.manifest_entry()\n",
      "            spine += finfo.spine_entry()\n",
      "            guide += finfo.guide_entry()\n",
      "        return self._opf.format(metadata, manif, spine, guide)\n",
      "\n",
      "def __init__(self, epub_file):\n",
      "        '''Initialize the EpubBuilder instance. \"epub_file\" is the\n",
      "        filename of the epub to be created.'''\n",
      "        self.epub_f = zipfile.ZipFile(epub_file, 'w', zipfile.ZIP_DEFLATED)\n",
      "        self.epub_f.writestr('mimetype', 'application/epub+zip')\n",
      "        self.epub_f.writestr('META-INF/container.xml', self._container_xml)\n",
      "        self.toc = EpubTOC()\n",
      "        self.opf = _OPFfile()\n",
      "        self.opf.filelist.append(_Fileinfo('toc.ncx', False))\n",
      "        self.opf.filelist.append(_Fileinfo('style.css', False))\n",
      "        self._authors = []\n",
      "        self.opt_meta = {} # Optional metadata (other than authors)\n",
      "        self.content = ''\n",
      "        self.part_no = 0\n",
      "        self.cont_filename = 'part%03d.html' % self.part_no\n",
      "\n",
      "def __enter__(self):\n",
      "        'Return self for use in with ... as ... statement.'\n",
      "        return self\n",
      "\n",
      "def __exit__(self, except_type, except_val, traceback):\n",
      "        'Call finalize() and close the file.'\n",
      "        try:\n",
      "            self.finalize()\n",
      "        finally:\n",
      "            # Close again in case an exception happened in finalize()\n",
      "            self.epub_f.close()\n",
      "        return False\n",
      "\n",
      "def uid(self):\n",
      "        '''Unique identifier of the ebook. (mandatory)\n",
      "\n",
      "        If this property is left unset, a pseudo-random string will be\n",
      "        generated which is long enough for collisions with existing\n",
      "        ebooks to be extremely unlikely.'''\n",
      "        try:\n",
      "            return self._uid\n",
      "        except AttributeError:\n",
      "            import random\n",
      "            from string import (ascii_letters, digits)\n",
      "            alnum = ascii_letters + digits\n",
      "            self.uid = ''.join([random.choice(alnum) for i in range(15)])\n",
      "            return self._uid\n",
      "\n",
      "def uid(self, val):\n",
      "        self._uid = _EpubMeta('dc:identifier', str(val), ('id', 'uid_id'))\n",
      "\n",
      "def title(self):\n",
      "        '''Title of the ebook. (mandatory)\n",
      "\n",
      "        If this property is left unset, it defaults to \"Untitled\".'''\n",
      "        try:\n",
      "            return self._title\n",
      "        except AttributeError:\n",
      "            self.title = 'Untitled'\n",
      "            return self._title\n",
      "\n",
      "def title(self, val):\n",
      "        # If val is not a string, raise TypeError now rather than later.\n",
      "        self._title = _EpubMeta('dc:title', '' + val)\n",
      "\n",
      "def lang(self):\n",
      "        '''Language of the ebook. (mandatory)\n",
      "\n",
      "        The language must be given as a lower-case two-letter code, optionally\n",
      "        followed by a \"-\" and an upper-case two-letter country code.\n",
      "        (e.g., \"en\", \"en-US\", \"en-UK\", \"de\", \"de-DE\", \"de-AT\")\n",
      "\n",
      "        If this property is left unset, it defaults to \"en\".'''\n",
      "        try:\n",
      "            return self._lang\n",
      "        except AttributeError:\n",
      "            self.lang = 'en'\n",
      "            return self._lang\n",
      "\n",
      "def lang(self, val):\n",
      "        self._lang = _EpubLang(val)\n",
      "\n",
      "def author(self):\n",
      "        '''Name of the author. (optional)\n",
      "\n",
      "def author(self, val):\n",
      "        if isinstance(val, Author) or isinstance(val, str):\n",
      "            authors = [val]\n",
      "        else:\n",
      "            authors = val\n",
      "        for aut in authors:\n",
      "            try:\n",
      "                self._authors.append(Author('' + aut))\n",
      "            except TypeError:\n",
      "                # aut is not a string, so it should be an Author object\n",
      "                self._authors.append(aut)\n",
      "\n",
      "def author(self):\n",
      "        self._authors = []\n",
      "\n",
      "def date(self):\n",
      "        '''Publication date. (optional)\n",
      "\n",
      "def date(self, val):\n",
      "        self.opt_meta['date'] = _EpubDate(val)\n",
      "\n",
      "def date(self):\n",
      "        del self._date\n",
      "\n",
      "def rights(self):\n",
      "        'Copyright/licensing information. (optional)'\n",
      "        try:\n",
      "            return self.opt_meta['rights']\n",
      "        except KeyError:\n",
      "            return None\n",
      "\n",
      "def rights(self, val):\n",
      "        self.opt_meta['rights'] = _EpubMeta('dc:rights', '' + val)\n",
      "\n",
      "def rights(self):\n",
      "        del self._rights\n",
      "\n",
      "def publisher(self):\n",
      "        'Publisher name. (optional)'\n",
      "        try:\n",
      "            return self.opt_meta['publisher']\n",
      "        except KeyError:\n",
      "            return None\n",
      "\n",
      "def publisher(self, val):\n",
      "        self.opt_meta['publisher'] = _EpubMeta('dc:publisher', '' + val)\n",
      "\n",
      "def publisher(self):\n",
      "        del self._publisher\n",
      "\n",
      "def style_css(self):\n",
      "        '''CSS stylesheet for the files that are generated by the EpubBuilder\n",
      "        instance. Can be overwritten or extended, but not deleted.'''\n",
      "        return self._style_css\n",
      "\n",
      "def style_css(self, val):\n",
      "        self._style_css = '' + val\n",
      "\n",
      "def titlepage(self, main_title = None, subtitle = None):\n",
      "        '''Create a title page for the ebook. If no main_title is given,\n",
      "        the title attribute of the EpubBuilder instance is used.'''\n",
      "        tp = '<div class=\"getebook-tp\">\\n'\n",
      "        if len(self._authors) >= 1:\n",
      "            if len(self._authors) == 1:\n",
      "                aut_str = str(self._authors[0])\n",
      "            else:\n",
      "                aut_str = ', '.join(str(self._authors[0:-1])) + ', and ' \\\n",
      "                                                       + str(self._authors[-1])\n",
      "            tp += '<div class=\"getebook-tp-authors\">%s</div>\\n' % aut_str\n",
      "        if not main_title:\n",
      "            main_title = str(self.title)\n",
      "        tp += '<div class=\"getebook-tp-title\">%s' % main_title\n",
      "        if subtitle:\n",
      "            tp += '<div class=\"getebook-tp-sub\">%s</div>' % subtitle\n",
      "        tp += '</div>\\n</div>\\n'\n",
      "        self.opf.filelist.insert(0, _Fileinfo('title.html',\n",
      "          guide_title = 'Titlepage', guide_type = 'title-page'))\n",
      "        self.epub_f.writestr('title.html', self._html.format(self.title, tp))\n",
      "\n",
      "def headingpage(self, heading, subtitle = None, toc_text = None):\n",
      "        '''Create a page containing only a (large) heading, optionally\n",
      "        with a smaller subtitle. If toc_text is not given, it defaults\n",
      "        to the heading.'''\n",
      "        self.new_part()\n",
      "        tag = 'h%d' % min(6, self.toc.depth)\n",
      "        self.content += '<div class=\"getebook-tp\">'\n",
      "        self.content += '<{} class=\"getebook-tp-title\">{}'.format(tag, heading)\n",
      "        if subtitle:\n",
      "            self.content += '<div class=\"getebook-tp-sub\">%s</div>' % subtitle\n",
      "        self.content += '</%s>\\n' % tag\n",
      "        if not toc_text:\n",
      "            toc_text = heading\n",
      "        self.toc.new_entry(toc_text, self.cont_filename)\n",
      "        self.new_part()\n",
      "\n",
      "def insert_file(self, name, in_spine = False, guide_title = None,\n",
      "      guide_type = None, arcname = None):\n",
      "        '''Include an external file into the ebook. By default, it will\n",
      "        be added to the archive under its basename; the argument\n",
      "        \"arcname\" can be used to specify a different name.'''\n",
      "        if not arcname:\n",
      "            arcname = os.path.basename(name)\n",
      "        self.opf.filelist.append(_Fileinfo(arcname, in_spine, guide_title,\n",
      "                                 guide_type))\n",
      "        self.epub_f.write(name, arcname)\n",
      "\n",
      "def add_file(self, arcname, str_or_bytes, in_spine = False,\n",
      "      guide_title = None, guide_type = None):\n",
      "        '''Add the string or bytes instance str_or_bytes to the archive\n",
      "        under the name arcname.'''\n",
      "        self.opf.filelist.append(_Fileinfo(arcname, in_spine, guide_title,\n",
      "                                 guide_type))\n",
      "        self.epub_f.writestr(arcname, str_or_bytes)\n",
      "\n",
      "def false_heading(self, elem):\n",
      "        '''Handle a \"false heading\", i.e., text that appears in heading\n",
      "        tags in the source even though it is not a chapter heading.'''\n",
      "        elem.attrs['class'] = 'getebook-false-h'\n",
      "        elem.tag = 'p'\n",
      "        self.handle_elem(elem)\n",
      "\n",
      "def _heading(self, elem):\n",
      "        '''Write a heading.'''\n",
      "        # Handle paragraph heading if we have one waiting (see the\n",
      "        # par_heading method). We don\\'t use _handle_par_h here because\n",
      "        # we merge it with the subsequent proper heading.\n",
      "        try:\n",
      "            par_h = self.par_h\n",
      "            del self.par_h\n",
      "        except AttributeError:\n",
      "            toc_text = elem.text\n",
      "        else:\n",
      "            # There is a waiting paragraph heading, we merge it with the\n",
      "            # new heading.\n",
      "            toc_text = par_h.text + '. ' + elem.text\n",
      "            par_h.tag = 'div'\n",
      "            par_h.attrs['class'] = 'getebook-small-h'\n",
      "            elem.children.insert(0, par_h)\n",
      "        # Set the class attribute value.\n",
      "        elem.attrs['class'] = 'getebook-chapter-h'\n",
      "        self.toc.new_entry(toc_text, self.cont_filename)\n",
      "        # Add heading to the epub.\n",
      "        tag = 'h%d' % min(self.toc.depth, 6)\n",
      "        self.content += _make_starttag(tag, elem.attrs)\n",
      "        for elem in elem.children:\n",
      "            self.handle_elem(elem)\n",
      "        self.content += '</%s>\\n' % tag\n",
      "\n",
      "def par_heading(self, elem):\n",
      "        '''Handle a \"paragraph heading\", i.e., a chaper heading or part\n",
      "        of a chapter heading inside paragraph tags. If it is immediately\n",
      "        followed by a heading, they will be merged into one.'''\n",
      "        self.par_h = elem\n",
      "\n",
      "def _handle_par_h(self):\n",
      "        'Check if there is a waiting paragraph heading and handle it.'\n",
      "        try:\n",
      "            self._heading(self.par_h)\n",
      "        except AttributeError:\n",
      "            pass\n",
      "\n",
      "def handle_elem(self, elem):\n",
      "        'Handle html element as supplied by getebook.EbookParser.'\n",
      "        try:\n",
      "            tag = elem.tag\n",
      "        except AttributeError:\n",
      "            # elem should be a string\n",
      "            is_string = True\n",
      "            tag = None\n",
      "        else:\n",
      "            is_string = False\n",
      "        if tag in getebook._headings:\n",
      "            self._heading(elem)\n",
      "        else:\n",
      "            # Handle waiting par_h if necessary (see par_heading)\n",
      "            try:\n",
      "                self._heading(self.par_h)\n",
      "            except AttributeError:\n",
      "                pass\n",
      "            if is_string:\n",
      "                self.content += elem\n",
      "            elif tag == 'br':\n",
      "                self.content += '<br />\\n'\n",
      "            elif tag == 'img':\n",
      "                self.content += self._handle_image(elem.attrs) + '\\n'\n",
      "            elif tag == 'a' or tag == 'noscript':\n",
      "                # Ignore tag, just write child elements\n",
      "                for child in elem.children:\n",
      "                    self.handle_elem(child)\n",
      "            else:\n",
      "                self.content += _make_starttag(tag, elem.attrs)\n",
      "                for child in elem.children:\n",
      "                    self.handle_elem(child)\n",
      "                self.content += '</%s>' % tag\n",
      "                if tag == 'p':\n",
      "                    self.content += '\\n'\n",
      "\n",
      "def _handle_image(self, attrs):\n",
      "        'Returns the alt text of an image tag.'\n",
      "        try:\n",
      "            return attrs['alt']\n",
      "        except KeyError:\n",
      "            return ''\n",
      "\n",
      "def new_part(self):\n",
      "        '''Begin a new part of the epub. Write the current html document\n",
      "        to the archive and begin a new one.'''\n",
      "        # Handle waiting par_h (see par_heading)\n",
      "        try:\n",
      "            self._heading(self.par_h)\n",
      "        except AttributeError:\n",
      "            pass\n",
      "        if self.content:\n",
      "            html = self._html.format(self.title, self.content)\n",
      "            self.epub_f.writestr(self.cont_filename, html)\n",
      "            self.part_no += 1\n",
      "        self.content = ''\n",
      "        self.cont_filename = 'part%03d.html' % self.part_no\n",
      "        self.opf.filelist.append(_Fileinfo(self.cont_filename))\n",
      "\n",
      "def __init__(self):\n",
      "        columns = ['mean_height', 'min_height', 'max_height', 'mean_width', 'min_width', 'max_width', 'time', 'girth','id']\n",
      "        self.data = DataFrame(columns=columns)\n",
      "        self.event = []\n",
      "\n",
      "def subscribe(ch, method, properties, body):\n",
      "        \"\"\"\n",
      "        prints the body message. It's the default callback method\n",
      "        :param ch: keep null\n",
      "        :param method: keep null\n",
      "        :param properties: keep null\n",
      "        :param body: the message\n",
      "        :return:\n",
      "        \"\"\"\n",
      "        #first we get the JSON from body\n",
      "\n",
      "        #we check if it's part of the walking event\n",
      "\n",
      "        #if walking event is completed, we\n",
      "\n",
      "def _wrapper(*args, **kw):\n",
      "            att, retry = 0, 0\n",
      "            while retry < times:\n",
      "                retry += 1\n",
      "                try:\n",
      "                    return func(*args, **kw)\n",
      "                except:\n",
      "                    att += timeout\n",
      "                    if retry < times:\n",
      "                        time.sleep(att)\n",
      "\n",
      "def _wrapper(*args, **kw):\n",
      "            att, retry = 0, 0\n",
      "            while retry < times:\n",
      "                retry += 1\n",
      "                ret = func(*args, **kw)\n",
      "                if ret:\n",
      "                    return ret\n",
      "                att += timeout\n",
      "                time.sleep(att)\n",
      "\n",
      "def wrapper(*args, **kwargs):\n",
      "            if level == \"warn\":\n",
      "                print (\"level:%s, %s is running\" % (level, func.__name__))\n",
      "            elif level == \"info\":\n",
      "                print (\"level:%s, %s is running\" % (level, func.__name__))\n",
      "            return func(*args, **kwargs)\n",
      "\n",
      "def foo(name='foo'):\n",
      "        print(\"i am %s\" % name)\n",
      "\n",
      "def test_iterators_are_a_type(self):\n",
      "        it = iter(range(1,6))\n",
      "\n",
      "        total = 0\n",
      "\n",
      "        for num in it:\n",
      "            total += num\n",
      "\n",
      "        self.assertEqual(15 , total)\n",
      "\n",
      "def test_iterating_with_next(self):\n",
      "        stages = iter(['alpha','beta','gamma'])\n",
      "\n",
      "        try:\n",
      "            self.assertEqual('alpha', next(stages))\n",
      "            next(stages)\n",
      "            self.assertEqual('gamma', next(stages))\n",
      "            next(stages)\n",
      "        except StopIteration as ex:\n",
      "            err_msg = 'Ran out of iterations'\n",
      "\n",
      "        self.assertRegex(err_msg, 'Ran out')\n",
      "\n",
      "def add_ten(self, item):\n",
      "        return item + 10\n",
      "\n",
      "def test_map_transforms_elements_of_a_list(self):\n",
      "        seq = [1, 2, 3]\n",
      "        mapped_seq = list()\n",
      "\n",
      "        mapping = map(self.add_ten, seq)\n",
      "\n",
      "        self.assertNotEqual(list, mapping.__class__)\n",
      "        self.assertEqual(map, mapping.__class__)\n",
      "        # In Python 3 built in iterator funcs return iterable view objects\n",
      "        # instead of lists\n",
      "\n",
      "        for item in mapping:\n",
      "            mapped_seq.append(item)\n",
      "\n",
      "        self.assertEqual([11, 12, 13], mapped_seq)\n",
      "\n",
      "        # Note, iterator methods actually return objects of iter type in\n",
      "        # python 3. In python 2 map() would give you a list.\n",
      "\n",
      "def is_even(item):\n",
      "            return (item % 2) == 0\n",
      "\n",
      "def is_big_name(item):\n",
      "            return len(item) > 4\n",
      "\n",
      "def add(self,accum,item):\n",
      "        return accum + item\n",
      "\n",
      "def multiply(self,accum,item):\n",
      "        return accum * item\n",
      "\n",
      "def test_reduce_will_blow_your_mind(self):\n",
      "        import functools\n",
      "        # As of Python 3 reduce() has been demoted from a builtin function\n",
      "        # to the functools module.\n",
      "\n",
      "        result = functools.reduce(self.add, [2, 3, 4])\n",
      "        self.assertEqual(int, result.__class__)\n",
      "        # Reduce() syntax is same as Python 2\n",
      "\n",
      "        self.assertEqual(9, result)\n",
      "\n",
      "        result2 = functools.reduce(self.multiply, [2, 3, 4], 1)\n",
      "        self.assertEqual(24, result2)\n",
      "\n",
      "        # Extra Credit:\n",
      "        # Describe in your own words what reduce does.\n",
      "\n",
      "def test_use_pass_for_iterations_with_no_body(self):\n",
      "        for num in range(1,5):\n",
      "            pass\n",
      "\n",
      "        self.assertEqual(4, num)\n",
      "\n",
      "def make_upcase(line):\n",
      "                    return line.strip().upper()\n",
      "\n",
      "def __init__(self, dict_):\n",
      "        super(self.__class__, self).__init__(dict_)\n",
      "\n",
      "def find(cls, params=None):\n",
      "        if params is None:\n",
      "            params = dict()\n",
      "        response = cls(Api.call('sales/detail_sale', params))\n",
      "        return response.sale\n",
      "\n",
      "def list(cls, params=None):\n",
      "        if params is None:\n",
      "            params = dict()\n",
      "        response = cls(Api.call('sales/list_sales', params))\n",
      "        return response.sale_summary\n",
      "\n",
      "def refund(self, params=None):\n",
      "        if params is None:\n",
      "            params = dict()\n",
      "        if hasattr(self, 'lineitem_id'):\n",
      "            params['lineitem_id'] = self.lineitem_id\n",
      "            url = 'sales/refund_lineitem'\n",
      "        elif hasattr(self, 'invoice_id'):\n",
      "            params['invoice_id'] = self.invoice_id\n",
      "            url = 'sales/refund_invoice'\n",
      "        else:\n",
      "            params['sale_id'] = self.sale_id\n",
      "            url = 'sales/refund_invoice'\n",
      "        return Sale(Api.call(url, params))\n",
      "\n",
      "def stop(self, params=None):\n",
      "        if params is None:\n",
      "            params = dict()\n",
      "        if hasattr(self, 'lineitem_id'):\n",
      "            params['lineitem_id'] = self.lineitem_id\n",
      "            return Api.call('sales/stop_lineitem_recurring', params)\n",
      "        elif hasattr(self, 'sale_id'):\n",
      "            active_lineitems = Util.active(self)\n",
      "            if dict(active_lineitems):\n",
      "                result = dict()\n",
      "                i = 0\n",
      "                for k, v in active_lineitems.items():\n",
      "                    lineitem_id = v\n",
      "                    params = {'lineitem_id': lineitem_id}\n",
      "                    result[i] = Api.call('sales/stop_lineitem_recurring', params)\n",
      "                    i += 1\n",
      "                response = { \"response_code\": \"OK\",\n",
      "                             \"response_message\": str(len(result)) + \" lineitems stopped successfully\"\n",
      "                }\n",
      "            else:\n",
      "                response = {\n",
      "                    \"response_code\": \"NOTICE\",\n",
      "                    \"response_message\": \"No active recurring lineitems\"\n",
      "                }\n",
      "        else:\n",
      "            response = { \"response_code\": \"NOTICE\",\n",
      "                          \"response_message\": \"This method can only be called on a sale or lineitem\"\n",
      "            }\n",
      "        return Sale(response)\n",
      "\n",
      "def active(self):\n",
      "        active_lineitems = Util.active(self)\n",
      "        if dict(active_lineitems):\n",
      "            result = dict()\n",
      "            i = 0\n",
      "            for k, v in active_lineitems.items():\n",
      "                lineitem_id = v\n",
      "                result[i] = lineitem_id\n",
      "                i += 1\n",
      "            response = { \"response_code\": \"ACTIVE\",\n",
      "                         \"response_message\": str(len(result)) + \" active recurring lineitems\"\n",
      "            }\n",
      "        else:\n",
      "            response = {\n",
      "                \"response_code\": \"NOTICE\",\"response_message\":\n",
      "                \"No active recurring lineitems\"\n",
      "            }\n",
      "        return Sale(response)\n",
      "\n",
      "def comment(self, params=None):\n",
      "        if params is None:\n",
      "            params = dict()\n",
      "        params['sale_id'] = self.sale_id\n",
      "        return Sale(Api.call('sales/create_comment', params))\n",
      "\n",
      "def connect_client():\n",
      "    \"\"\"Connects to Mongo client\"\"\"\n",
      "    try:\n",
      "        return MongoClient(app.config['DB_HOST'], int(app.config['DB_PORT']))\n",
      "    except errors.ConnectionFailure as e:\n",
      "        raise e\n",
      "\n",
      "def get_db():\n",
      "    \"\"\"Connects to Mongo database\"\"\"\n",
      "    if not hasattr(g, 'mongo_client'):\n",
      "        g.mongo_client = connect_client()\n",
      "        g.mongo_db = getattr(g.mongo_client, app.config['DB_NAME'])\n",
      "        g.groups_collection = g.mongo_db[os.environ.get('DB_GROUPS_COLLECTION')]\n",
      "    return g.mongo_db\n",
      "\n",
      "def close_db(error):\n",
      "    \"\"\"Closes connection with Mongo client\"\"\"\n",
      "    if hasattr(g, 'mongo_client'):\n",
      "        g.mongo_client.close()\n",
      "\n",
      "def index():\n",
      "    \"\"\"Landing page for SciNet\"\"\"\n",
      "    return render_template(\"index.html\")\n",
      "\n",
      "def faq():\n",
      "    \"\"\"FAQ page for SciNet\"\"\"\n",
      "    return render_template(\"faq.html\")\n",
      "\n",
      "def leaderboard():\n",
      "    \"\"\"Leaderboard page for SciNet\"\"\"\n",
      "    get_db()\n",
      "    groups = get_groups(g.groups_collection)\n",
      "    return render_template(\"leaderboard.html\", groups=groups)\n",
      "\n",
      "def ping_endpoint():\n",
      "    \"\"\"API endpoint determines potential article hash exists in db\n",
      "\n",
      "    :return: status code 204 -- hash not present, continue submission\n",
      "    :return: status code 201 -- hash already exists, drop submission\n",
      "    \"\"\"\n",
      "    db = get_db()\n",
      "    target_hash = request.form.get('hash')\n",
      "    if db.raw.find({'hash': target_hash}).count():\n",
      "        return Response(status=201)\n",
      "    else:\n",
      "        return Response(status=204)\n",
      "\n",
      "def ArticleEndpoint():\n",
      "    \"\"\"Eventual landing page for searching/retrieving articles\"\"\"\n",
      "    if request.method == 'GET':\n",
      "        return render_template(\"articles.html\")\n",
      "\n",
      "def raw_endpoint():\n",
      "    \"\"\"API endpoint for submitting raw article data\n",
      "\n",
      "    :return: status code 405 - invalid JSON or invalid request type\n",
      "    :return: status code 400 - unsupported content-type or invalid publisher\n",
      "    :return: status code 201 - successful submission\n",
      "    \"\"\"\n",
      "    # Ensure post's content-type is supported\n",
      "    if request.headers['content-type'] == 'application/json':\n",
      "        # Ensure data is a valid JSON\n",
      "        try:\n",
      "            user_submission = json.loads(request.data)\n",
      "        except ValueError:\n",
      "            return Response(status=405)\n",
      "        # generate UID for new entry\n",
      "        uid = get_id()\n",
      "        # store incoming JSON in raw storage\n",
      "        file_path = os.path.join(\n",
      "                        HERE,\n",
      "                        'raw_payloads',\n",
      "                        str(uid)\n",
      "                    )\n",
      "        store_json_to_file(user_submission, file_path)\n",
      "        # hand submission to controller and return Resposne\n",
      "        db = get_db()\n",
      "        controller_response = JSONController(user_submission, db=db, _id=uid).submit()\n",
      "        return controller_response\n",
      "\n",
      "    # User submitted an unsupported content-type\n",
      "    else:\n",
      "        return Response(status=400)\n",
      "\n",
      "def request_new_group():\n",
      "    # Grab submission form data and prepare email message\n",
      "    data = request.json\n",
      "    msg = \"Someone has request that you add {group_name} to the leaderboard \\\n",
      "        groups. The groups website is {group_website} and the submitter can \\\n",
      "        be reached at {submitter_email}.\".format(\n",
      "                                            group_name=data['new_group_name'],\n",
      "                                            group_website=data['new_group_website'],\n",
      "                                            submitter_email=data['submitter_email'])\n",
      "    return Response(status=200)\n",
      "    '''\n",
      "    try:\n",
      "        email(\n",
      "            subject=\"SciNet: A new group has been requested\",\n",
      "            fro=\"no-reply@scinet.osf.io\",\n",
      "            to='harry@scinet.osf.io',\n",
      "            msg=msg)\n",
      "        return Response(status=200)\n",
      "    except:\n",
      "        return Response(status=500)\n",
      "    '''\n",
      "\n",
      "def not_found(error):\n",
      "    return make_response(jsonify( { 'error': 'Page Not Found' } ), 404)\n",
      "\n",
      "def main():\n",
      "    argspec = hashivault_argspec()\n",
      "    argspec['name'] = dict(required=True, type='str')\n",
      "    argspec['mount_point'] = dict(required=False, type='str', default='approle')\n",
      "    module = hashivault_init(argspec)\n",
      "    result = hashivault_approle_role_get(module.params)\n",
      "    if result.get('failed'):\n",
      "        module.fail_json(**result)\n",
      "    else:\n",
      "        module.exit_json(**result)\n",
      "\n",
      "def hashivault_approle_role_get(params):\n",
      "    name = params.get('name')\n",
      "    client = hashivault_auth_client(params)\n",
      "    result = client.get_role(name, mount_point=params.get('mount_point'))\n",
      "    return {'role': result}\n",
      "\n",
      "def parse(self, response):\n",
      "        #obtains links from page to page and passes links to parse_playerURL\n",
      "        sel = Selector(response)    #define selector based on response object (points to urls in start_urls by default) \n",
      "        url_list = sel.xpath('//a[@class=\"display-block padding-0\"]/@href')   #obtain a list of href links that contain relative links of players\n",
      "        for i in url_list:\n",
      "            relative_url = self.clean_str(i.extract())    #i is a selector and hence need to extract it to obtain unicode object\n",
      "            print urljoin(response.url, relative_url)   #urljoin is able to merge absolute and relative paths to form 1 coherent link\n",
      "            req = Request(urljoin(response.url, relative_url),callback=self.parse_playerURL)   #pass on request with new urls to parse_playerURL\n",
      "            req.headers[\"User-Agent\"] = self.random_ua()    \n",
      "            yield req\n",
      "\n",
      "def parse_playerURL(self, response):    \n",
      "        #parses player specific data into items list\n",
      "        site = Selector(response)\n",
      "        items = []\n",
      "        item = PlayerItem()\n",
      "        item['1name'] = (response.url).rsplit(\"/\")[-2].replace(\"-\",\" \")\n",
      "        title = self.clean_str(site.xpath('/html/head/title/text()').extract_first())\n",
      "        item['OVR'] = title.partition(\"FIFA 16 -\")[1].split(\"-\")[0]\n",
      "        item['POS'] = self.clean_str(site.xpath('//div[@class=\"playercard-position\"]/text()').extract_first())\n",
      "        #stats = site.xpath('//div[@class=\"row player-center-container\"]/div/a')\n",
      "        stat_names = site.xpath('//span[@class=\"player-stat-title\"]')\n",
      "        stat_values = site.xpath('//span[contains(@class, \"player-stat-value\")]')\n",
      "        for index in range(len(stat_names)):\n",
      "            attr_name = stat_names[index].xpath('.//text()').extract_first()\n",
      "            item[attr_name] = stat_values[index].xpath('.//text()').extract_first()\n",
      "        items.append(item)\n",
      "        return items\n",
      "\n",
      "def clean_str(self,ustring):    \n",
      "        #removes wierd unicode chars (/u102 bla), whitespaces, tabspaces, etc to form clean string \n",
      "        return str(ustring.encode('ascii', 'replace')).strip()\n",
      "\n",
      "def random_ua(self):\n",
      "        #randomise user-agent from list to reduce chance of being banned\n",
      "        ua  = random.choice(settings.get('USER_AGENT_LIST'))\n",
      "        if ua:\n",
      "            ua='Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36'\n",
      "        return ua\n",
      "\n",
      "def __init__(\n",
      "        self,\n",
      "        credential: \"AsyncTokenCredential\",\n",
      "        subscription_id: str,\n",
      "        base_url: str = \"https://management.azure.com\",\n",
      "        **kwargs: Any\n",
      "    ) -> None:\n",
      "        self._config = SqlVirtualMachineManagementClientConfiguration(credential=credential, subscription_id=subscription_id, **kwargs)\n",
      "        self._client = AsyncARMPipelineClient(base_url=base_url, config=self._config, **kwargs)\n",
      "\n",
      "        client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)}\n",
      "        self._serialize = Serializer(client_models)\n",
      "        self._deserialize = Deserializer(client_models)\n",
      "        self._serialize.client_side_validation = False\n",
      "        self.availability_group_listeners = AvailabilityGroupListenersOperations(self._client, self._config, self._serialize, self._deserialize)\n",
      "        self.operations = Operations(self._client, self._config, self._serialize, self._deserialize)\n",
      "        self.sql_virtual_machine_groups = SqlVirtualMachineGroupsOperations(self._client, self._config, self._serialize, self._deserialize)\n",
      "        self.sql_virtual_machines = SqlVirtualMachinesOperations(self._client, self._config, self._serialize, self._deserialize)\n",
      "\n",
      "def _send_request(\n",
      "        self,\n",
      "        request: HttpRequest,\n",
      "        **kwargs: Any\n",
      "    ) -> Awaitable[AsyncHttpResponse]:\n",
      "        \"\"\"Runs the network request through the client's chained policies.\n",
      "\n",
      "        >>> from azure.core.rest import HttpRequest\n",
      "        >>> request = HttpRequest(\"GET\", \"https://www.example.org/\")\n",
      "        <HttpRequest [GET], url: 'https://www.example.org/'>\n",
      "        >>> response = await client._send_request(request)\n",
      "        <AsyncHttpResponse: 200 OK>\n",
      "\n",
      "        For more information on this code flow, see https://aka.ms/azsdk/python/protocol/quickstart\n",
      "\n",
      "        :param request: The network request you want to make. Required.\n",
      "        :type request: ~azure.core.rest.HttpRequest\n",
      "        :keyword bool stream: Whether the response payload will be streamed. Defaults to False.\n",
      "        :return: The response of your network call. Does not do error handling on your response.\n",
      "        :rtype: ~azure.core.rest.AsyncHttpResponse\n",
      "        \"\"\"\n",
      "\n",
      "        request_copy = deepcopy(request)\n",
      "        request_copy.url = self._client.format_url(request_copy.url)\n",
      "        return self._client.send_request(request_copy, **kwargs)\n",
      "\n",
      "async def close(self) -> None:\n",
      "        await self._client.close()\n",
      "\n",
      "async def __aenter__(self) -> \"SqlVirtualMachineManagementClient\":\n",
      "        await self._client.__aenter__()\n",
      "        return self\n",
      "\n",
      "def __init__(self, bot, settings):\n",
      "\t\tself.bot = bot\n",
      "\t\tself.settings = settings\n",
      "\n",
      "async def settz(self, ctx, *, tz : str = None):\n",
      "\t\t\"\"\"Sets your TimeZone - Overrides your UTC offset - and accounts for DST.\"\"\"\n",
      "\t\tusage = 'Usage: `{}settz [Region/City]`\\nYou can get a list of available TimeZones with `{}listtz`'.format(ctx.prefix, ctx.prefix)\n",
      "\t\tif not tz:\n",
      "\t\t\tself.settings.setGlobalUserStat(ctx.author, \"TimeZone\", None)\n",
      "\t\t\tawait ctx.channel.send(\"*{}*, your TimeZone has been removed!\".format(DisplayName.name(ctx.author)))\n",
      "\t\t\treturn\n",
      "\n",
      "async def listtz(self, ctx, *, tz_search = None):\n",
      "\t\t\"\"\"List all the supported TimeZones in PM.\"\"\"\n",
      "\n",
      "async def tz(self, ctx, *, member = None):\n",
      "\t\t\"\"\"See a member's TimeZone.\"\"\"\n",
      "\t\t# Check if we're suppressing @here and @everyone mentions\n",
      "\t\tif self.settings.getServerStat(ctx.message.guild, \"SuppressMentions\").lower() == \"yes\":\n",
      "\t\t\tsuppress = True\n",
      "\t\telse:\n",
      "\t\t\tsuppress = False\n",
      "\n",
      "async def setoffset(self, ctx, *, offset : str = None):\n",
      "\t\t\"\"\"Set your UTC offset.\"\"\"\n",
      "\n",
      "async def offset(self, ctx, *, member = None):\n",
      "\t\t\"\"\"See a member's UTC offset.\"\"\"\n",
      "\n",
      "async def time(self, ctx, *, offset : str = None):\n",
      "\t\t\"\"\"Get UTC time +- an offset.\"\"\"\n",
      "\t\ttimezone = None\n",
      "\t\tif offset == None:\n",
      "\t\t\tmember = ctx.message.author\n",
      "\t\telse:\n",
      "\t\t\t# Try to get a user first\n",
      "\t\t\tmember = DisplayName.memberForName(offset, ctx.message.guild)\n",
      "\n",
      "def getTimeFromOffset(self, offset):\n",
      "\t\toffset = offset.replace('+', '')\n",
      "\t\t# Split time string by : and get hour/minute values\n",
      "\t\ttry:\n",
      "\t\t\thours, minutes = map(int, offset.split(':'))\n",
      "\t\texcept Exception:\n",
      "\t\t\ttry:\n",
      "\t\t\t\thours = int(offset)\n",
      "\t\t\t\tminutes = 0\n",
      "\t\t\texcept Exception:\n",
      "\t\t\t\treturn None\n",
      "\t\t\t\t# await ctx.channel.send('Offset has to be in +-H:M!')\n",
      "\t\t\t\t# return\n",
      "\t\tmsg = 'UTC'\n",
      "\t\t# Get the time\n",
      "\t\tt = datetime.datetime.utcnow()\n",
      "\t\t# Apply offset\n",
      "\t\tif hours > 0:\n",
      "\t\t\t# Apply positive offset\n",
      "\t\t\tmsg += '+{}'.format(offset)\n",
      "\t\t\ttd = datetime.timedelta(hours=hours, minutes=minutes)\n",
      "\t\t\tnewTime = t + td\n",
      "\t\telif hours < 0:\n",
      "\t\t\t# Apply negative offset\n",
      "\t\t\tmsg += '{}'.format(offset)\n",
      "\t\t\ttd = datetime.timedelta(hours=(-1*hours), minutes=(-1*minutes))\n",
      "\t\t\tnewTime = t - td\n",
      "\t\telse:\n",
      "\t\t\t# No offset\n",
      "\t\t\tnewTime = t\n",
      "\t\treturn { \"zone\" : msg, \"time\" : newTime.strftime(\"%I:%M %p\") }\n",
      "\n",
      "def test_equal_1(self):\n",
      "        self.assertEqual(string_color('Jack'), '79CAE5')\n",
      "\n",
      "def test_equal_2(self):\n",
      "        self.assertEqual(string_color('Joshua'), '6A10D6')\n",
      "\n",
      "def test_equal_3(self):\n",
      "        self.assertEqual(string_color('Joshua Smith'), '8F00FB')\n",
      "\n",
      "def test_equal_4(self):\n",
      "        self.assertEqual(string_color('Hayden Smith'), '7E00EE')\n",
      "\n",
      "def test_equal_5(self):\n",
      "        self.assertEqual(string_color('Mathew Smith'), '8B00F1')\n",
      "\n",
      "def setup_class(cls):\n",
      "        global users, users_autoinc, metadata\n",
      "        metadata = MetaData(testing.db)\n",
      "        users = Table('users', metadata,\n",
      "            Column('user_id', INT, primary_key=True, autoincrement=False),\n",
      "            Column('user_name', VARCHAR(20)),\n",
      "        )\n",
      "        users_autoinc = Table('users_autoinc', metadata,\n",
      "            Column('user_id', INT, primary_key=True,\n",
      "                                    test_needs_autoincrement=True),\n",
      "            Column('user_name', VARCHAR(20)),\n",
      "        )\n",
      "        metadata.create_all()\n",
      "\n",
      "def teardown(self):\n",
      "        testing.db.execute(users.delete())\n",
      "\n",
      "def teardown_class(cls):\n",
      "        metadata.drop_all()\n",
      "\n",
      "def test_no_params_option(self):\n",
      "        stmt = \"SELECT '%'\" + testing.db.dialect.statement_compiler(\n",
      "                                    testing.db.dialect, None).default_from()\n",
      "\n",
      "        conn = testing.db.connect()\n",
      "        result = conn.\\\n",
      "                execution_options(no_parameters=True).\\\n",
      "                scalar(stmt)\n",
      "        eq_(result, '%')\n",
      "\n",
      "def go(conn):\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (?, ?)', (1, 'jack'))\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (?, ?)', [2, 'fred'])\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (?, ?)', [3, 'ed'], [4, 'horse'])\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (?, ?)', (5, 'barney'), (6, 'donkey'))\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (?, ?)', 7, 'sally')\n",
      "            res = conn.execute('select * from users order by user_id')\n",
      "            assert res.fetchall() == [\n",
      "                (1, 'jack'),\n",
      "                (2, 'fred'),\n",
      "                (3, 'ed'),\n",
      "                (4, 'horse'),\n",
      "                (5, 'barney'),\n",
      "                (6, 'donkey'),\n",
      "                (7, 'sally'),\n",
      "                ]\n",
      "            for multiparam, param in [\n",
      "                ((\"jack\", \"fred\"), {}),\n",
      "                (([\"jack\", \"fred\"],), {})\n",
      "            ]:\n",
      "                res = conn.execute(\n",
      "                    \"select * from users where user_name=? or \"\n",
      "                    \"user_name=? order by user_id\",\n",
      "                    *multiparam, **param)\n",
      "                assert res.fetchall() == [\n",
      "                    (1, 'jack'),\n",
      "                    (2, 'fred')\n",
      "                ]\n",
      "            res = conn.execute(\"select * from users where user_name=?\",\n",
      "                \"jack\"\n",
      "            )\n",
      "            assert res.fetchall() == [(1, 'jack')]\n",
      "            conn.execute('delete from users')\n",
      "\n",
      "def go(conn):\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (%s, %s)', [1, 'jack'])\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (%s, %s)', [2, 'ed'], [3, 'horse'])\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (%s, %s)', 4, 'sally')\n",
      "            conn.execute('insert into users (user_id) values (%s)', 5)\n",
      "            res = conn.execute('select * from users order by user_id')\n",
      "            assert res.fetchall() == [(1, 'jack'), (2, 'ed'), (3,\n",
      "                    'horse'), (4, 'sally'), (5, None)]\n",
      "            for multiparam, param in [\n",
      "                ((\"jack\", \"ed\"), {}),\n",
      "                (([\"jack\", \"ed\"],), {})\n",
      "            ]:\n",
      "                res = conn.execute(\n",
      "                    \"select * from users where user_name=%s or \"\n",
      "                    \"user_name=%s order by user_id\",\n",
      "                    *multiparam, **param)\n",
      "                assert res.fetchall() == [\n",
      "                    (1, 'jack'),\n",
      "                    (2, 'ed')\n",
      "                ]\n",
      "            res = conn.execute(\"select * from users where user_name=%s\",\n",
      "                \"jack\"\n",
      "            )\n",
      "            assert res.fetchall() == [(1, 'jack')]\n",
      "\n",
      "            conn.execute('delete from users')\n",
      "\n",
      "def go(conn):\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (%(id)s, %(name)s)', {'id': 1, 'name'\n",
      "                         : 'jack'})\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (%(id)s, %(name)s)', {'id': 2, 'name'\n",
      "                         : 'ed'}, {'id': 3, 'name': 'horse'})\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (%(id)s, %(name)s)', id=4, name='sally'\n",
      "                         )\n",
      "            res = conn.execute('select * from users order by user_id')\n",
      "            assert res.fetchall() == [(1, 'jack'), (2, 'ed'), (3,\n",
      "                    'horse'), (4, 'sally')]\n",
      "            conn.execute('delete from users')\n",
      "\n",
      "def go(conn):\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (:id, :name)', {'id': 1, 'name': 'jack'\n",
      "                         })\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (:id, :name)', {'id': 2, 'name': 'ed'\n",
      "                         }, {'id': 3, 'name': 'horse'})\n",
      "            conn.execute('insert into users (user_id, user_name) '\n",
      "                         'values (:id, :name)', id=4, name='sally')\n",
      "            res = conn.execute('select * from users order by user_id')\n",
      "            assert res.fetchall() == [(1, 'jack'), (2, 'ed'), (3,\n",
      "                    'horse'), (4, 'sally')]\n",
      "            conn.execute('delete from users')\n",
      "\n",
      "def test_exception_wrapping_dbapi(self):\n",
      "        conn = testing.db.connect()\n",
      "        for _c in testing.db, conn:\n",
      "            assert_raises_message(\n",
      "                tsa.exc.DBAPIError,\n",
      "                r\"not_a_valid_statement\",\n",
      "                _c.execute, 'not_a_valid_statement'\n",
      "            )\n",
      "\n",
      "def test_exception_wrapping_non_dbapi_error(self):\n",
      "        e = create_engine('sqlite://')\n",
      "        e.dialect.is_disconnect = is_disconnect = Mock()\n",
      "\n",
      "        with e.connect() as c:\n",
      "            c.connection.cursor = Mock(\n",
      "                    return_value=Mock(\n",
      "                        execute=Mock(\n",
      "                                side_effect=TypeError(\"I'm not a DBAPI error\")\n",
      "                        ))\n",
      "                    )\n",
      "\n",
      "            assert_raises_message(\n",
      "                TypeError,\n",
      "                \"I'm not a DBAPI error\",\n",
      "                c.execute, \"select \"\n",
      "            )\n",
      "            eq_(is_disconnect.call_count, 0)\n",
      "\n",
      "def process_bind_param(self, value, dialect):\n",
      "                raise Exception(\"nope\")\n",
      "\n",
      "def _go(conn):\n",
      "            assert_raises_message(\n",
      "                tsa.exc.StatementError,\n",
      "                r\"nope \\(original cause: Exception: nope\\) u?'SELECT 1 \",\n",
      "                conn.execute,\n",
      "                    select([1]).\\\n",
      "                        where(\n",
      "                            column('foo') == literal('bar', MyType())\n",
      "                        )\n",
      "            )\n",
      "\n",
      "def test_stmt_exception_non_ascii(self):\n",
      "        name = util.u('méil')\n",
      "        with testing.db.connect() as conn:\n",
      "            assert_raises_message(\n",
      "                tsa.exc.StatementError,\n",
      "                util.u(\n",
      "                    \"A value is required for bind parameter 'uname'\"\n",
      "                    r'.*SELECT users.user_name AS .m\\\\xe9il.') if util.py2k\n",
      "                else\n",
      "                    util.u(\n",
      "                        \"A value is required for bind parameter 'uname'\"\n",
      "                        '.*SELECT users.user_name AS .méil.')\n",
      "                    ,\n",
      "                conn.execute,\n",
      "                select([users.c.user_name.label(name)]).where(\n",
      "                                users.c.user_name == bindparam(\"uname\")),\n",
      "                {'uname_incorrect': 'foo'}\n",
      "            )\n",
      "\n",
      "def test_stmt_exception_pickleable_no_dbapi(self):\n",
      "        self._test_stmt_exception_pickleable(Exception(\"hello world\"))\n",
      "\n",
      "def test_stmt_exception_pickleable_plus_dbapi(self):\n",
      "        raw = testing.db.raw_connection()\n",
      "        the_orig = None\n",
      "        try:\n",
      "            try:\n",
      "                cursor = raw.cursor()\n",
      "                cursor.execute(\"SELECTINCORRECT\")\n",
      "            except testing.db.dialect.dbapi.DatabaseError as orig:\n",
      "                # py3k has \"orig\" in local scope...\n",
      "                the_orig = orig\n",
      "        finally:\n",
      "            raw.close()\n",
      "        self._test_stmt_exception_pickleable(the_orig)\n",
      "\n",
      "def _test_stmt_exception_pickleable(self, orig):\n",
      "        for sa_exc in (\n",
      "            tsa.exc.StatementError(\"some error\",\n",
      "                            \"select * from table\",\n",
      "                           {\"foo\":\"bar\"},\n",
      "                            orig),\n",
      "            tsa.exc.InterfaceError(\"select * from table\",\n",
      "                            {\"foo\":\"bar\"},\n",
      "                            orig),\n",
      "            tsa.exc.NoReferencedTableError(\"message\", \"tname\"),\n",
      "            tsa.exc.NoReferencedColumnError(\"message\", \"tname\", \"cname\"),\n",
      "            tsa.exc.CircularDependencyError(\"some message\", [1, 2, 3], [(1, 2), (3, 4)]),\n",
      "        ):\n",
      "            for loads, dumps in picklers():\n",
      "                repickled = loads(dumps(sa_exc))\n",
      "                eq_(repickled.args[0], sa_exc.args[0])\n",
      "                if isinstance(sa_exc, tsa.exc.StatementError):\n",
      "                    eq_(repickled.params, {\"foo\":\"bar\"})\n",
      "                    eq_(repickled.statement, sa_exc.statement)\n",
      "                    if hasattr(sa_exc, \"connection_invalidated\"):\n",
      "                        eq_(repickled.connection_invalidated,\n",
      "                            sa_exc.connection_invalidated)\n",
      "                    eq_(repickled.orig.args[0], orig.args[0])\n",
      "\n",
      "def process_bind_param(self, value, dialect):\n",
      "                raise MyException(\"nope\")\n",
      "\n",
      "def _go(conn):\n",
      "            assert_raises_message(\n",
      "                MyException,\n",
      "                \"nope\",\n",
      "                conn.execute,\n",
      "                    select([1]).\\\n",
      "                        where(\n",
      "                            column('foo') == literal('bar', MyType())\n",
      "                        )\n",
      "            )\n",
      "\n",
      "def test_empty_insert(self):\n",
      "        \"\"\"test that execute() interprets [] as a list with no params\"\"\"\n",
      "\n",
      "        testing.db.execute(users_autoinc.insert().\n",
      "                    values(user_name=bindparam('name', None)), [])\n",
      "        eq_(testing.db.execute(users_autoinc.select()).fetchall(), [(1, None)])\n",
      "\n",
      "def test_engine_level_options(self):\n",
      "        eng = engines.testing_engine(options={'execution_options':\n",
      "                                            {'foo': 'bar'}})\n",
      "        with eng.contextual_connect() as conn:\n",
      "            eq_(conn._execution_options['foo'], 'bar')\n",
      "            eq_(conn.execution_options(bat='hoho')._execution_options['foo'\n",
      "                ], 'bar')\n",
      "            eq_(conn.execution_options(bat='hoho')._execution_options['bat'\n",
      "                ], 'hoho')\n",
      "            eq_(conn.execution_options(foo='hoho')._execution_options['foo'\n",
      "                ], 'hoho')\n",
      "            eng.update_execution_options(foo='hoho')\n",
      "            conn = eng.contextual_connect()\n",
      "            eq_(conn._execution_options['foo'], 'hoho')\n",
      "\n",
      "def test_generative_engine_execution_options(self):\n",
      "        eng = engines.testing_engine(options={'execution_options':\n",
      "                                            {'base': 'x1'}})\n",
      "\n",
      "        eng1 = eng.execution_options(foo=\"b1\")\n",
      "        eng2 = eng.execution_options(foo=\"b2\")\n",
      "        eng1a = eng1.execution_options(bar=\"a1\")\n",
      "        eng2a = eng2.execution_options(foo=\"b3\", bar=\"a2\")\n",
      "\n",
      "        eq_(eng._execution_options,\n",
      "                {'base': 'x1'})\n",
      "        eq_(eng1._execution_options,\n",
      "                {'base': 'x1', 'foo': 'b1'})\n",
      "        eq_(eng2._execution_options,\n",
      "                {'base': 'x1', 'foo': 'b2'})\n",
      "        eq_(eng1a._execution_options,\n",
      "                {'base': 'x1', 'foo': 'b1', 'bar': 'a1'})\n",
      "        eq_(eng2a._execution_options,\n",
      "                {'base': 'x1', 'foo': 'b3', 'bar': 'a2'})\n",
      "        is_(eng1a.pool, eng.pool)\n",
      "\n",
      "        # test pool is shared\n",
      "        eng2.dispose()\n",
      "        is_(eng1a.pool, eng2.pool)\n",
      "        is_(eng.pool, eng2.pool)\n",
      "\n",
      "def l1(*arg, **kw):\n",
      "            canary.append(\"l1\")\n",
      "\n",
      "def l2(*arg, **kw):\n",
      "            canary.append(\"l2\")\n",
      "\n",
      "def l3(*arg, **kw):\n",
      "            canary.append(\"l3\")\n",
      "\n",
      "def l1(*arg, **kw):\n",
      "            pass\n",
      "\n",
      "def execute(self, stmt, params=None, **kw):\n",
      "                if \"test unicode returns\" in stmt:\n",
      "                    raise self.engine.dialect.dbapi.DatabaseError(\"boom\")\n",
      "                else:\n",
      "                    return super(MockCursor, self).execute(stmt, params, **kw)\n",
      "\n",
      "def test_works_after_dispose(self):\n",
      "        eng = create_engine(testing.db.url)\n",
      "        for i in range(3):\n",
      "            eq_(eng.scalar(select([1])), 1)\n",
      "            eng.dispose()\n",
      "\n",
      "def test_works_after_dispose_testing_engine(self):\n",
      "        eng = engines.testing_engine()\n",
      "        for i in range(3):\n",
      "            eq_(eng.scalar(select([1])), 1)\n",
      "            eng.dispose()\n",
      "\n",
      "def define_tables(cls, metadata):\n",
      "        cls.table = Table('exec_test', metadata,\n",
      "            Column('a', Integer),\n",
      "            Column('b', Integer),\n",
      "            test_needs_acid=True\n",
      "        )\n",
      "\n",
      "def go(conn, x, value=None):\n",
      "            if is_transaction:\n",
      "                conn = conn.connection\n",
      "            conn.execute(self.table.insert().values(a=x, b=value))\n",
      "\n",
      "def go(conn, x, value=None):\n",
      "            if is_transaction:\n",
      "                conn = conn.connection\n",
      "            conn.execute(self.table.insert().values(a=x, b=value))\n",
      "            raise Exception(\"breakage\")\n",
      "\n",
      "def _assert_no_data(self):\n",
      "        eq_(\n",
      "            testing.db.scalar(self.table.count()), 0\n",
      "        )\n",
      "\n",
      "def _assert_fn(self, x, value=None):\n",
      "        eq_(\n",
      "            testing.db.execute(self.table.select()).fetchall(),\n",
      "            [(x, value)]\n",
      "        )\n",
      "\n",
      "def test_transaction_engine_ctx_commit(self):\n",
      "        fn = self._trans_fn()\n",
      "        ctx = testing.db.begin()\n",
      "        testing.run_as_contextmanager(ctx, fn, 5, value=8)\n",
      "        self._assert_fn(5, value=8)\n",
      "\n",
      "def test_transaction_engine_ctx_begin_fails(self):\n",
      "        engine = engines.testing_engine()\n",
      "\n",
      "        mock_connection = Mock(\n",
      "            return_value=Mock(\n",
      "                        begin=Mock(side_effect=Exception(\"boom\"))\n",
      "                    )\n",
      "        )\n",
      "        engine._connection_cls = mock_connection\n",
      "        assert_raises(\n",
      "            Exception,\n",
      "            engine.begin\n",
      "        )\n",
      "\n",
      "        eq_(\n",
      "            mock_connection.return_value.close.mock_calls,\n",
      "            [call()]\n",
      "        )\n",
      "\n",
      "def test_transaction_engine_ctx_rollback(self):\n",
      "        fn = self._trans_rollback_fn()\n",
      "        ctx = testing.db.begin()\n",
      "        assert_raises_message(\n",
      "            Exception,\n",
      "            \"breakage\",\n",
      "            testing.run_as_contextmanager, ctx, fn, 5, value=8\n",
      "        )\n",
      "        self._assert_no_data()\n",
      "\n",
      "def test_transaction_tlocal_engine_ctx_commit(self):\n",
      "        fn = self._trans_fn()\n",
      "        engine = engines.testing_engine(options=dict(\n",
      "                                strategy='threadlocal',\n",
      "                                pool=testing.db.pool))\n",
      "        ctx = engine.begin()\n",
      "        testing.run_as_contextmanager(ctx, fn, 5, value=8)\n",
      "        self._assert_fn(5, value=8)\n",
      "\n",
      "def test_transaction_tlocal_engine_ctx_rollback(self):\n",
      "        fn = self._trans_rollback_fn()\n",
      "        engine = engines.testing_engine(options=dict(\n",
      "                                strategy='threadlocal',\n",
      "                                pool=testing.db.pool))\n",
      "        ctx = engine.begin()\n",
      "        assert_raises_message(\n",
      "            Exception,\n",
      "            \"breakage\",\n",
      "            testing.run_as_contextmanager, ctx, fn, 5, value=8\n",
      "        )\n",
      "        self._assert_no_data()\n",
      "\n",
      "def test_transaction_connection_ctx_commit(self):\n",
      "        fn = self._trans_fn(True)\n",
      "        conn = testing.db.connect()\n",
      "        ctx = conn.begin()\n",
      "        testing.run_as_contextmanager(ctx, fn, 5, value=8)\n",
      "        self._assert_fn(5, value=8)\n",
      "\n",
      "def test_transaction_connection_ctx_rollback(self):\n",
      "        fn = self._trans_rollback_fn(True)\n",
      "        conn = testing.db.connect()\n",
      "        ctx = conn.begin()\n",
      "        assert_raises_message(\n",
      "            Exception,\n",
      "            \"breakage\",\n",
      "            testing.run_as_contextmanager, ctx, fn, 5, value=8\n",
      "        )\n",
      "        self._assert_no_data()\n",
      "\n",
      "def test_connection_as_ctx(self):\n",
      "        fn = self._trans_fn()\n",
      "        ctx = testing.db.connect()\n",
      "        testing.run_as_contextmanager(ctx, fn, 5, value=8)\n",
      "        # autocommit is on\n",
      "        self._assert_fn(5, value=8)\n",
      "\n",
      "def test_connect_as_ctx_noautocommit(self):\n",
      "        fn = self._trans_fn()\n",
      "        self._assert_no_data()\n",
      "        ctx = testing.db.connect().execution_options(autocommit=False)\n",
      "        testing.run_as_contextmanager(ctx, fn, 5, value=8)\n",
      "        # autocommit is off\n",
      "        self._assert_no_data()\n",
      "\n",
      "def test_transaction_engine_fn_commit(self):\n",
      "        fn = self._trans_fn()\n",
      "        testing.db.transaction(fn, 5, value=8)\n",
      "        self._assert_fn(5, value=8)\n",
      "\n",
      "def test_transaction_engine_fn_rollback(self):\n",
      "        fn = self._trans_rollback_fn()\n",
      "        assert_raises_message(\n",
      "            Exception,\n",
      "            \"breakage\",\n",
      "            testing.db.transaction, fn, 5, value=8\n",
      "        )\n",
      "        self._assert_no_data()\n",
      "\n",
      "def test_transaction_connection_fn_commit(self):\n",
      "        fn = self._trans_fn()\n",
      "        conn = testing.db.connect()\n",
      "        conn.transaction(fn, 5, value=8)\n",
      "        self._assert_fn(5, value=8)\n",
      "\n",
      "def test_transaction_connection_fn_rollback(self):\n",
      "        fn = self._trans_rollback_fn()\n",
      "        conn = testing.db.connect()\n",
      "        assert_raises(\n",
      "            Exception,\n",
      "            conn.transaction, fn, 5, value=8\n",
      "        )\n",
      "        self._assert_no_data()\n",
      "\n",
      "def setup_class(cls):\n",
      "        global users, metadata\n",
      "        metadata = MetaData(testing.db)\n",
      "        users = Table('users', metadata,\n",
      "            Column('user_id', INT, primary_key=True,\n",
      "                            test_needs_autoincrement=True),\n",
      "            Column('user_name', VARCHAR(20)),\n",
      "        )\n",
      "        metadata.create_all()\n",
      "\n",
      "def teardown(self):\n",
      "        testing.db.execute(users.delete())\n",
      "\n",
      "def teardown_class(cls):\n",
      "        metadata.drop_all()\n",
      "\n",
      "def test_cache(self):\n",
      "        conn = testing.db.connect()\n",
      "        cache = {}\n",
      "        cached_conn = conn.execution_options(compiled_cache=cache)\n",
      "\n",
      "        ins = users.insert()\n",
      "        cached_conn.execute(ins, {'user_name':'u1'})\n",
      "        cached_conn.execute(ins, {'user_name':'u2'})\n",
      "        cached_conn.execute(ins, {'user_name':'u3'})\n",
      "        assert len(cache) == 1\n",
      "        eq_(conn.execute(\"select count(*) from users\").scalar(), 3)\n",
      "\n",
      "def dump(sql, *multiparams, **params):\n",
      "            buf.write(util.text_type(sql.compile(dialect=engine.dialect)))\n",
      "\n",
      "def test_sequence_not_duped(self):\n",
      "        engine, buf = self._engine_fixture()\n",
      "        metadata = MetaData()\n",
      "        t = Table('testtable', metadata,\n",
      "           Column('pk', Integer, Sequence('testtable_pk_seq'), primary_key=True)\n",
      "        )\n",
      "\n",
      "        t.create(engine)\n",
      "        t.drop(engine)\n",
      "\n",
      "        eq_(\n",
      "            re.findall(r'CREATE (\\w+)', buf.getvalue()),\n",
      "            [\"SEQUENCE\", \"TABLE\"]\n",
      "        )\n",
      "\n",
      "        eq_(\n",
      "            re.findall(r'DROP (\\w+)', buf.getvalue()),\n",
      "            [\"SEQUENCE\", \"TABLE\"]\n",
      "        )\n",
      "\n",
      "def __init__(self, l):\n",
      "                self.l = l\n",
      "\n",
      "def __len__(self):\n",
      "                return len(self.l)\n",
      "\n",
      "def __getitem__(self, i):\n",
      "                return list.__getitem__(self.l, i)\n",
      "\n",
      "def test_no_rowcount_on_selects_inserts(self):\n",
      "        \"\"\"assert that rowcount is only called on deletes and updates.\n",
      "\n",
      "        This because cursor.rowcount may can be expensive on some dialects\n",
      "        such as Firebird, however many dialects require it be called\n",
      "        before the cursor is closed.\n",
      "\n",
      "        \"\"\"\n",
      "\n",
      "        metadata = self.metadata\n",
      "\n",
      "        engine = engines.testing_engine()\n",
      "\n",
      "        t = Table('t1', metadata,\n",
      "            Column('data', String(10))\n",
      "        )\n",
      "        metadata.create_all(engine)\n",
      "\n",
      "        with patch.object(engine.dialect.execution_ctx_cls, \"rowcount\") as mock_rowcount:\n",
      "            mock_rowcount.__get__ = Mock()\n",
      "            engine.execute(t.insert(),\n",
      "                                {'data': 'd1'},\n",
      "                                {'data': 'd2'},\n",
      "                                {'data': 'd3'})\n",
      "\n",
      "            eq_(len(mock_rowcount.__get__.mock_calls), 0)\n",
      "\n",
      "            eq_(\n",
      "                    engine.execute(t.select()).fetchall(),\n",
      "                    [('d1', ), ('d2', ), ('d3', )]\n",
      "            )\n",
      "            eq_(len(mock_rowcount.__get__.mock_calls), 0)\n",
      "\n",
      "            engine.execute(t.update(), {'data': 'd4'})\n",
      "\n",
      "            eq_(len(mock_rowcount.__get__.mock_calls), 1)\n",
      "\n",
      "            engine.execute(t.delete())\n",
      "            eq_(len(mock_rowcount.__get__.mock_calls), 2)\n",
      "\n",
      "def test_rowproxy_is_sequence(self):\n",
      "        import collections\n",
      "        from sqlalchemy.engine import RowProxy\n",
      "\n",
      "        row = RowProxy(object(), ['value'], [None], {'key'\n",
      "                         : (None, None, 0), 0: (None, None, 0)})\n",
      "        assert isinstance(row, collections.Sequence)\n",
      "\n",
      "def test_row_c_sequence_check(self):\n",
      "        import csv\n",
      "        import collections\n",
      "\n",
      "        metadata = MetaData()\n",
      "        metadata.bind = 'sqlite://'\n",
      "        users = Table('users', metadata,\n",
      "            Column('id', Integer, primary_key=True),\n",
      "            Column('name', String(40)),\n",
      "        )\n",
      "        users.create()\n",
      "\n",
      "        users.insert().execute(name='Test')\n",
      "        row = users.select().execute().fetchone()\n",
      "\n",
      "        s = util.StringIO()\n",
      "        writer = csv.writer(s)\n",
      "        # csv performs PySequenceCheck call\n",
      "        writer.writerow(row)\n",
      "        assert s.getvalue().strip() == '1,Test'\n",
      "\n",
      "def test_empty_accessors(self):\n",
      "        statements = [\n",
      "            (\n",
      "                \"select 1\",\n",
      "                [\n",
      "                    lambda r: r.last_inserted_params(),\n",
      "                    lambda r: r.last_updated_params(),\n",
      "                    lambda r: r.prefetch_cols(),\n",
      "                    lambda r: r.postfetch_cols(),\n",
      "                    lambda r : r.inserted_primary_key\n",
      "                ],\n",
      "                \"Statement is not a compiled expression construct.\"\n",
      "            ),\n",
      "            (\n",
      "                select([1]),\n",
      "                [\n",
      "                    lambda r: r.last_inserted_params(),\n",
      "                    lambda r : r.inserted_primary_key\n",
      "                ],\n",
      "                r\"Statement is not an insert\\(\\) expression construct.\"\n",
      "            ),\n",
      "            (\n",
      "                select([1]),\n",
      "                [\n",
      "                    lambda r: r.last_updated_params(),\n",
      "                ],\n",
      "                r\"Statement is not an update\\(\\) expression construct.\"\n",
      "            ),\n",
      "            (\n",
      "                select([1]),\n",
      "                [\n",
      "                    lambda r: r.prefetch_cols(),\n",
      "                    lambda r : r.postfetch_cols()\n",
      "                ],\n",
      "                r\"Statement is not an insert\\(\\) \"\n",
      "                r\"or update\\(\\) expression construct.\"\n",
      "            ),\n",
      "        ]\n",
      "\n",
      "        for stmt, meths, msg in statements:\n",
      "            r = testing.db.execute(stmt)\n",
      "            try:\n",
      "                for meth in meths:\n",
      "                    assert_raises_message(\n",
      "                        tsa.exc.InvalidRequestError,\n",
      "                        msg,\n",
      "                        meth, r\n",
      "                    )\n",
      "\n",
      "            finally:\n",
      "                r.close()\n",
      "\n",
      "def test_dialect_conn_options(self):\n",
      "        engine = testing_engine(\"sqlite://\", options=dict(_initialize=False))\n",
      "        engine.dialect = Mock()\n",
      "        conn = engine.connect()\n",
      "        c2 = conn.execution_options(foo=\"bar\")\n",
      "        eq_(\n",
      "            engine.dialect.set_connection_execution_options.mock_calls,\n",
      "            [call(c2, {\"foo\": \"bar\"})]\n",
      "        )\n",
      "\n",
      "def test_dialect_engine_options(self):\n",
      "        engine = testing_engine(\"sqlite://\")\n",
      "        engine.dialect = Mock()\n",
      "        e2 = engine.execution_options(foo=\"bar\")\n",
      "        eq_(\n",
      "            engine.dialect.set_engine_execution_options.mock_calls,\n",
      "            [call(e2, {\"foo\": \"bar\"})]\n",
      "        )\n",
      "\n",
      "def test_dialect_engine_construction_options(self):\n",
      "        dialect = Mock()\n",
      "        engine = Engine(Mock(), dialect, Mock(),\n",
      "                                execution_options={\"foo\": \"bar\"})\n",
      "        eq_(\n",
      "            dialect.set_engine_execution_options.mock_calls,\n",
      "            [call(engine, {\"foo\": \"bar\"})]\n",
      "        )\n",
      "\n",
      "def test_propagate_engine_to_connection(self):\n",
      "        engine = testing_engine(\"sqlite://\",\n",
      "                        options=dict(execution_options={\"foo\": \"bar\"}))\n",
      "        conn = engine.connect()\n",
      "        eq_(conn._execution_options, {\"foo\": \"bar\"})\n",
      "\n",
      "def test_propagate_option_engine_to_connection(self):\n",
      "        e1 = testing_engine(\"sqlite://\",\n",
      "                        options=dict(execution_options={\"foo\": \"bar\"}))\n",
      "        e2 = e1.execution_options(bat=\"hoho\")\n",
      "        c1 = e1.connect()\n",
      "        c2 = e2.connect()\n",
      "        eq_(c1._execution_options, {\"foo\": \"bar\"})\n",
      "        eq_(c2._execution_options, {\"foo\": \"bar\", \"bat\": \"hoho\"})\n",
      "\n",
      "def setup_class(cls):\n",
      "        from sqlalchemy.engine import base, default\n",
      "        cls.engine = engine = testing_engine('sqlite://')\n",
      "        m = MetaData()\n",
      "        cls.table = t = Table('test', m,\n",
      "            Column('x', Integer, primary_key=True),\n",
      "            Column('y', String(50, convert_unicode='force'))\n",
      "        )\n",
      "        m.create_all(engine)\n",
      "        engine.execute(t.insert(), [\n",
      "            {'x':i, 'y':\"t_%d\" % i} for i in range(1, 12)\n",
      "        ])\n",
      "\n",
      "def get_result_proxy(self):\n",
      "                return cls(self)\n",
      "\n",
      "def test_plain(self):\n",
      "        self._test_proxy(_result.ResultProxy)\n",
      "\n",
      "def test_buffered_row_result_proxy(self):\n",
      "        self._test_proxy(_result.BufferedRowResultProxy)\n",
      "\n",
      "def test_fully_buffered_result_proxy(self):\n",
      "        self._test_proxy(_result.FullyBufferedResultProxy)\n",
      "\n",
      "def test_buffered_column_result_proxy(self):\n",
      "        self._test_proxy(_result.BufferedColumnResultProxy)\n",
      "\n",
      "def tearDown(self):\n",
      "        Engine.dispatch._clear()\n",
      "        Engine._has_events = False\n",
      "\n",
      "def _assert_stmts(self, expected, received):\n",
      "        orig = list(received)\n",
      "        for stmt, params, posn in expected:\n",
      "            if not received:\n",
      "                assert False, \"Nothing available for stmt: %s\" % stmt\n",
      "            while received:\n",
      "                teststmt, testparams, testmultiparams = \\\n",
      "                    received.pop(0)\n",
      "                teststmt = re.compile(r'[\\n\\t ]+', re.M).sub(' ',\n",
      "                        teststmt).strip()\n",
      "                if teststmt.startswith(stmt) and (testparams\n",
      "                        == params or testparams == posn):\n",
      "                    break\n",
      "\n",
      "def test_per_engine_independence(self):\n",
      "        e1 = testing_engine(config.db_url)\n",
      "        e2 = testing_engine(config.db_url)\n",
      "\n",
      "        canary = Mock()\n",
      "        event.listen(e1, \"before_execute\", canary)\n",
      "        s1 = select([1])\n",
      "        s2 = select([2])\n",
      "        e1.execute(s1)\n",
      "        e2.execute(s2)\n",
      "        eq_(\n",
      "            [arg[1][1] for arg in canary.mock_calls], [s1]\n",
      "        )\n",
      "        event.listen(e2, \"before_execute\", canary)\n",
      "        e1.execute(s1)\n",
      "        e2.execute(s2)\n",
      "        eq_([arg[1][1] for arg in canary.mock_calls], [s1, s1, s2])\n",
      "\n",
      "def test_per_engine_plus_global(self):\n",
      "        canary = Mock()\n",
      "        event.listen(Engine, \"before_execute\", canary.be1)\n",
      "        e1 = testing_engine(config.db_url)\n",
      "        e2 = testing_engine(config.db_url)\n",
      "\n",
      "        event.listen(e1, \"before_execute\", canary.be2)\n",
      "\n",
      "        event.listen(Engine, \"before_execute\", canary.be3)\n",
      "        e1.connect()\n",
      "        e2.connect()\n",
      "\n",
      "        e1.execute(select([1]))\n",
      "        eq_(canary.be1.call_count, 1)\n",
      "        eq_(canary.be2.call_count, 1)\n",
      "\n",
      "        e2.execute(select([1]))\n",
      "\n",
      "        eq_(canary.be1.call_count, 2)\n",
      "        eq_(canary.be2.call_count, 1)\n",
      "        eq_(canary.be3.call_count, 2)\n",
      "\n",
      "def test_per_connection_plus_engine(self):\n",
      "        canary = Mock()\n",
      "        e1 = testing_engine(config.db_url)\n",
      "\n",
      "        event.listen(e1, \"before_execute\", canary.be1)\n",
      "\n",
      "        conn = e1.connect()\n",
      "        event.listen(conn, \"before_execute\", canary.be2)\n",
      "        conn.execute(select([1]))\n",
      "\n",
      "        eq_(canary.be1.call_count, 1)\n",
      "        eq_(canary.be2.call_count, 1)\n",
      "\n",
      "        conn._branch().execute(select([1]))\n",
      "        eq_(canary.be1.call_count, 2)\n",
      "        eq_(canary.be2.call_count, 2)\n",
      "\n",
      "def test_add_event_after_connect(self):\n",
      "        # new feature as of #2978\n",
      "        canary = Mock()\n",
      "        e1 = create_engine(config.db_url)\n",
      "        assert not e1._has_events\n",
      "\n",
      "        conn = e1.connect()\n",
      "\n",
      "        event.listen(e1, \"before_execute\", canary.be1)\n",
      "        conn.execute(select([1]))\n",
      "\n",
      "        eq_(canary.be1.call_count, 1)\n",
      "\n",
      "        conn._branch().execute(select([1]))\n",
      "        eq_(canary.be1.call_count, 2)\n",
      "\n",
      "def test_force_conn_events_false(self):\n",
      "        canary = Mock()\n",
      "        e1 = create_engine(config.db_url)\n",
      "        assert not e1._has_events\n",
      "\n",
      "        event.listen(e1, \"before_execute\", canary.be1)\n",
      "\n",
      "        conn = e1._connection_cls(e1, connection=e1.raw_connection(),\n",
      "                            _has_events=False)\n",
      "\n",
      "        conn.execute(select([1]))\n",
      "\n",
      "        eq_(canary.be1.call_count, 0)\n",
      "\n",
      "        conn._branch().execute(select([1]))\n",
      "        eq_(canary.be1.call_count, 0)\n",
      "\n",
      "def test_cursor_events_ctx_execute_scalar(self):\n",
      "        canary = Mock()\n",
      "        e1 = testing_engine(config.db_url)\n",
      "\n",
      "        event.listen(e1, \"before_cursor_execute\", canary.bce)\n",
      "        event.listen(e1, \"after_cursor_execute\", canary.ace)\n",
      "\n",
      "        stmt = str(select([1]).compile(dialect=e1.dialect))\n",
      "\n",
      "        with e1.connect() as conn:\n",
      "            dialect = conn.dialect\n",
      "\n",
      "            ctx = dialect.execution_ctx_cls._init_statement(\n",
      "                            dialect, conn, conn.connection, stmt, {})\n",
      "\n",
      "            ctx._execute_scalar(stmt, Integer())\n",
      "\n",
      "        eq_(canary.bce.mock_calls,\n",
      "                [call(conn, ctx.cursor, stmt, ctx.parameters[0], ctx, False)])\n",
      "        eq_(canary.ace.mock_calls,\n",
      "                [call(conn, ctx.cursor, stmt, ctx.parameters[0], ctx, False)])\n",
      "\n",
      "def test_cursor_events_execute(self):\n",
      "        canary = Mock()\n",
      "        e1 = testing_engine(config.db_url)\n",
      "\n",
      "        event.listen(e1, \"before_cursor_execute\", canary.bce)\n",
      "        event.listen(e1, \"after_cursor_execute\", canary.ace)\n",
      "\n",
      "        stmt = str(select([1]).compile(dialect=e1.dialect))\n",
      "\n",
      "        with e1.connect() as conn:\n",
      "\n",
      "            result = conn.execute(stmt)\n",
      "\n",
      "        ctx = result.context\n",
      "        eq_(canary.bce.mock_calls,\n",
      "                [call(conn, ctx.cursor, stmt, ctx.parameters[0], ctx, False)])\n",
      "        eq_(canary.ace.mock_calls,\n",
      "                [call(conn, ctx.cursor, stmt, ctx.parameters[0], ctx, False)])\n",
      "\n",
      "def before_execute(conn, clauseelement, multiparams, params):\n",
      "            assert isinstance(multiparams, (list, tuple))\n",
      "            assert isinstance(params, dict)\n",
      "\n",
      "def after_execute(conn, clauseelement, multiparams, params, result):\n",
      "            assert isinstance(multiparams, (list, tuple))\n",
      "            assert isinstance(params, dict)\n",
      "\n",
      "def execute(conn, clauseelement, multiparams,\n",
      "                                                    params ):\n",
      "            stmts.append((str(clauseelement), params, multiparams))\n",
      "\n",
      "def cursor_execute(conn, cursor, statement, parameters,\n",
      "                                context, executemany):\n",
      "            cursor_stmts.append((str(statement), parameters, None))\n",
      "\n",
      "def execute(conn, *args, **kw):\n",
      "            canary.append('execute')\n",
      "\n",
      "def cursor_execute(conn, *args, **kw):\n",
      "            canary.append('cursor_execute')\n",
      "\n",
      "def go(conn, *args, **kw):\n",
      "                canary.append(name)\n",
      "\n",
      "def execute(conn, clauseelement, multiparams, params):\n",
      "            canary.append('execute')\n",
      "            return clauseelement, multiparams, params\n",
      "\n",
      "def cursor_execute(conn, cursor, statement,\n",
      "                        parameters, context, executemany):\n",
      "            canary.append('cursor_execute')\n",
      "            return statement, parameters\n",
      "\n",
      "def test_engine_connect(self):\n",
      "        engine = engines.testing_engine()\n",
      "\n",
      "        tracker = Mock()\n",
      "        event.listen(engine, \"engine_connect\", tracker)\n",
      "\n",
      "        c1 = engine.connect()\n",
      "        c2 = c1._branch()\n",
      "        c1.close()\n",
      "        eq_(\n",
      "            tracker.mock_calls,\n",
      "            [call(c1, False), call(c2, True)]\n",
      "        )\n",
      "\n",
      "def test_execution_options(self):\n",
      "        engine = engines.testing_engine()\n",
      "\n",
      "        engine_tracker = Mock()\n",
      "        conn_tracker = Mock()\n",
      "\n",
      "        event.listen(engine, \"set_engine_execution_options\", engine_tracker)\n",
      "        event.listen(engine, \"set_connection_execution_options\", conn_tracker)\n",
      "\n",
      "        e2 = engine.execution_options(e1='opt_e1')\n",
      "        c1 = engine.connect()\n",
      "        c2 = c1.execution_options(c1='opt_c1')\n",
      "        c3 = e2.connect()\n",
      "        c4 = c3.execution_options(c3='opt_c3')\n",
      "        eq_(\n",
      "            engine_tracker.mock_calls,\n",
      "            [call(e2, {'e1': 'opt_e1'})]\n",
      "        )\n",
      "        eq_(\n",
      "            conn_tracker.mock_calls,\n",
      "            [call(c2, {\"c1\": \"opt_c1\"}), call(c4, {\"c3\": \"opt_c3\"})]\n",
      "        )\n",
      "\n",
      "def go(conn, cursor, statement, parameters, context, executemany):\n",
      "                canary.append((statement, context))\n",
      "\n",
      "def go(conn, *args, **kw):\n",
      "                canary.append(name)\n",
      "\n",
      "def go(*args, **kw):\n",
      "                canary1.append(name)\n",
      "\n",
      "def go(*args, **kw):\n",
      "                canary2.append(name)\n",
      "\n",
      "def tearDown(self):\n",
      "        Engine.dispatch._clear()\n",
      "        Engine._has_events = False\n",
      "\n",
      "def test_legacy_dbapi_error(self):\n",
      "        engine = engines.testing_engine()\n",
      "        canary = Mock()\n",
      "\n",
      "        event.listen(engine, \"dbapi_error\", canary)\n",
      "\n",
      "        with engine.connect() as conn:\n",
      "            try:\n",
      "                conn.execute(\"SELECT FOO FROM I_DONT_EXIST\")\n",
      "                assert False\n",
      "            except tsa.exc.DBAPIError as e:\n",
      "                eq_(canary.mock_calls[0][1][5], e.orig)\n",
      "                eq_(canary.mock_calls[0][1][2], \"SELECT FOO FROM I_DONT_EXIST\")\n",
      "\n",
      "def process_bind_param(self, value, dialect):\n",
      "                raise nope\n",
      "\n",
      "def test_legacy_dbapi_error_non_dbapi_error(self):\n",
      "        engine = engines.testing_engine()\n",
      "\n",
      "        listener = Mock(return_value=None)\n",
      "        event.listen(engine, 'dbapi_error', listener)\n",
      "\n",
      "        nope = TypeError(\"I'm not a DBAPI error\")\n",
      "        with engine.connect() as c:\n",
      "            c.connection.cursor = Mock(\n",
      "                    return_value=Mock(\n",
      "                        execute=Mock(\n",
      "                                side_effect=nope\n",
      "                        ))\n",
      "                    )\n",
      "\n",
      "            assert_raises_message(\n",
      "                TypeError,\n",
      "                \"I'm not a DBAPI error\",\n",
      "                c.execute, \"select \"\n",
      "            )\n",
      "        # no legacy event\n",
      "        eq_(listener.mock_calls, [])\n",
      "\n",
      "def test_handle_error(self):\n",
      "        engine = engines.testing_engine()\n",
      "        canary = Mock(return_value=None)\n",
      "\n",
      "        event.listen(engine, \"handle_error\", canary)\n",
      "\n",
      "        with engine.connect() as conn:\n",
      "            try:\n",
      "                conn.execute(\"SELECT FOO FROM I_DONT_EXIST\")\n",
      "                assert False\n",
      "            except tsa.exc.DBAPIError as e:\n",
      "                ctx = canary.mock_calls[0][1][0]\n",
      "\n",
      "                eq_(ctx.original_exception, e.orig)\n",
      "                is_(ctx.sqlalchemy_exception, e)\n",
      "                eq_(ctx.statement, \"SELECT FOO FROM I_DONT_EXIST\")\n",
      "\n",
      "def err(context):\n",
      "            stmt = context.statement\n",
      "            exception = context.original_exception\n",
      "            if \"ERROR ONE\" in str(stmt):\n",
      "                return MyException(\"my exception\")\n",
      "            elif \"ERROR TWO\" in str(stmt):\n",
      "                return exception\n",
      "            else:\n",
      "                return None\n",
      "\n",
      "def err1(context):\n",
      "            stmt = context.statement\n",
      "\n",
      "            if \"ERROR ONE\" in str(stmt) or \"ERROR TWO\" in str(stmt) \\\n",
      "                    or \"ERROR THREE\" in str(stmt):\n",
      "                return MyException1(\"my exception\")\n",
      "            elif \"ERROR FOUR\" in str(stmt):\n",
      "                raise MyException3(\"my exception short circuit\")\n",
      "\n",
      "def err2(context):\n",
      "            stmt = context.statement\n",
      "            if (\"ERROR ONE\" in str(stmt) or \"ERROR FOUR\" in str(stmt)) \\\n",
      "                    and isinstance(context.chained_exception, MyException1):\n",
      "                raise MyException2(\"my exception chained\")\n",
      "            elif \"ERROR TWO\" in str(stmt):\n",
      "                return context.chained_exception\n",
      "            else:\n",
      "                return None\n",
      "\n",
      "def process_bind_param(self, value, dialect):\n",
      "                raise nope\n",
      "\n",
      "def test_exception_event_non_dbapi_error(self):\n",
      "        \"\"\"test that dbapi_error is called with a context in\n",
      "        cases where DBAPI raises an exception that is not a DBAPI\n",
      "        exception, e.g. internal errors or encoding problems.\n",
      "\n",
      "        \"\"\"\n",
      "        engine = engines.testing_engine()\n",
      "\n",
      "        listener = Mock(return_value=None)\n",
      "        event.listen(engine, 'handle_error', listener)\n",
      "\n",
      "        nope = TypeError(\"I'm not a DBAPI error\")\n",
      "        with engine.connect() as c:\n",
      "            c.connection.cursor = Mock(\n",
      "                    return_value=Mock(\n",
      "                        execute=Mock(\n",
      "                                side_effect=nope\n",
      "                        ))\n",
      "                    )\n",
      "\n",
      "            assert_raises_message(\n",
      "                TypeError,\n",
      "                \"I'm not a DBAPI error\",\n",
      "                c.execute, \"select \"\n",
      "            )\n",
      "        ctx = listener.mock_calls[0][1][0]\n",
      "        eq_(ctx.statement, \"select \")\n",
      "        is_(ctx.is_disconnect, False)\n",
      "        is_(ctx.original_exception, nope)\n",
      "\n",
      "def evt(ctx):\n",
      "            ctx.is_disconnect = evt_value\n",
      "\n",
      "def test_alter_disconnect_to_true(self):\n",
      "        self._test_alter_disconnect(False, True)\n",
      "        self._test_alter_disconnect(True, True)\n",
      "\n",
      "def test_alter_disconnect_to_false(self):\n",
      "        self._test_alter_disconnect(True, False)\n",
      "        self._test_alter_disconnect(False, False)\n",
      "\n",
      "def execute(\n",
      "                self,\n",
      "                conn,\n",
      "                execute,\n",
      "                clauseelement,\n",
      "                *multiparams,\n",
      "                **params\n",
      "                ):\n",
      "                stmts.append((str(clauseelement), params, multiparams))\n",
      "                return execute(clauseelement, *multiparams, **params)\n",
      "\n",
      "def cursor_execute(\n",
      "                self,\n",
      "                execute,\n",
      "                cursor,\n",
      "                statement,\n",
      "                parameters,\n",
      "                context,\n",
      "                executemany,\n",
      "                ):\n",
      "                cursor_stmts.append((str(statement), parameters, None))\n",
      "                return execute(cursor, statement, parameters, context)\n",
      "\n",
      "def assert_stmts(expected, received):\n",
      "            for stmt, params, posn in expected:\n",
      "                if not received:\n",
      "                    assert False, \"Nothing available for stmt: %s\" % stmt\n",
      "                while received:\n",
      "                    teststmt, testparams, testmultiparams = \\\n",
      "                        received.pop(0)\n",
      "                    teststmt = re.compile(r'[\\n\\t ]+', re.M).sub(' ',\n",
      "                            teststmt).strip()\n",
      "                    if teststmt.startswith(stmt) and (testparams\n",
      "                            == params or testparams == posn):\n",
      "                        break\n",
      "\n",
      "def go(*arg, **kw):\n",
      "                    canary.append(fn.__name__)\n",
      "                    return fn(*arg, **kw)\n",
      "\n",
      "def go(*arg, **kw):\n",
      "                    canary.append(fn.__name__)\n",
      "                    return fn(*arg, **kw)\n",
      "\n",
      "def go(*arg, **kw):\n",
      "                    canary.append(fn.__name__)\n",
      "                    return fn(*arg, **kw)\n",
      "\n",
      "def mock_the_cursor(cursor, *arg):\n",
      "            arg[-1].get_result_proxy = Mock(return_value=Mock(context=arg[-1]))\n",
      "            return retval\n",
      "\n",
      "def _assert(self, retval, m1, m2, mock_calls):\n",
      "        eq_(m1.mock_calls, mock_calls)\n",
      "        if retval:\n",
      "            eq_(m2.mock_calls, [])\n",
      "        else:\n",
      "            eq_(m2.mock_calls, mock_calls)\n",
      "\n",
      "def _test_do_execute(self, retval):\n",
      "        with self._run_test(retval) as (conn, m1):\n",
      "            result = conn.execute(\"insert into table foo\", {\"foo\": \"bar\"})\n",
      "        self._assert(\n",
      "            retval,\n",
      "            m1.do_execute, m1.real_do_execute,\n",
      "            [call(\n",
      "                    result.context.cursor,\n",
      "                    \"insert into table foo\",\n",
      "                    {\"foo\": \"bar\"}, result.context)]\n",
      "        )\n",
      "\n",
      "def _test_do_executemany(self, retval):\n",
      "        with self._run_test(retval) as (conn, m1):\n",
      "            result = conn.execute(\"insert into table foo\",\n",
      "                            [{\"foo\": \"bar\"}, {\"foo\": \"bar\"}])\n",
      "        self._assert(\n",
      "            retval,\n",
      "            m1.do_executemany, m1.real_do_executemany,\n",
      "            [call(\n",
      "                    result.context.cursor,\n",
      "                    \"insert into table foo\",\n",
      "                    [{\"foo\": \"bar\"}, {\"foo\": \"bar\"}], result.context)]\n",
      "        )\n",
      "\n",
      "def _test_do_execute_no_params(self, retval):\n",
      "        with self._run_test(retval) as (conn, m1):\n",
      "            result = conn.execution_options(no_parameters=True).\\\n",
      "                execute(\"insert into table foo\")\n",
      "        self._assert(\n",
      "            retval,\n",
      "            m1.do_execute_no_params, m1.real_do_execute_no_params,\n",
      "            [call(\n",
      "                    result.context.cursor,\n",
      "                    \"insert into table foo\", result.context)]\n",
      "        )\n",
      "\n",
      "def _test_cursor_execute(self, retval):\n",
      "        with self._run_test(retval) as (conn, m1):\n",
      "            dialect = conn.dialect\n",
      "\n",
      "            stmt = \"insert into table foo\"\n",
      "            params = {\"foo\": \"bar\"}\n",
      "            ctx = dialect.execution_ctx_cls._init_statement(\n",
      "                            dialect, conn, conn.connection, stmt, [params])\n",
      "\n",
      "            conn._cursor_execute(ctx.cursor, stmt, params, ctx)\n",
      "\n",
      "        self._assert(\n",
      "            retval,\n",
      "            m1.do_execute, m1.real_do_execute,\n",
      "            [call(\n",
      "                    ctx.cursor,\n",
      "                    \"insert into table foo\",\n",
      "                    {\"foo\": \"bar\"}, ctx)]\n",
      "        )\n",
      "\n",
      "def test_do_execute_w_replace(self):\n",
      "        self._test_do_execute(True)\n",
      "\n",
      "def test_do_execute_wo_replace(self):\n",
      "        self._test_do_execute(False)\n",
      "\n",
      "def test_do_executemany_w_replace(self):\n",
      "        self._test_do_executemany(True)\n",
      "\n",
      "def test_do_executemany_wo_replace(self):\n",
      "        self._test_do_executemany(False)\n",
      "\n",
      "def test_do_execute_no_params_w_replace(self):\n",
      "        self._test_do_execute_no_params(True)\n",
      "\n",
      "def test_do_execute_no_params_wo_replace(self):\n",
      "        self._test_do_execute_no_params(False)\n",
      "\n",
      "def test_cursor_execute_w_replace(self):\n",
      "        self._test_cursor_execute(True)\n",
      "\n",
      "def t_ID(self, t):\n",
      "        r'\\d+\\.([uU]|[lL]|[uU][lL]|[lL][uU])?'\n",
      "        t.value = int(t.value[:-1])\n",
      "        return t\n",
      "\n",
      "def t_DONE(self, t):\n",
      "        r'(\\(x\\))'\n",
      "        return t\n",
      "\n",
      "def t_TASK(self, t):\n",
      "        r'((?!\\(x\\))).+'\n",
      "        return t\n",
      "\n",
      "def t_newline(self, t):\n",
      "        r'\\n+'\n",
      "        t.lexer.lineno += len(t.value)\n",
      "\n",
      "def t_error(self, t):\n",
      "        raise SyntaxError(\n",
      "            \"Illegal character: '%s' at Line %d\" % (t.value[0], t.lineno)\n",
      "        )\n",
      "\n",
      "def __init__(self):\n",
      "        self.lexer = lex.lex(module=self)\n",
      "\n",
      "def p_error(self, p):\n",
      "        if p:\n",
      "            raise SyntaxError(\n",
      "                \"Character '%s' at line %d\" % (p.value[0], p.lineno)\n",
      "            )\n",
      "        else:\n",
      "            raise SyntaxError(\"SyntaxError at EOF\")\n",
      "\n",
      "def p_start(self, p):\n",
      "        \"start : translation_unit\"\n",
      "        p[0] = self.todo\n",
      "\n",
      "def p_translation_unit(self, p):\n",
      "        \"\"\"\n",
      "        translation_unit : translate_task\n",
      "                         | translation_unit translate_task\n",
      "                         |\n",
      "        \"\"\"\n",
      "        pass\n",
      "\n",
      "def p_translation_task(self, p):\n",
      "        \"\"\"\n",
      "        translate_task : ID DONE TASK\n",
      "                       | ID TASK\n",
      "        \"\"\"\n",
      "        if len(p) == 4:\n",
      "            done = True\n",
      "            content = p[3]\n",
      "        elif len(p) == 3:\n",
      "            done = False\n",
      "            content = p[2]\n",
      "        task = Task(p[1], content, done)\n",
      "        self.todo.append(task)\n",
      "\n",
      "def __init__(self):\n",
      "        self.parser = yacc.yacc(module=self, debug=0, write_tables=0)\n",
      "\n",
      "def parse(self, data):\n",
      "        # reset list\n",
      "        self.todo = Todo()\n",
      "        return self.parser.parse(data)\n",
      "\n",
      "def setup(self, memcached, memcached_cluster):\n",
      "        self.storage_url = \"memcached://localhost:22122\"\n",
      "\n",
      "def test_init_options(self, mocker):\n",
      "        constructor = mocker.spy(pymemcache.client, \"PooledClient\")\n",
      "        assert storage_from_string(self.storage_url, connect_timeout=1).check()\n",
      "        assert constructor.call_args[1][\"connect_timeout\"] == 1\n",
      "\n",
      "def test_fixed_window(self):\n",
      "        storage = MemcachedStorage(\"memcached://localhost:22122\")\n",
      "        limiter = FixedWindowRateLimiter(storage)\n",
      "        per_min = RateLimitItemPerSecond(10)\n",
      "        start = time.time()\n",
      "        count = 0\n",
      "\n",
      "        while time.time() - start < 0.5 and count < 10:\n",
      "            assert limiter.hit(per_min)\n",
      "            count += 1\n",
      "        assert not limiter.hit(per_min)\n",
      "\n",
      "        while time.time() - start <= 1:\n",
      "            time.sleep(0.1)\n",
      "        assert limiter.hit(per_min)\n",
      "\n",
      "def test_fixed_window_cluster(self):\n",
      "        storage = MemcachedStorage(\"memcached://localhost:22122,localhost:22123\")\n",
      "        limiter = FixedWindowRateLimiter(storage)\n",
      "        per_min = RateLimitItemPerSecond(10)\n",
      "        start = time.time()\n",
      "        count = 0\n",
      "\n",
      "        while time.time() - start < 0.5 and count < 10:\n",
      "            assert limiter.hit(per_min)\n",
      "            count += 1\n",
      "        assert not limiter.hit(per_min)\n",
      "\n",
      "        while time.time() - start <= 1:\n",
      "            time.sleep(0.1)\n",
      "        assert limiter.hit(per_min)\n",
      "\n",
      "def test_fixed_window_with_elastic_expiry(self):\n",
      "        storage = MemcachedStorage(\"memcached://localhost:22122\")\n",
      "        limiter = FixedWindowElasticExpiryRateLimiter(storage)\n",
      "        per_sec = RateLimitItemPerSecond(2, 2)\n",
      "\n",
      "        assert limiter.hit(per_sec)\n",
      "        time.sleep(1)\n",
      "        assert limiter.hit(per_sec)\n",
      "        assert not limiter.test(per_sec)\n",
      "        time.sleep(1)\n",
      "        assert not limiter.test(per_sec)\n",
      "        time.sleep(1)\n",
      "        assert limiter.test(per_sec)\n",
      "\n",
      "def test_fixed_window_with_elastic_expiry_cluster(self):\n",
      "        storage = MemcachedStorage(\"memcached://localhost:22122,localhost:22123\")\n",
      "        limiter = FixedWindowElasticExpiryRateLimiter(storage)\n",
      "        per_sec = RateLimitItemPerSecond(2, 2)\n",
      "\n",
      "        assert limiter.hit(per_sec)\n",
      "        time.sleep(1)\n",
      "        assert limiter.hit(per_sec)\n",
      "        assert not limiter.test(per_sec)\n",
      "        time.sleep(1)\n",
      "        assert not limiter.test(per_sec)\n",
      "        time.sleep(1)\n",
      "        assert limiter.test(per_sec)\n",
      "\n",
      "def _get_data_dir(self, db_version):\n",
      "        # Try to get from svc first\n",
      "        output = run('svcprop -p config/data postgresql')\n",
      "        if output.stdout and exists(output.stdout, use_sudo=True):\n",
      "            return output.stdout\n",
      "        return base_postgres.PostgresInstall._get_data_dir(self, db_version)\n",
      "\n",
      "def _install_package(self, db_version):\n",
      "        sudo(\"pkg_add postgresql%s-server\" %db_version)\n",
      "        sudo(\"pkg_add postgresql%s-replicationtools\" %db_version)\n",
      "        sudo(\"svcadm enable postgresql\")\n",
      "\n",
      "def _restart_db_server(self, db_version):\n",
      "        sudo('svcadm restart postgresql')\n",
      "\n",
      "def _stop_db_server(self, db_version):\n",
      "        sudo('svcadm disable postgresql')\n",
      "\n",
      "def _start_db_server(self, db_version):\n",
      "        sudo('svcadm enable postgresql')\n",
      "\n",
      "def install_package(self):\n",
      "        sudo('pkg_add libevent')\n",
      "        with cd('/tmp'):\n",
      "            run('wget %s' %self.pgbouncer_src)\n",
      "            sudo('pkg_add %s' %self.pkg_name)\n",
      "\n",
      "def _setup_parameter(self, file_name, **kwargs):\n",
      "        for key, value in kwargs.items():\n",
      "            origin = \"%s =\" %key\n",
      "            new = \"%s = %s\" %(key, value)\n",
      "            sudo('sed -i \"/%s/ c\\%s\" %s' %(origin, new, file_name))\n",
      "\n",
      "def _get_passwd(self, username):\n",
      "        with hide('output'):\n",
      "            string = run('echo \"select usename, passwd from pg_shadow where '\n",
      "                         'usename=\\'%s\\' order by 1\" | sudo su postgres -c '\n",
      "                         '\"psql\"' %username)\n",
      "\n",
      "        user, passwd = string.split('\\n')[2].split('|')\n",
      "        user = user.strip()\n",
      "        passwd = passwd.strip()\n",
      "\n",
      "        __, tmp_name = tempfile.mkstemp()\n",
      "        fn = open(tmp_name, 'w')\n",
      "        fn.write('\"%s\" \"%s\" \"\"\\n' %(user, passwd))\n",
      "        fn.close()\n",
      "        put(tmp_name, '%s/pgbouncer.userlist'%self.config_dir, use_sudo=True)\n",
      "        local('rm %s' %tmp_name)\n",
      "\n",
      "def _get_username(self, section=None):\n",
      "        try:\n",
      "            names = env.config_object.get_list(section, env.config_object.USERNAME)\n",
      "            username = names[0]\n",
      "        except:\n",
      "            print ('You must first set up a database server on this machine, '\n",
      "                   'and create a database user')\n",
      "            raise\n",
      "        return username\n",
      "\n",
      "def run(self, section=None):\n",
      "        \"\"\"\n",
      "        \"\"\"\n",
      "\n",
      "        sudo('mkdir -p /opt/pkg/bin')\n",
      "        sudo(\"ln -sf /opt/local/bin/awk /opt/pkg/bin/nawk\")\n",
      "        sudo(\"ln -sf /opt/local/bin/sed /opt/pkg/bin/nbsed\")\n",
      "\n",
      "        self.install_package()\n",
      "\n",
      "        svc_method = os.path.join(env.configs_dir, 'pgbouncer.xml')\n",
      "        put(svc_method, self.config_dir, use_sudo=True)\n",
      "\n",
      "        home = run('bash -c \"echo ~postgres\"')\n",
      "        bounce_home = os.path.join(home, 'pgbouncer')\n",
      "\n",
      "        pidfile = os.path.join(bounce_home, 'pgbouncer.pid')\n",
      "        self._setup_parameter('%s/pgbouncer.ini' %self.config_dir,\n",
      "                              pidfile=pidfile, **self.config)\n",
      "\n",
      "        if not section:\n",
      "            section = 'db-server'\n",
      "        username = self._get_username(section)\n",
      "        self._get_passwd(username)\n",
      "        # postgres should be the owner of these config files\n",
      "        sudo('chown -R postgres:postgres %s' %self.config_dir)\n",
      "\n",
      "        sudo('mkdir -p %s' % bounce_home)\n",
      "        sudo('chown postgres:postgres %s' % bounce_home)\n",
      "\n",
      "        sudo('mkdir -p /var/log/pgbouncer')\n",
      "        sudo('chown postgres:postgres /var/log/pgbouncer')\n",
      "\n",
      "        # set up log\n",
      "        sudo('logadm -C 3 -p1d -c -w /var/log/pgbouncer/pgbouncer.log -z 1')\n",
      "        run('svccfg import %s/pgbouncer.xml' %self.config_dir)\n",
      "\n",
      "        # start pgbouncer\n",
      "        sudo('svcadm enable pgbouncer')\n",
      "\n",
      "def parse_dsn(dsn_string):\n",
      "    \"\"\"Parse a connection string and return the associated driver\"\"\"\n",
      "    dsn = urlparse(dsn_string)\n",
      "    scheme = dsn.scheme.split('+')[0]\n",
      "    username = password = host = port = None\n",
      "    host = dsn.netloc\n",
      "    if '@' in host:\n",
      "        username, host = host.split('@')\n",
      "        if ':' in username:\n",
      "            username, password = username.split(':')\n",
      "            password = unquote(password)\n",
      "        username = unquote(username)\n",
      "    if ':' in host:\n",
      "        host, port = host.split(':')\n",
      "        port = int(port)\n",
      "    database = dsn.path.split('?')[0][1:]\n",
      "    query = dsn.path.split('?')[1] if '?' in dsn.path else dsn.query\n",
      "    kwargs = dict(parse_qsl(query, True))\n",
      "    if scheme == 'sqlite':\n",
      "        return SQLiteDriver, [dsn.path], {}\n",
      "    elif scheme == 'mysql':\n",
      "        kwargs['user'] = username or 'root'\n",
      "        kwargs['db'] = database\n",
      "        if port:\n",
      "            kwargs['port'] = port\n",
      "        if host:\n",
      "            kwargs['host'] = host\n",
      "        if password:\n",
      "            kwargs['passwd'] = password\n",
      "        return MySQLDriver, [], kwargs\n",
      "    elif scheme == 'postgresql':\n",
      "        kwargs['user'] = username or 'postgres'\n",
      "        kwargs['database'] = database\n",
      "        if port:\n",
      "            kwargs['port'] = port\n",
      "        if 'unix_socket' in kwargs:\n",
      "            kwargs['host'] = kwargs.pop('unix_socket')\n",
      "        elif host:\n",
      "            kwargs['host'] = host\n",
      "        if password:\n",
      "            kwargs['password'] = password\n",
      "        return PostgreSQLDriver, [], kwargs\n",
      "    else:\n",
      "        raise ValueError('Unknown driver %s' % dsn_string)\n",
      "\n",
      "def test_deploy(self, cav, ue):\n",
      "        s3 = boto.connect_s3()\n",
      "        s3.create_bucket(\"laterpay-rubberjack-ebdeploy\")  # FIXME Remove hardcoded bucket name\n",
      "\n",
      "        with tempfile.NamedTemporaryFile() as tmp:\n",
      "            result = CliRunner().invoke(rubberjack, ['deploy', tmp.name], catch_exceptions=False)\n",
      "\n",
      "            self.assertEquals(result.exit_code, 0, result.output)\n",
      "\n",
      "def test_promote(self, ue, de):\n",
      "        de.return_value = {\n",
      "            'DescribeEnvironmentsResponse': {\n",
      "                'DescribeEnvironmentsResult': {\n",
      "                    'Environments': [\n",
      "                        {\n",
      "                            'EnvironmentName': 'laterpay-devnull-live',  # FIXME Remove hardcoded EnvName\n",
      "                            'VersionLabel': 'old',\n",
      "                        },\n",
      "                        {\n",
      "                            'EnvironmentName': 'laterpay-devnull-dev',  # FIXME Remove hardcoded EnvName\n",
      "                            'VersionLabel': 'new',\n",
      "                        },\n",
      "                    ],\n",
      "                },\n",
      "            },\n",
      "        }\n",
      "\n",
      "        CliRunner().invoke(rubberjack, ['promote'], catch_exceptions=False)\n",
      "\n",
      "def test_promoting_same_version(self, ue, de, se):\n",
      "        de.return_value = {\n",
      "            'DescribeEnvironmentsResponse': {\n",
      "                'DescribeEnvironmentsResult': {\n",
      "                    'Environments': [\n",
      "                        {\n",
      "                            'EnvironmentName': 'laterpay-devnull-live',  # FIXME Remove hardcoded EnvName\n",
      "                            'VersionLabel': 'same',\n",
      "                        },\n",
      "                        {\n",
      "                            'EnvironmentName': 'laterpay-devnull-dev',  # FIXME Remove hardcoded EnvName\n",
      "                            'VersionLabel': 'same',\n",
      "                        },\n",
      "                    ],\n",
      "                },\n",
      "            },\n",
      "        }\n",
      "\n",
      "        CliRunner().invoke(rubberjack, ['promote'], catch_exceptions=False)\n",
      "\n",
      "        self.assertTrue(se.called)\n",
      "\n",
      "def test_sigv4(self):\n",
      "        CliRunner().invoke(rubberjack, ['--sigv4-host', 'foo', 'deploy'], catch_exceptions=False)\n",
      "\n",
      "def test_deploy_to_custom_environment(self, ue, cav):\n",
      "        s3 = boto.connect_s3()\n",
      "        s3.create_bucket(\"laterpay-rubberjack-ebdeploy\")  # FIXME Remove hardcoded bucket name\n",
      "\n",
      "        with tempfile.NamedTemporaryFile() as tmp:\n",
      "            result = CliRunner().invoke(rubberjack, ['deploy', '--environment', 'wibble', tmp.name], catch_exceptions=False)\n",
      "\n",
      "            self.assertEquals(result.exit_code, 0, result.output)\n",
      "\n",
      "        self.assertEqual(cav.call_count, 1, \"create_application_version wasn't called, but it should\")\n",
      "        self.assertEqual(ue.call_count, 1, \"update_environment wasn't called, but it should\")\n",
      "\n",
      "def test_deploy_without_updating_the_environment(self, ue, cav):\n",
      "        s3 = boto.connect_s3()\n",
      "        s3.create_bucket(\"laterpay-rubberjack-ebdeploy\")  # FIXME Remove hardcoded bucket name\n",
      "\n",
      "        with tempfile.NamedTemporaryFile() as tmp:\n",
      "            result = CliRunner().invoke(rubberjack, ['deploy', '--no-update-environment', tmp.name], catch_exceptions=False)\n",
      "\n",
      "            self.assertEquals(result.exit_code, 0, result.output)\n",
      "\n",
      "        self.assertEqual(cav.call_count, 1, \"create_application_version wasn't called, but it should\")\n",
      "        self.assertEqual(ue.call_count, 0, \"update_environment was called, but it shouldn't\")\n",
      "\n",
      "def test_deploy_to_custom_bucket(self, ue, cav):\n",
      "        bucket_name = 'rbbrjck-test'\n",
      "        s3 = boto.connect_s3()\n",
      "        s3.create_bucket(bucket_name)\n",
      "\n",
      "        with tempfile.NamedTemporaryFile() as tmp:\n",
      "            result = CliRunner().invoke(rubberjack, ['--bucket', bucket_name, 'deploy', tmp.name], catch_exceptions=False)\n",
      "\n",
      "            self.assertEquals(result.exit_code, 0, result.output)\n",
      "\n",
      "        self.assertEqual(cav.call_count, 1, \"create_application_version wasn't called, but it should\")\n",
      "        self.assertEqual(ue.call_count, 1, \"update_environment wasn't called, but it should\")\n",
      "\n",
      "        _, cav_kwargs = cav.call_args\n",
      "        self.assertEqual(bucket_name, cav_kwargs['s3_bucket'])\n",
      "\n",
      "def tobytes(obj):\n",
      "        if isinstance(obj, str):\n",
      "            obj = obj.encode('UTF-8')\n",
      "        assert isinstance(obj, bytes)\n",
      "        return obj\n",
      "\n",
      "def totext(obj):\n",
      "        if isinstance(obj, bytes):\n",
      "            obj = str(obj, 'UTF-8')\n",
      "        assert isinstance(obj, str)\n",
      "        return obj\n",
      "\n",
      "def tobytes(obj):\n",
      "        if isinstance(obj, unicode):\n",
      "            obj = obj.encode('UTF-8')\n",
      "        assert isinstance(obj, str)\n",
      "        return obj\n",
      "\n",
      "def totext(obj):\n",
      "        if isinstance(obj, str):\n",
      "            obj = unicode(obj, 'UTF-8')\n",
      "        assert isinstance(obj, unicode)\n",
      "        return obj\n",
      "\n",
      "def oswritebytes(fd, obj):\n",
      "    os.write(fd, tobytes(obj))\n",
      "\n",
      "def StdCaptureFD(out=True, err=True, in_=True):\n",
      "    return capture.MultiCapture(out, err, in_, Capture=capture.FDCapture)\n",
      "\n",
      "def StdCapture(out=True, err=True, in_=True):\n",
      "    return capture.MultiCapture(out, err, in_, Capture=capture.SysCapture)\n",
      "\n",
      "def test_getmethod_default_no_fd(self, monkeypatch):\n",
      "        from _pytest.capture import pytest_addoption\n",
      "        from _pytest.config import Parser\n",
      "        parser = Parser()\n",
      "        pytest_addoption(parser)\n",
      "        default = parser._groups[0].options[0].default\n",
      "        assert default == \"fd\" if hasattr(os, \"dup\") else \"sys\"\n",
      "        parser = Parser()\n",
      "        monkeypatch.delattr(os, 'dup', raising=False)\n",
      "        pytest_addoption(parser)\n",
      "        assert parser._groups[0].options[0].default == \"sys\"\n",
      "\n",
      "def test_capturing_basic_api(self, method):\n",
      "        capouter = StdCaptureFD()\n",
      "        old = sys.stdout, sys.stderr, sys.stdin\n",
      "        try:\n",
      "            capman = CaptureManager(method)\n",
      "            capman.start_global_capturing()\n",
      "            outerr = capman.suspend_global_capture()\n",
      "            assert outerr == (\"\", \"\")\n",
      "            outerr = capman.suspend_global_capture()\n",
      "            assert outerr == (\"\", \"\")\n",
      "            print(\"hello\")\n",
      "            out, err = capman.suspend_global_capture()\n",
      "            if method == \"no\":\n",
      "                assert old == (sys.stdout, sys.stderr, sys.stdin)\n",
      "            else:\n",
      "                assert not out\n",
      "            capman.resume_global_capture()\n",
      "            print(\"hello\")\n",
      "            out, err = capman.suspend_global_capture()\n",
      "            if method != \"no\":\n",
      "                assert out == \"hello\\n\"\n",
      "            capman.stop_global_capturing()\n",
      "        finally:\n",
      "            capouter.stop_capturing()\n",
      "\n",
      "def test_init_capturing(self):\n",
      "        capouter = StdCaptureFD()\n",
      "        try:\n",
      "            capman = CaptureManager(\"fd\")\n",
      "            capman.start_global_capturing()\n",
      "            pytest.raises(AssertionError, \"capman.start_global_capturing()\")\n",
      "            capman.stop_global_capturing()\n",
      "        finally:\n",
      "            capouter.stop_capturing()\n",
      "\n",
      "def test_unicode():\n",
      "            import sys\n",
      "            print (sys.stdout)\n",
      "            print (%s)\n",
      "\n",
      "def test_unicode():\n",
      "            print ('b\\\\u00f6y')\n",
      "\n",
      "def test_collect_capturing(testdir):\n",
      "    p = testdir.makepyfile(\"\"\"\n",
      "        print (\"collect %s failure\" % 13)\n",
      "        import xyz42123\n",
      "    \"\"\")\n",
      "    result = testdir.runpytest(p)\n",
      "    result.stdout.fnmatch_lines([\n",
      "        \"*Captured stdout*\",\n",
      "        \"*collect 13 failure*\",\n",
      "    ])\n",
      "\n",
      "def setup_module(mod):\n",
      "                print (\"setup module\")\n",
      "\n",
      "def setup_function(function):\n",
      "                print (\"setup \" + function.__name__)\n",
      "\n",
      "def test_func1():\n",
      "                print (\"in func1\")\n",
      "                assert 0\n",
      "\n",
      "def test_func2():\n",
      "                print (\"in func2\")\n",
      "                assert 0\n",
      "\n",
      "def setup_module(func):\n",
      "                print (\"module-setup\")\n",
      "\n",
      "def setup_function(func):\n",
      "                print (\"function-setup\")\n",
      "\n",
      "def test_func():\n",
      "                print (\"in function\")\n",
      "                assert 0\n",
      "\n",
      "def teardown_function(func):\n",
      "                print (\"in teardown\")\n",
      "\n",
      "def test_func1():\n",
      "                print (\"in func1\")\n",
      "\n",
      "def test_func2():\n",
      "                print (\"in func2\")\n",
      "                assert 0\n",
      "\n",
      "def setup_function(function):\n",
      "                print (\"setup func1\")\n",
      "\n",
      "def teardown_function(function):\n",
      "                print (\"teardown func1\")\n",
      "                assert 0\n",
      "\n",
      "def test_func1():\n",
      "                print (\"in func1\")\n",
      "                pass\n",
      "\n",
      "def teardown_module(mod):\n",
      "                print (\"teardown module\")\n",
      "                assert 0\n",
      "\n",
      "def test_func():\n",
      "                pass\n",
      "\n",
      "def test_capturing():\n",
      "                print (42)\n",
      "                sys.stderr.write(str(23))\n",
      "\n",
      "def test_capturing_error():\n",
      "                print (1)\n",
      "                sys.stderr.write(str(2))\n",
      "                raise ValueError\n",
      "\n",
      "def test_logging():\n",
      "                import logging\n",
      "                import pytest\n",
      "                stream = capture.CaptureIO()\n",
      "                logging.basicConfig(stream=stream)\n",
      "                stream.close() # to free memory/release resources\n",
      "\n",
      "def setup_function(function):\n",
      "                logging.warn(\"hello1\")\n",
      "\n",
      "def test_logging():\n",
      "                logging.warn(\"hello2\")\n",
      "                assert 0\n",
      "\n",
      "def teardown_function(function):\n",
      "                logging.warn(\"hello3\")\n",
      "                assert 0\n",
      "\n",
      "def setup_module(function):\n",
      "                logging.warn(\"hello1\")\n",
      "\n",
      "def test_logging():\n",
      "                logging.warn(\"hello2\")\n",
      "                assert 0\n",
      "\n",
      "def teardown_module(function):\n",
      "                logging.warn(\"hello3\")\n",
      "                assert 0\n",
      "\n",
      "def test_conftestlogging_is_shown(self, testdir):\n",
      "        testdir.makeconftest(\"\"\"\n",
      "                import logging\n",
      "                logging.basicConfig()\n",
      "                logging.warn(\"hello435\")\n",
      "        \"\"\")\n",
      "        # make sure that logging is still captured in tests\n",
      "        result = testdir.runpytest_subprocess(\"-s\", \"-p\", \"no:capturelog\")\n",
      "        assert result.ret == EXIT_NOTESTSCOLLECTED\n",
      "        result.stderr.fnmatch_lines([\n",
      "            \"WARNING*hello435*\",\n",
      "        ])\n",
      "        assert 'operation on closed file' not in result.stderr.str()\n",
      "\n",
      "def test_hello():\n",
      "                import logging\n",
      "                logging.warn(\"hello433\")\n",
      "                assert 0\n",
      "\n",
      "def test_hello(capsys):\n",
      "                print (42)\n",
      "                out, err = capsys.readouterr()\n",
      "                assert out.startswith(\"42\")\n",
      "\n",
      "def test_one(capsys, capfd):\n",
      "                pass\n",
      "\n",
      "def test_two(capfd, capsys):\n",
      "                pass\n",
      "\n",
      "def test_one(capsys, request):\n",
      "                request.getfixturevalue(\"capfd\")\n",
      "\n",
      "def test_two(capfd, request):\n",
      "                request.getfixturevalue(\"capsys\")\n",
      "\n",
      "def test_one(capsys, capfdbinary):\n",
      "                pass\n",
      "\n",
      "def test_hello(cap%s):\n",
      "                print (\"xxx42xxx\")\n",
      "                assert 0\n",
      "\n",
      "def test_hello(capfd):\n",
      "                import os\n",
      "                os.write(1, \"42\".encode('ascii'))\n",
      "                out, err = capfd.readouterr()\n",
      "                assert out.startswith(\"42\")\n",
      "                capfd.close()\n",
      "\n",
      "def test_hello(capfdbinary):\n",
      "                import os\n",
      "                # some likely un-decodable bytes\n",
      "                os.write(1, b'\\\\xfe\\\\x98\\\\x20')\n",
      "                out, err = capfdbinary.readouterr()\n",
      "                assert out == b'\\\\xfe\\\\x98\\\\x20'\n",
      "                assert err == b''\n",
      "\n",
      "def test_hello(capsysbinary):\n",
      "                import sys\n",
      "                # some likely un-decodable bytes\n",
      "                sys.stdout.buffer.write(b'\\\\xfe\\\\x98\\\\x20')\n",
      "                out, err = capsysbinary.readouterr()\n",
      "                assert out == b'\\\\xfe\\\\x98\\\\x20'\n",
      "                assert err == b''\n",
      "\n",
      "def test_hello(capsysbinary):\n",
      "                pass\n",
      "\n",
      "def test_hello(capsys, missingarg):\n",
      "                pass\n",
      "\n",
      "def test_hello(capfd):\n",
      "                import os\n",
      "                os.write(1, str(42).encode('ascii'))\n",
      "                raise KeyboardInterrupt()\n",
      "\n",
      "def test_log(capsys):\n",
      "                logging.error('x')\n",
      "\n",
      "def test_disabled({fixture}):\n",
      "                print('captured before')\n",
      "                with {fixture}.disabled():\n",
      "                    print('while capture is disabled')\n",
      "                print('captured after')\n",
      "                assert {fixture}.readouterr() == ('captured before\\\\ncaptured after\\\\n', '')\n",
      "\n",
      "def test_normal():\n",
      "                print('test_normal executed')\n",
      "\n",
      "def captured_print({fixture}):\n",
      "                print('stdout contents begin')\n",
      "                print('stderr contents begin', file=sys.stderr)\n",
      "                out, err = {fixture}.readouterr()\n",
      "\n",
      "                yield out, err\n",
      "\n",
      "                print('stdout contents end')\n",
      "                print('stderr contents end', file=sys.stderr)\n",
      "                out, err = {fixture}.readouterr()\n",
      "                assert out == 'stdout contents end\\\\n'\n",
      "                assert err == 'stderr contents end\\\\n'\n",
      "\n",
      "def test_captured_print(captured_print):\n",
      "                out, err = captured_print\n",
      "                assert out == 'stdout contents begin\\\\n'\n",
      "                assert err == 'stderr contents begin\\\\n'\n",
      "\n",
      "def pytest_runtest_setup(item):\n",
      "            raise ValueError(42)\n",
      "\n",
      "def test_hello(capfd):\n",
      "            pass\n",
      "\n",
      "def pytest_runtest_setup():\n",
      "            print (\"hello19\")\n",
      "\n",
      "def test_func():\n",
      "            omg = bytearray([1,129,1])\n",
      "            os.write(1, omg)\n",
      "            assert 0\n",
      "\n",
      "def pytest_runtest_setup():\n",
      "            print (\"hello19\")\n",
      "\n",
      "def test_a():\n",
      "            import sys\n",
      "            import subprocess\n",
      "            subprocess.call([sys.executable, __file__])\n",
      "\n",
      "def test_foo():\n",
      "            import os;os.write(1, b'\\xc3')\n",
      "\n",
      "def bad_snap(self):\n",
      "            raise Exception('boom')\n",
      "\n",
      "def test_text(self):\n",
      "        f = capture.CaptureIO()\n",
      "        f.write(\"hello\")\n",
      "        s = f.getvalue()\n",
      "        assert s == \"hello\"\n",
      "        f.close()\n",
      "\n",
      "def test_unicode_and_str_mixture(self):\n",
      "        f = capture.CaptureIO()\n",
      "        if sys.version_info >= (3, 0):\n",
      "            f.write(\"\\u00f6\")\n",
      "            pytest.raises(TypeError, \"f.write(bytes('hello', 'UTF-8'))\")\n",
      "        else:\n",
      "            f.write(unicode(\"\\u00f6\", 'UTF-8'))\n",
      "            f.write(\"hello\")  # bytes\n",
      "            s = f.getvalue()\n",
      "            f.close()\n",
      "            assert isinstance(s, unicode)\n",
      "\n",
      "def test_write_bytes_to_buffer(self):\n",
      "        \"\"\"In python3, stdout / stderr are text io wrappers (exposing a buffer\n",
      "        property of the underlying bytestream).  See issue #1407\n",
      "        \"\"\"\n",
      "        f = capture.CaptureIO()\n",
      "        f.buffer.write(b'foo\\r\\n')\n",
      "        assert f.getvalue() == 'foo\\r\\n'\n",
      "\n",
      "def test_bytes_io():\n",
      "    f = py.io.BytesIO()\n",
      "    f.write(tobytes(\"hello\"))\n",
      "    pytest.raises(TypeError, \"f.write(totext('hello'))\")\n",
      "    s = f.getvalue()\n",
      "    assert s == tobytes(\"hello\")\n",
      "\n",
      "def test_dontreadfrominput():\n",
      "    from _pytest.capture import DontReadFromInput\n",
      "    f = DontReadFromInput()\n",
      "    assert not f.isatty()\n",
      "    pytest.raises(IOError, f.read)\n",
      "    pytest.raises(IOError, f.readlines)\n",
      "    pytest.raises(IOError, iter, f)\n",
      "    pytest.raises(UnsupportedOperation, f.fileno)\n",
      "    f.close()  # just for completeness\n",
      "\n",
      "def test_dontreadfrominput_buffer_python3():\n",
      "    from _pytest.capture import DontReadFromInput\n",
      "    f = DontReadFromInput()\n",
      "    fb = f.buffer\n",
      "    assert not fb.isatty()\n",
      "    pytest.raises(IOError, fb.read)\n",
      "    pytest.raises(IOError, fb.readlines)\n",
      "    pytest.raises(IOError, iter, fb)\n",
      "    pytest.raises(ValueError, fb.fileno)\n",
      "    f.close()  # just for completeness\n",
      "\n",
      "def test_dontreadfrominput_buffer_python2():\n",
      "    from _pytest.capture import DontReadFromInput\n",
      "    f = DontReadFromInput()\n",
      "    with pytest.raises(AttributeError):\n",
      "        f.buffer\n",
      "    f.close()  # just for completeness\n",
      "\n",
      "def tmpfile(testdir):\n",
      "    f = testdir.makepyfile(\"\").open('wb+')\n",
      "    yield f\n",
      "    if not f.closed:\n",
      "        f.close()\n",
      "\n",
      "def test_dupfile(tmpfile):\n",
      "    flist = []\n",
      "    for i in range(5):\n",
      "        nf = capture.safe_text_dupfile(tmpfile, \"wb\")\n",
      "        assert nf != tmpfile\n",
      "        assert nf.fileno() != tmpfile.fileno()\n",
      "        assert nf not in flist\n",
      "        print(i, end=\"\", file=nf)\n",
      "        flist.append(nf)\n",
      "\n",
      "    fname_open = flist[0].name\n",
      "    assert fname_open == repr(flist[0].buffer)\n",
      "\n",
      "    for i in range(5):\n",
      "        f = flist[i]\n",
      "        f.close()\n",
      "    fname_closed = flist[0].name\n",
      "    assert fname_closed == repr(flist[0].buffer)\n",
      "    assert fname_closed != fname_open\n",
      "    tmpfile.seek(0)\n",
      "    s = tmpfile.read()\n",
      "    assert \"01234\" in repr(s)\n",
      "    tmpfile.close()\n",
      "    assert fname_closed == repr(flist[0].buffer)\n",
      "\n",
      "def test_dupfile_on_bytesio():\n",
      "    io = py.io.BytesIO()\n",
      "    f = capture.safe_text_dupfile(io, \"wb\")\n",
      "    f.write(\"hello\")\n",
      "    assert io.getvalue() == b\"hello\"\n",
      "    assert 'BytesIO object' in f.name\n",
      "\n",
      "def test_dupfile_on_textio():\n",
      "    io = py.io.TextIO()\n",
      "    f = capture.safe_text_dupfile(io, \"wb\")\n",
      "    f.write(\"hello\")\n",
      "    assert io.getvalue() == \"hello\"\n",
      "    assert not hasattr(f, 'name')\n",
      "\n",
      "def lsof_check():\n",
      "    pid = os.getpid()\n",
      "    try:\n",
      "        out = py.process.cmdexec(\"lsof -p %d\" % pid)\n",
      "    except (py.process.cmdexec.Error, UnicodeDecodeError):\n",
      "        # about UnicodeDecodeError, see note on pytester\n",
      "        pytest.skip(\"could not run 'lsof'\")\n",
      "    yield\n",
      "    out2 = py.process.cmdexec(\"lsof -p %d\" % pid)\n",
      "    len1 = len([x for x in out.split(\"\\n\") if \"REG\" in x])\n",
      "    len2 = len([x for x in out2.split(\"\\n\") if \"REG\" in x])\n",
      "    assert len2 < len1 + 3, out2\n",
      "\n",
      "def test_simple(self, tmpfile):\n",
      "        fd = tmpfile.fileno()\n",
      "        cap = capture.FDCapture(fd)\n",
      "        data = tobytes(\"hello\")\n",
      "        os.write(fd, data)\n",
      "        s = cap.snap()\n",
      "        cap.done()\n",
      "        assert not s\n",
      "        cap = capture.FDCapture(fd)\n",
      "        cap.start()\n",
      "        os.write(fd, data)\n",
      "        s = cap.snap()\n",
      "        cap.done()\n",
      "        assert s == \"hello\"\n",
      "\n",
      "def test_simple_many(self, tmpfile):\n",
      "        for i in range(10):\n",
      "            self.test_simple(tmpfile)\n",
      "\n",
      "def test_simple_many_check_open_files(self, testdir):\n",
      "        with lsof_check():\n",
      "            with testdir.makepyfile(\"\").open('wb+') as tmpfile:\n",
      "                self.test_simple_many(tmpfile)\n",
      "\n",
      "def test_simple_fail_second_start(self, tmpfile):\n",
      "        fd = tmpfile.fileno()\n",
      "        cap = capture.FDCapture(fd)\n",
      "        cap.done()\n",
      "        pytest.raises(ValueError, cap.start)\n",
      "\n",
      "def test_stderr(self):\n",
      "        cap = capture.FDCapture(2)\n",
      "        cap.start()\n",
      "        print(\"hello\", file=sys.stderr)\n",
      "        s = cap.snap()\n",
      "        cap.done()\n",
      "        assert s == \"hello\\n\"\n",
      "\n",
      "def test_stdin(self, tmpfile):\n",
      "        cap = capture.FDCapture(0)\n",
      "        cap.start()\n",
      "        x = os.read(0, 100).strip()\n",
      "        cap.done()\n",
      "        assert x == tobytes('')\n",
      "\n",
      "def test_writeorg(self, tmpfile):\n",
      "        data1, data2 = tobytes(\"foo\"), tobytes(\"bar\")\n",
      "        cap = capture.FDCapture(tmpfile.fileno())\n",
      "        cap.start()\n",
      "        tmpfile.write(data1)\n",
      "        tmpfile.flush()\n",
      "        cap.writeorg(data2)\n",
      "        scap = cap.snap()\n",
      "        cap.done()\n",
      "        assert scap == totext(data1)\n",
      "        with open(tmpfile.name, 'rb') as stmp_file:\n",
      "            stmp = stmp_file.read()\n",
      "            assert stmp == data2\n",
      "\n",
      "def test_simple_resume_suspend(self, tmpfile):\n",
      "        with saved_fd(1):\n",
      "            cap = capture.FDCapture(1)\n",
      "            cap.start()\n",
      "            data = tobytes(\"hello\")\n",
      "            os.write(1, data)\n",
      "            sys.stdout.write(\"whatever\")\n",
      "            s = cap.snap()\n",
      "            assert s == \"hellowhatever\"\n",
      "            cap.suspend()\n",
      "            os.write(1, tobytes(\"world\"))\n",
      "            sys.stdout.write(\"qlwkej\")\n",
      "            assert not cap.snap()\n",
      "            cap.resume()\n",
      "            os.write(1, tobytes(\"but now\"))\n",
      "            sys.stdout.write(\" yes\\n\")\n",
      "            s = cap.snap()\n",
      "            assert s == \"but now yes\\n\"\n",
      "            cap.suspend()\n",
      "            cap.done()\n",
      "            pytest.raises(AttributeError, cap.suspend)\n",
      "\n",
      "def saved_fd(fd):\n",
      "    new_fd = os.dup(fd)\n",
      "    try:\n",
      "        yield\n",
      "    finally:\n",
      "        os.dup2(new_fd, fd)\n",
      "        os.close(new_fd)\n",
      "\n",
      "def getcapture(self, **kw):\n",
      "        cap = self.__class__.captureclass(**kw)\n",
      "        cap.start_capturing()\n",
      "        try:\n",
      "            yield cap\n",
      "        finally:\n",
      "            cap.stop_capturing()\n",
      "\n",
      "def test_capturing_done_simple(self):\n",
      "        with self.getcapture() as cap:\n",
      "            sys.stdout.write(\"hello\")\n",
      "            sys.stderr.write(\"world\")\n",
      "            out, err = cap.readouterr()\n",
      "        assert out == \"hello\"\n",
      "        assert err == \"world\"\n",
      "\n",
      "def test_capturing_reset_simple(self):\n",
      "        with self.getcapture() as cap:\n",
      "            print(\"hello world\")\n",
      "            sys.stderr.write(\"hello error\\n\")\n",
      "            out, err = cap.readouterr()\n",
      "        assert out == \"hello world\\n\"\n",
      "        assert err == \"hello error\\n\"\n",
      "\n",
      "def test_capturing_readouterr(self):\n",
      "        with self.getcapture() as cap:\n",
      "            print(\"hello world\")\n",
      "            sys.stderr.write(\"hello error\\n\")\n",
      "            out, err = cap.readouterr()\n",
      "            assert out == \"hello world\\n\"\n",
      "            assert err == \"hello error\\n\"\n",
      "            sys.stderr.write(\"error2\")\n",
      "            out, err = cap.readouterr()\n",
      "        assert err == \"error2\"\n",
      "\n",
      "def test_capture_results_accessible_by_attribute(self):\n",
      "        with self.getcapture() as cap:\n",
      "            sys.stdout.write(\"hello\")\n",
      "            sys.stderr.write(\"world\")\n",
      "            capture_result = cap.readouterr()\n",
      "        assert capture_result.out == \"hello\"\n",
      "        assert capture_result.err == \"world\"\n",
      "\n",
      "def test_capturing_readouterr_unicode(self):\n",
      "        with self.getcapture() as cap:\n",
      "            print(\"hx\\xc4\\x85\\xc4\\x87\")\n",
      "            out, err = cap.readouterr()\n",
      "        assert out == py.builtin._totext(\"hx\\xc4\\x85\\xc4\\x87\\n\", \"utf8\")\n",
      "\n",
      "def test_capturing_readouterr_decode_error_handling(self):\n",
      "        with self.getcapture() as cap:\n",
      "            # triggered a internal error in pytest\n",
      "            print('\\xa6')\n",
      "            out, err = cap.readouterr()\n",
      "        assert out == py.builtin._totext('\\ufffd\\n', 'unicode-escape')\n",
      "\n",
      "def test_reset_twice_error(self):\n",
      "        with self.getcapture() as cap:\n",
      "            print(\"hello\")\n",
      "            out, err = cap.readouterr()\n",
      "        pytest.raises(ValueError, cap.stop_capturing)\n",
      "        assert out == \"hello\\n\"\n",
      "        assert not err\n",
      "\n",
      "def test_capturing_modify_sysouterr_in_between(self):\n",
      "        oldout = sys.stdout\n",
      "        olderr = sys.stderr\n",
      "        with self.getcapture() as cap:\n",
      "            sys.stdout.write(\"hello\")\n",
      "            sys.stderr.write(\"world\")\n",
      "            sys.stdout = capture.CaptureIO()\n",
      "            sys.stderr = capture.CaptureIO()\n",
      "            print(\"not seen\")\n",
      "            sys.stderr.write(\"not seen\\n\")\n",
      "            out, err = cap.readouterr()\n",
      "        assert out == \"hello\"\n",
      "        assert err == \"world\"\n",
      "        assert sys.stdout == oldout\n",
      "        assert sys.stderr == olderr\n",
      "\n",
      "def test_capturing_error_recursive(self):\n",
      "        with self.getcapture() as cap1:\n",
      "            print(\"cap1\")\n",
      "            with self.getcapture() as cap2:\n",
      "                print(\"cap2\")\n",
      "                out2, err2 = cap2.readouterr()\n",
      "                out1, err1 = cap1.readouterr()\n",
      "        assert out1 == \"cap1\\n\"\n",
      "        assert out2 == \"cap2\\n\"\n",
      "\n",
      "def test_just_out_capture(self):\n",
      "        with self.getcapture(out=True, err=False) as cap:\n",
      "            sys.stdout.write(\"hello\")\n",
      "            sys.stderr.write(\"world\")\n",
      "            out, err = cap.readouterr()\n",
      "        assert out == \"hello\"\n",
      "        assert not err\n",
      "\n",
      "def test_just_err_capture(self):\n",
      "        with self.getcapture(out=False, err=True) as cap:\n",
      "            sys.stdout.write(\"hello\")\n",
      "            sys.stderr.write(\"world\")\n",
      "            out, err = cap.readouterr()\n",
      "        assert err == \"world\"\n",
      "        assert not out\n",
      "\n",
      "def test_stdin_restored(self):\n",
      "        old = sys.stdin\n",
      "        with self.getcapture(in_=True):\n",
      "            newstdin = sys.stdin\n",
      "        assert newstdin != sys.stdin\n",
      "        assert sys.stdin is old\n",
      "\n",
      "def test_stdin_nulled_by_default(self):\n",
      "        print(\"XXX this test may well hang instead of crashing\")\n",
      "        print(\"XXX which indicates an error in the underlying capturing\")\n",
      "        print(\"XXX mechanisms\")\n",
      "        with self.getcapture():\n",
      "            pytest.raises(IOError, \"sys.stdin.read()\")\n",
      "\n",
      "def test_x():\n",
      "                os.write(1, \"hello\\\\n\".encode(\"ascii\"))\n",
      "                assert 0\n",
      "\n",
      "def test_intermingling(self):\n",
      "        with self.getcapture() as cap:\n",
      "            oswritebytes(1, \"1\")\n",
      "            sys.stdout.write(str(2))\n",
      "            sys.stdout.flush()\n",
      "            oswritebytes(1, \"3\")\n",
      "            oswritebytes(2, \"a\")\n",
      "            sys.stderr.write(\"b\")\n",
      "            sys.stderr.flush()\n",
      "            oswritebytes(2, \"c\")\n",
      "            out, err = cap.readouterr()\n",
      "        assert out == \"123\"\n",
      "        assert err == \"abc\"\n",
      "\n",
      "def test_many(self, capfd):\n",
      "        with lsof_check():\n",
      "            for i in range(10):\n",
      "                cap = StdCaptureFD()\n",
      "                cap.stop_capturing()\n",
      "\n",
      "def StdCaptureFD(out=True, err=True, in_=True):\n",
      "                return capture.MultiCapture(out, err, in_,\n",
      "                                              Capture=capture.FDCapture)\n",
      "\n",
      "def test_stdout():\n",
      "                os.close(1)\n",
      "                cap = StdCaptureFD(out=True, err=False, in_=False)\n",
      "                cap.stop_capturing()\n",
      "\n",
      "def test_stderr():\n",
      "                os.close(2)\n",
      "                cap = StdCaptureFD(out=False, err=True, in_=False)\n",
      "                cap.stop_capturing()\n",
      "\n",
      "def test_stdin():\n",
      "                os.close(0)\n",
      "                cap = StdCaptureFD(out=False, err=False, in_=True)\n",
      "                cap.stop_capturing()\n",
      "\n",
      "def test_capture_not_started_but_reset():\n",
      "    capsys = StdCapture()\n",
      "    capsys.stop_capturing()\n",
      "\n",
      "def test_using_capsys_fixture_works_with_sys_stdout_encoding(capsys):\n",
      "    test_text = 'test text'\n",
      "\n",
      "    print(test_text.encode(sys.stdout.encoding, 'replace'))\n",
      "    (out, err) = capsys.readouterr()\n",
      "    assert out\n",
      "    assert err == ''\n",
      "\n",
      "def test_capsys_results_accessible_by_attribute(capsys):\n",
      "    sys.stdout.write(\"spam\")\n",
      "    sys.stderr.write(\"eggs\")\n",
      "    capture_result = capsys.readouterr()\n",
      "    assert capture_result.out == \"spam\"\n",
      "    assert capture_result.err == \"eggs\"\n",
      "\n",
      "def test_fdcapture_tmpfile_remains_the_same(tmpfile, use):\n",
      "    if not use:\n",
      "        tmpfile = True\n",
      "    cap = StdCaptureFD(out=False, err=tmpfile)\n",
      "    try:\n",
      "        cap.start_capturing()\n",
      "        capfile = cap.err.tmpfile\n",
      "        cap.readouterr()\n",
      "    finally:\n",
      "        cap.stop_capturing()\n",
      "    capfile2 = cap.err.tmpfile\n",
      "    assert capfile2 == capfile\n",
      "\n",
      "def test_close():\n",
      "            os.close(1)\n",
      "\n",
      "def test_capture_again():\n",
      "            os.write(1, b\"hello\\\\n\")\n",
      "            assert 0\n",
      "\n",
      "def test_capturing_and_logging_fundamentals(testdir, method):\n",
      "    if method == \"StdCaptureFD\" and not hasattr(os, 'dup'):\n",
      "        pytest.skip(\"need os.dup\")\n",
      "    # here we check a fundamental feature\n",
      "    p = testdir.makepyfile(\"\"\"\n",
      "        import sys, os\n",
      "        import py, logging\n",
      "        from _pytest import capture\n",
      "        cap = capture.MultiCapture(out=False, in_=False,\n",
      "                                     Capture=capture.%s)\n",
      "        cap.start_capturing()\n",
      "\n",
      "        logging.warn(\"hello1\")\n",
      "        outerr = cap.readouterr()\n",
      "        print (\"suspend, captured %%s\" %%(outerr,))\n",
      "        logging.warn(\"hello2\")\n",
      "\n",
      "        cap.pop_outerr_to_orig()\n",
      "        logging.warn(\"hello3\")\n",
      "\n",
      "        outerr = cap.readouterr()\n",
      "        print (\"suspend2, captured %%s\" %% (outerr,))\n",
      "    \"\"\" % (method,))\n",
      "    result = testdir.runpython(p)\n",
      "    result.stdout.fnmatch_lines(\"\"\"\n",
      "        suspend, captured*hello1*\n",
      "        suspend2, captured*WARNING:root:hello3*\n",
      "    \"\"\")\n",
      "    result.stderr.fnmatch_lines(\"\"\"\n",
      "        WARNING:root:hello2\n",
      "    \"\"\")\n",
      "    assert \"atexit\" not in result.stderr.str()\n",
      "\n",
      "def test_capattr():\n",
      "            assert sys.stdout.errors == \"strict\"\n",
      "            assert sys.stderr.errors == \"strict\"\n",
      "\n",
      "def write(self, s):\n",
      "            pass\n",
      "\n",
      "def test_capattr():\n",
      "            # should not raise AttributeError\n",
      "            assert sys.stdout.encoding\n",
      "            assert sys.stderr.encoding\n",
      "\n",
      "def spam():\n",
      "            f = sys.stderr\n",
      "            while True:\n",
      "                print('.', end='', file=f)\n",
      "\n",
      "def test_silly():\n",
      "            t = threading.Thread(target=spam)\n",
      "            t.daemon = True\n",
      "            t.start()\n",
      "            time.sleep(0.5)\n",
      "\n",
      "def log(self, message):\n",
      "        f = open(settings.TASK_LOG_PATH, 'a')\n",
      "        now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)\n",
      "        log_message = \"%s\\t%s\\n\" % (now, message)\n",
      "        self.stdout.write(log_message)\n",
      "        f.write(log_message)\n",
      "        f.close()\n",
      "\n",
      "def handle(self, *args, **options):\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for function in functions:\n",
    "    print(function)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/michaelnath/.virtualenvs/224n_final_proj/lib/python3.9/site-packages/transformers/generation/utils.py:1273: UserWarning: Neither `max_length` nor `max_new_tokens` has been set, `max_length` will default to 20 (`generation_config.max_length`). Controlling `max_length` via the config is deprecated and `max_length` will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import RobertaTokenizer, T5ForConditionalGeneration\n",
    "\n",
    "tokenizer = RobertaTokenizer.from_pretrained(\"Salesforce/codet5-base\")\n",
    "\n",
    "model = T5ForConditionalGeneration.from_pretrained(\"michaelnath/c2c_model_with_chrf_and_nonzero_reps\")\n",
    "prompt = \"refine: def hello_world(): print('hello')\"\n",
    "\n",
    "tokenized_prompt = tokenizer(prompt, return_tensors=\"pt\")\n",
    "\n",
    "output = model.generate(**tokenized_prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<pad><s><extra_id_0>( self ) : return</s>\n"
     ]
    }
   ],
   "source": [
    "prompt = \"refine: def sort():\\n sort\"\n",
    "\n",
    "tokenized_prompt = tokenizer(prompt, return_tensors=\"pt\")\n",
    "\n",
    "output = model.generate(**tokenized_prompt)\n",
    "print(tokenizer.batch_decode(output)[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration michaelnath--annotated_github_dataset_2-06810c1c12facc21\n",
      "Found cached dataset parquet (/home/ubuntu/.cache/huggingface/datasets/michaelnath___parquet/michaelnath--annotated_github_dataset_2-06810c1c12facc21/0.0.0/2a3b91fbd88a2c90d1dbbb32b460cf621d31bd5b05b934492fdef7d8d6f236ec)\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "\n",
    "dataset = load_dataset(\"michaelnath/annotated_github_dataset_2\", split=\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10003\n"
     ]
    }
   ],
   "source": [
    "print(len(dataset))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}