03_tldr/01_GPT-2_with_abstracts.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "abstracts_huggingface_transformers.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "d-3CyESydUAq",
        "QJOiwV2ekLRw",
        "m_UNbotJkZAQ",
        "_f6LtvPAx4AD",
        "cYwrPLu4AKTX",
        "BLN2OQfTxD1n"
      ],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mTc5nORFYA-H"
      },
      "source": [
        "#Using the GPT-2 to Generate TL;DRs for the EconStore DataBase\n",
        "Notebook to generate TL;DRs of scientific papers using GTP-2. \n",
        "\n",
        "Data: Abstracts and fulltexts of over 7000 papers from econstor. This paper uses a subset of 193 climate change related papers.\n",
        "\n",
        "Sources for code and notebooks:\n",
        "  [Huggingface Transformers library](https://github.com/huggingface/transformers).\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d-3CyESydUAq"
      },
      "source": [
        "##Colab preparation & Data preparation\n",
        "\n",
        "\n",
        "1.   Mount Google Drive\n",
        "2.   Download the data\n",
        "1.   Extract the abstracts\n",
        "2.   Create training and validation datasets\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NIVMJ2ISYuJ1",
        "outputId": "799c0a2b-cfd3-48c6-d7f3-3cddeee74544"
      },
      "source": [
        "#1. Mount gDrive\n",
        "from google.colab import drive\n",
        "import os\n",
        "\n",
        "drive.mount('/content/gdrive')  # Mounting GoogleDrive to the content folder\n",
        "\n",
        "project_dir = 'NLP_scientific-text-generation'\n",
        "if not os.path.exists('/content/gdrive/MyDrive/'+project_dir):  # Create a project folder if it does not exist yet\n",
        "    os.makedirs('/content/gdrive/MyDrive/'+project_dir)\n",
        "os.chdir('/content/gdrive/MyDrive/'+project_dir)  # Changing the working directory to the project folder on GoogleDrive"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jlLbBqH3X_3J"
      },
      "source": [
        "#2. Get the data\n",
        "import urllib.request\n",
        "import tarfile\n",
        "\n",
        "def getData (dataset):\n",
        "    # Connect download stream as tar file object\n",
        "    url = \"https://www.econstor.eu/ki-hackathon/\" + dataset\n",
        "    ftpstream = urllib.request.urlopen(url)\n",
        "    tf = tarfile.open(fileobj=ftpstream, mode=\"r|gz\")\n",
        "    # Extract files (files will be extracted into a subfolder included in the zip file)\n",
        "    tf.extractall()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "21mr8DetY-V9"
      },
      "source": [
        "getData('econstor-cc-by-4.0-json.tgz')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QaM4rLk7Y_tE"
      },
      "source": [
        "getData('econstor-cc-by-4.0-txt.tgz')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M9tfnO6hNDSP",
        "outputId": "287b6f8d-b96c-4bfa-8dca-9111b33d86e5"
      },
      "source": [
        "#3. Data preparation\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import json\n",
        "import os\n",
        "\n",
        "###code for getting the handles, the journal name and the metadata\n",
        "#setting your personal path\n",
        "path = '/content/gdrive/MyDrive/NLP_scientific-text-generation/json'\n",
        "\n",
        "#initiating the lists for handles, journal name and metadata\n",
        "\n",
        "os.chdir(path)\n",
        "\n",
        "#metadata = []\n",
        "abstracts = []\n",
        "#keywords = []\n",
        "handles =[]\n",
        "journal_name = []\n",
        "##setting up the list of keywords to filter out the abstracts related to climate change\n",
        "\n",
        "#193 abstracts with this keyword list\n",
        "keyword_list_climate = ['Sustainability', 'sustainability','sustainable development','Sustainable development','Sustainable Development',\n",
        "                'globalization', 'Globalization',\n",
        "                'environment', 'Environment'\n",
        "                'climate change', 'Climate change', 'Climate Change',\n",
        "                'energy','Energy', 'sustainable development goals' ]\n",
        "\n",
        "#here we filter only the abstracts with keywords related to climate change\n",
        "for item in keyword_list_climate:\n",
        "    for i in os.listdir():\n",
        "\n",
        "        with open(i, \"r\", encoding='utf-8', errors='ignore') as f:\n",
        "            data = json.load(f)\n",
        "\n",
        "            handles.append(data['handle'])\n",
        "            journal_name.append(data['parentCollection']['name'])\n",
        "\n",
        "            for obj in data[\"metadata\"]:\n",
        "                if obj[\"key\"] == \"dc.subject.keyword\":\n",
        "                    if (obj['value'] == ''+ item):\n",
        "                        for ob in data['metadata']:\n",
        "                            if ob['key'] == 'dc.description.abstract':\n",
        "                                abstracts.append(ob[\"value\"])\n",
        "\n",
        "#put the list into a dataframe to print the abstracts with an index to count the number of abstracts\n",
        "df = pd.DataFrame(abstracts)\n",
        "df.columns = ['text'] #The column name should be 'text'\n",
        "\n",
        "print(df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                                                  text\n",
            "0    In this paper the contribution of technology m...\n",
            "1    Crop yield is influenced over time and space, ...\n",
            "2    Almost daily, news indicates that there are en...\n",
            "3    The paper examines the corporate social respon...\n",
            "4    Environmental assessment and pollution protect...\n",
            "..                                                 ...\n",
            "188  More and better quality private sector investm...\n",
            "189  Ensuring 'health for all' remains a persistent...\n",
            "190  Ensuring \"health for all\" remains a persistent...\n",
            "191  With the Sustainable Development Goals (SDGs),...\n",
            "192  The United Nation's Agenda 2030 and Sustainabl...\n",
            "\n",
            "[193 rows x 1 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xv7SGO90yubI"
      },
      "source": [
        "print(df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VxKfRLRotsdI"
      },
      "source": [
        "df = pd.DataFrame(abstracts)\r\n",
        "df.columns = ['text']\r\n",
        "df.to_csv('/content/test.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vqlggerbcfwV"
      },
      "source": [
        "#4. Create training, validation and test datasets\n",
        "#Train-, Validation-, & Testdata\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "import re\n",
        "\n",
        "train_valid_ratio = 0.8  #Proportion of training data\n",
        "data_train, data_valid = train_test_split(df, train_size = train_valid_ratio, random_state = 1)\n",
        "\n",
        "#Uncomment these lines to withheld a portion of the data testing\n",
        "#train_test_ratio = 0.9  #Proportion of training data\n",
        "#train_valid_ratio = 7/9 #Proportion of validation data\n",
        "#data_full_train, data_test = train_test_split(df, train_size = train_test_ratio, random_state = 1)\n",
        "#data_train, data_valid = train_test_split(data_full_train, train_size = train_valid_ratio, random_state = 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XKf-oTMNcmB3"
      },
      "source": [
        "#Create dataset\n",
        "#A BOS token is added before each abstract and a EOS token is added at the end of each abstract\n",
        "#as suggested here: \n",
        "def build_dataset(df, dest_path):\n",
        "    f = open(dest_path, 'w', encoding='utf-8')\n",
        "    data = ''\n",
        "    abstracts = df['abstract'].tolist()\n",
        "    for abstract in abstracts:\n",
        "        abstract = str(abstract).strip()\n",
        "        abstract = re.sub(r\"\\s\", \" \", abstract)\n",
        "        bos_token = '<BOS>'\n",
        "        eos_token = '<EOS>'\n",
        "        data += bos_token + ' ' + abstract + ' ' + eos_token + '\\n'\n",
        "\n",
        "        \n",
        "    f.write(data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "30tduazcc-VW"
      },
      "source": [
        "build_dataset(data_train, '/content/gdrive/My Drive/NLP_scientific-text-generation/train.txt')\n",
        "build_dataset(data_valid, '/content/gdrive/My Drive/NLP_scientific-text-generation/valid.txt')\n",
        "#build_dataset(df_test, '/content/gdrive/My Drive/NLP_scientific-text-generation/test.txt') #Uncomment if a test dataset was defined in the previous step."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G9vfT3YZu9dO"
      },
      "source": [
        "data_train.to_csv('/content/gdrive/My Drive/NLP_scientific-text-generation/train.csv')\r\n",
        "data_valid.to_csv('/content/gdrive/My Drive/NLP_scientific-text-generation/valid.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QJOiwV2ekLRw"
      },
      "source": [
        "##Get the huggingface transformers library"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mafkz0KCeKpJ"
      },
      "source": [
        "!pip install datasets\n",
        "#!pip install transformers\n",
        "os.chdir('/content/gdrive/My Drive/NLP_scientific-text-generation/')\n",
        "!pip install git+https://github.com/huggingface/transformers  #Clone the Huggingface Transformers Github"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ESLpPynEdjJO"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m_UNbotJkZAQ"
      },
      "source": [
        "## Train the model\n",
        "Different model sizes can be used by changing \"--model_name_or_path gpt2\"\n",
        "\n",
        "GTP-2 models from huggingface can be found [here](https://huggingface.co/transformers/pretrained_models.html). \n",
        "\n",
        "Pretrained \"standard\" GPT-2 models:\n",
        "*   gpt2 -> 12-layer, 768-hidden, 12-heads, 117M parameters\n",
        "*   gpt2-medium -> 24-layer, 1024-hidden, 16-heads, 345M parameters\n",
        "*   gpt2-large -> 36-layer, 1280-hidden, 20-heads, 774M parameters\n",
        "*   gpt2-xl -> 48-layer, 1600-hidden, 25-heads, 1558M parameters\n",
        "\n",
        "Distill GPT by huggingface:\n",
        "*   distilgpt2\n",
        "Model distilled from the GPT-2 (gpt2) model and checkpoints (6-layer, 768-hidden, 12-heads, 82M parameters).\n",
        "\n",
        "The large and xl version might not run on a colab because of RAM issues even when a batch size of one is used."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "IpQDR51pdhse",
        "outputId": "602c1b0e-d381-409d-c88c-cb892fc8e6bc"
      },
      "source": [
        "#Trains the model. Be careful with the batch_size when using larger models or datasets.\n",
        "%run \"/content/gdrive/My Drive/NLP_scientific-text-generation/transformers/examples/language-modeling/run_clm.py\" \\\n",
        "    --model_name_or_path gpt2 \\\n",
        "    --train_file \"/content/gdrive/My Drive/NLP_scientific-text-generation/train.csv\" \\\n",
        "    --validation_file \"/content/gdrive/My Drive/NLP_scientific-text-generation/valid.csv\" \\\n",
        "    --do_train \\\n",
        "    --do_eval \\\n",
        "    --output_dir \"/content/gdrive/My Drive/NLP_scientific-text-generation/output/\" \\\n",
        "    --per_device_train_batch_size=1 \\\n",
        "    --per_device_eval_batch_size=1 \\\n",
        "    --learning_rate 2e-5 \\\n",
        "    --num_train_epochs=15 \\\n",
        "    --overwrite_output_dir \\\n",
        "    --logging_steps 50 \\\n",
        "    --save_steps 1000"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[INFO|training_args.py:495] 2021-01-26 22:11:26,512 >> PyTorch: setting up devices\n",
            "01/26/2021 22:11:26 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: False\n",
            "01/26/2021 22:11:26 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/content/gdrive/My Drive/NLP_scientific-text-generation/output/, overwrite_output_dir=True, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=EvaluationStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=2e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=15.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_steps=0, logging_dir=runs/Jan26_22-11-26_65398e238d13, logging_first_step=False, logging_steps=50, save_steps=1000, save_total_limit=None, no_cuda=False, seed=42, fp16=False, fp16_opt_level=O1, fp16_backend=auto, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=False, dataloader_drop_last=False, eval_steps=50, dataloader_num_workers=0, past_index=-1, run_name=/content/gdrive/My Drive/NLP_scientific-text-generation/output/, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=False, deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, report_to=['tensorboard'], _n_gpu=1)\n",
            "Using custom data configuration default\n",
            "Reusing dataset csv (/root/.cache/huggingface/datasets/csv/default-ee6192f90d1cab93/0.0.0/2960f95a26e85d40ca41a230ac88787f715ee3003edaacb8b1f0891e9f04dda2)\n",
            "[INFO|configuration_utils.py:445] 2021-01-26 22:11:26,830 >> loading configuration file https://huggingface.co/gpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:481] 2021-01-26 22:11:26,831 >> Model config GPT2Config {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.3.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:445] 2021-01-26 22:11:26,850 >> loading configuration file https://huggingface.co/gpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:481] 2021-01-26 22:11:26,851 >> Model config GPT2Config {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.3.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|tokenization_utils_base.py:1766] 2021-01-26 22:11:26,924 >> loading file https://huggingface.co/gpt2/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/684fe667923972fb57f6b4dcb61a3c92763ad89882f3da5da9866baf14f2d60f.c7ed1f96aac49e745788faa77ba0a26a392643a50bb388b9c04ff469e555241f\n",
            "[INFO|tokenization_utils_base.py:1766] 2021-01-26 22:11:26,925 >> loading file https://huggingface.co/gpt2/resolve/main/merges.txt from cache at /root/.cache/huggingface/transformers/c0c761a63004025aeadd530c4c27b860ec4ecbe8a00531233de21d865a402598.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|tokenization_utils_base.py:1766] 2021-01-26 22:11:26,926 >> loading file https://huggingface.co/gpt2/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/16a2f78023c8dc511294f0c97b5e10fde3ef9889ad6d11ffaa2a00714e73926e.cf2d0ecb83b6df91b3dbb53f1d1e4c311578bfd3aa0e04934215a49bf9898df0\n",
            "[INFO|modeling_utils.py:1027] 2021-01-26 22:11:27,009 >> loading weights file https://huggingface.co/gpt2/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/752929ace039baa8ef70fe21cdf9ab9445773d20e733cf693d667982e210837e.323c769945a351daa25546176f8208b3004b6f563438a7603e7932bae9025925\n",
            "[INFO|modeling_utils.py:1143] 2021-01-26 22:11:31,410 >> All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
            "\n",
            "[INFO|modeling_utils.py:1152] 2021-01-26 22:11:31,411 >> All the weights of GPT2LMHeadModel were initialized from the model checkpoint at gpt2.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/csv/default-ee6192f90d1cab93/0.0.0/2960f95a26e85d40ca41a230ac88787f715ee3003edaacb8b1f0891e9f04dda2/cache-2d5bffe677cc8624.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/csv/default-ee6192f90d1cab93/0.0.0/2960f95a26e85d40ca41a230ac88787f715ee3003edaacb8b1f0891e9f04dda2/cache-f1d9c288eef794d3.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/csv/default-ee6192f90d1cab93/0.0.0/2960f95a26e85d40ca41a230ac88787f715ee3003edaacb8b1f0891e9f04dda2/cache-9fb0383c1988e4a0.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/csv/default-ee6192f90d1cab93/0.0.0/2960f95a26e85d40ca41a230ac88787f715ee3003edaacb8b1f0891e9f04dda2/cache-c7d7631c69908705.arrow\n",
            "[INFO|trainer.py:429] 2021-01-26 22:11:31,633 >> The following columns in the training set don't have a corresponding argument in `GPT2LMHeadModel.forward` and have been ignored: .\n",
            "[INFO|trainer.py:429] 2021-01-26 22:11:31,634 >> The following columns in the evaluation set don't have a corresponding argument in `GPT2LMHeadModel.forward` and have been ignored: .\n",
            "[INFO|trainer.py:800] 2021-01-26 22:11:31,645 >> ***** Running training *****\n",
            "[INFO|trainer.py:801] 2021-01-26 22:11:31,647 >>   Num examples = 31\n",
            "[INFO|trainer.py:802] 2021-01-26 22:11:31,648 >>   Num Epochs = 15\n",
            "[INFO|trainer.py:803] 2021-01-26 22:11:31,649 >>   Instantaneous batch size per device = 1\n",
            "[INFO|trainer.py:804] 2021-01-26 22:11:31,650 >>   Total train batch size (w. parallel, distributed & accumulation) = 1\n",
            "[INFO|trainer.py:805] 2021-01-26 22:11:31,652 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:806] 2021-01-26 22:11:31,653 >>   Total optimization steps = 465\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='465' max='465' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [465/465 03:13, Epoch 15/15]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>50</td>\n",
              "      <td>3.609200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>100</td>\n",
              "      <td>3.402100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>150</td>\n",
              "      <td>3.284800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>200</td>\n",
              "      <td>3.140000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>250</td>\n",
              "      <td>3.077300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>300</td>\n",
              "      <td>2.992000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>350</td>\n",
              "      <td>2.976700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>400</td>\n",
              "      <td>2.927900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>450</td>\n",
              "      <td>2.932800</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "[INFO|trainer.py:964] 2021-01-26 22:14:45,154 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "[INFO|trainer.py:1358] 2021-01-26 22:14:45,176 >> Saving model checkpoint to /content/gdrive/My Drive/NLP_scientific-text-generation/output/\n",
            "[INFO|configuration_utils.py:300] 2021-01-26 22:14:45,183 >> Configuration saved in /content/gdrive/My Drive/NLP_scientific-text-generation/output/config.json\n",
            "[INFO|modeling_utils.py:817] 2021-01-26 22:14:47,486 >> Model weights saved in /content/gdrive/My Drive/NLP_scientific-text-generation/output/pytorch_model.bin\n",
            "01/26/2021 22:14:47 - INFO - __main__ -   ***** Train results *****\n",
            "01/26/2021 22:14:47 - INFO - __main__ -     epoch = 15.0\n",
            "01/26/2021 22:14:47 - INFO - __main__ -     total_flos = 355519553863680\n",
            "01/26/2021 22:14:47 - INFO - __main__ -     train_runtime = 193.5011\n",
            "01/26/2021 22:14:47 - INFO - __main__ -     train_samples_per_second = 2.403\n",
            "01/26/2021 22:14:47 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:1550] 2021-01-26 22:14:47,619 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:1551] 2021-01-26 22:14:47,619 >>   Num examples = 8\n",
            "[INFO|trainer.py:1552] 2021-01-26 22:14:47,620 >>   Batch size = 1\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='8' max='8' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [8/8 00:00]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "01/26/2021 22:14:48 - INFO - __main__ -   ***** Eval results *****\n",
            "01/26/2021 22:14:48 - INFO - __main__ -     perplexity = 32.116767088050096\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_f6LtvPAx4AD"
      },
      "source": [
        "##Use the trained model to generate TL;DRs\n",
        "The trained model can now be used to generate TL;DRs of abstracts.\n",
        "\n",
        "Here we use one-shot and few-shot learning to generate TL;DRs for five abstracts. The TL;DRs of the abstracts used as examples in the one-shot and few-shot prompts can be generated either by hand or by using zero-shot learning (i.e. using only an example abstract as the prompt followed by TL;DR).\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KkGCLsdUNDSW"
      },
      "source": [
        "test_abstract_1 = \"Greenhouse gas emissions have significantly altered global climate, and will continue to do so in the future. Increases in the frequency, duration, and/or severity of drought and heat stress associated with climate change could fundamentally alter the composition, structure, and biogeography of forests in many regions. Of particular concern are potential increases in tree mortality associated with climate-induced physiological stress and interactions with other climate-mediated processes such as insect outbreaks and wildfire. Despite this risk, existing projections of tree mortality are based on models that lack functionally realistic mortality mechanisms, and there has been no attempt to track observations of climate-driven tree mortality globally. Here we present the first global assessment of recent tree mortality attributed to drought and heat stress. Although episodic mortality occurs in the absence of climate change, studies compiled here suggest that at least some of the world's forested ecosystems already may be responding to climate change and raise concern that forests may become increasingly vulnerable to higher background tree mortality rates and die-off in response to future warming and drought, even in environments that are not normally considered water-limited. This further suggests risks to ecosystem services, including the loss of sequestered forest carbon and associated atmospheric feedbacks. Our review also identifies key information gaps and scientific uncertainties that currently hinder our ability to predict tree mortality in response to climate change and emphasizes the need for a globally coordinated observation system. Overall, our review reveals the potential for amplified tree mortality due to drought and heat in forests worldwide.\"\n",
        "test_abstract_2 = \"The world's forests influence climate through physical, chemical, and biological processes that affect planetary energetics, the hydrologic cycle, and atmospheric composition. These complex and nonlinear forest-atmosphere interactions can dampen or amplify anthropogenic climate change. Tropical, temperate, and boreal reforestation and afforestation attenuate global warming through carbon sequestration. Biogeophysical feedbacks can enhance or diminish this negative climate forcing. Tropical forests mitigate warming through evaporative cooling, but the low albedo of boreal forests is a positive climate forcing. The evaporative effect of temperate forests is unclear. The net climate forcing from these and other processes is not known. Forests are under tremendous pressure from global change. Interdisciplinary science that integrates knowledge of the many interacting climate services of forests with the impacts of global change is necessary to identify and understand as yet unexplored feedbacks in the Earth system and the potential of forests to mitigate climate change.\"\n",
        "test_abstract_3 = \"The paper summarizes the current knowledge about the impact of livestock sector on climate change. The main sources of greenhouse gas (GHG) emissions from livestock are described and the contribution of livestock sector to the global GHG emissions is presented on the basis of the latest results obtained from the scientific research. The most recent mitigation strategies for reducing greenhouse gas emissions from livestock sector are also discussed. The paper aims to provide a general overview of an emergent environmental issue such as the impact of livestock sector on climate change. While the paper is easy to understand for non-expert readers, it may also be a relevant reference point for academic researchers and for policy makers aimed at achieving the sustainability of livestock/food sector.\"\n",
        "test_abstract_4 = \"Feeding a growing global population in a changing climate presents a significant challenge to society. The projected yields of crops under a range of agricultural and climatic scenarios are needed to assess food security prospects. Previous meta-analyses have summarized climate change impacts and adaptive potential as a function of temperature, but have not examined uncertainty, the timing of impacts, or the quantitative effectiveness of adaptation. Here we develop a new data set of more than 1,700 published simulations to evaluate yield impacts of climate change and adaptation. Without adaptation, losses in aggregate production are expected for wheat, rice and maize in both temperate and tropical regions by 2 °C of local warming. Crop-level adaptations increase simulated yields by an average of 7–15%, with adaptations more effective for wheat and rice than maize. Yield losses are greater in magnitude for the second half of the century than for the first. Consensus on yield decreases in the second half of the century is stronger in tropical than temperate regions, yet even moderate warming may reduce temperate crop yields in many locations. Although less is known about interannual variability than mean yields, the available data indicate that increases in yield variability are likely.\"\n",
        "test_abstract_5 = \"The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\"\n",
        "\n",
        "####################\n",
        "\n",
        "abstract_1 = \"Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\"\n",
        "tldr_1 = \"Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\"\n",
        "\n",
        "abstract_2 = \"Significantly more carbon is stored in the world's soils—including peatlands, wetlands and permafrost—than is present in the atmosphere. Disagreement exists, however, regarding the effects of climate change on global soil carbon stocks. If carbon stored belowground is transferred to the atmosphere by a warming-induced acceleration of its decomposition, a positive feedback to climate change would occur. Conversely, if increases of plant-derived carbon inputs to soils exceed increases in decomposition, the feedback would be negative. Despite much research, a consensus has not yet emerged on the temperature sensitivity of soil carbon decomposition. Unravelling the feedback effect is particularly difficult, because the diverse soil organic compounds exhibit a wide range of kinetic properties, which determine the intrinsic temperature sensitivity of their decomposition. Moreover, several environmental constraints obscure the intrinsic temperature sensitivity of substrate decomposition, causing lower observed ‘apparent’ temperature sensitivity, and these constraints may, themselves, be sensitive to climate.\"\n",
        "tldr_2 = \"Soil carbon decomposition may be sensitive to climate, but the amount of decomposition is constrained by other factors.\"\n",
        "\n",
        "abstract_3 = \"Climate change over the past ∼30 years has produced numerous shifts in the distributions and abundances of species and has been implicated in one species-level extinction. Using projections of species' distributions for future climate scenarios, we assess extinction risks for sample regions that cover some 20% of the Earth's terrestrial surface. Exploring three approaches in which the estimated probability of extinction shows a power-law relationship with geographical range size, we predict, on the basis of mid-range climate-warming scenarios for 2050, that 15–37% of species in our sample of regions and taxa will be ‘committed to extinction’. When the average of the three methods and two dispersal scenarios is taken, minimal climate-warming scenarios produce lower projections of species committed to extinction (∼18%) than mid-range (∼24%) and maximum-change (∼35%) scenarios. These estimates show the importance of rapid implementation of technologies to decrease greenhouse gas emissions and strategies for carbon sequestration.\"\n",
        "tldr_3 = \"Using predictions of future climate, the authors predict that if greenhouse gases continue to increase, 15-37% of species will be committed to extinction.\"\n",
        "\n",
        "####################\n",
        "# One-Shot Prompt\n",
        "os_prompt = \"Abstract:\\n\" + abstract_1 + \"\\nTL;DR:\\n\" + tldr_1 + \"\\n\\nAbstract:\\n\"\n",
        "\n",
        "# Few-Shot Prompt (2-Shot)\n",
        "fs_prompt = \"Abstract:\\n\" + abstract_1 + \"\\nTL;DR:\\n\" + tldr_1 + \"\\n\\nAbstract:\\n\" + abstract_2 + \"\\nTL;DR:\\n\" + tldr_2 +  \"\\n\\nAbstract:\\n\"\n",
        "\n",
        "####################"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "33NP0uZ5p0yH",
        "outputId": "5fde6462-3032-48a9-99af-a43a49aa9da5"
      },
      "source": [
        "# Test Abstract 1 with One-Shot TL;DR generation\n",
        "prompt = os_prompt + test_abstract_5 + \"\\nTL;DR:\\n\"\n",
        "\n",
        "%run \"/content/gdrive/MyDrive/NLP_scientific-text-generation/transformers/examples/text-generation/run_generation.py\" \\\n",
        "--model_type gpt2 \\\n",
        "--model_name_or_path '/content/gdrive/MyDrive/NLP_scientific-text-generation/output/' \\\n",
        "--length 100 \\\n",
        "--prompt \"$prompt\" \\\n",
        "--stop_token \"<EOS>\" \\\n",
        "--temperature 0.7 \\\n",
        "--k 50 \\\n",
        "--num_return_sequences 5"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "01/26/2021 23:14:29 - WARNING - __main__ -   device: cuda, n_gpu: 1, 16-bits training: False\n",
            "[INFO|tokenization_utils_base.py:1685] 2021-01-26 23:14:29,409 >> Model name '/content/gdrive/MyDrive/NLP_scientific-text-generation/output/' not found in model shortcut name list (gpt2, gpt2-medium, gpt2-large, gpt2-xl, distilgpt2). Assuming '/content/gdrive/MyDrive/NLP_scientific-text-generation/output/' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
            "[INFO|tokenization_utils_base.py:1718] 2021-01-26 23:14:29,414 >> Didn't find file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/added_tokens.json. We won't load it.\n",
            "[INFO|tokenization_utils_base.py:1718] 2021-01-26 23:14:29,418 >> Didn't find file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/tokenizer.json. We won't load it.\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 23:14:29,421 >> loading file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/vocab.json\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 23:14:29,421 >> loading file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/merges.txt\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 23:14:29,422 >> loading file None\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 23:14:29,424 >> loading file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/special_tokens_map.json\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 23:14:29,425 >> loading file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 23:14:29,427 >> loading file None\n",
            "[INFO|configuration_utils.py:443] 2021-01-26 23:14:29,526 >> loading configuration file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/config.json\n",
            "[INFO|configuration_utils.py:481] 2021-01-26 23:14:29,528 >> Model config GPT2Config {\n",
            "  \"_name_or_path\": \"gpt2\",\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.3.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|modeling_utils.py:1025] 2021-01-26 23:14:29,531 >> loading weights file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/pytorch_model.bin\n",
            "[INFO|modeling_utils.py:1143] 2021-01-26 23:14:34,755 >> All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
            "\n",
            "[INFO|modeling_utils.py:1152] 2021-01-26 23:14:34,756 >> All the weights of GPT2LMHeadModel were initialized from the model checkpoint at /content/gdrive/MyDrive/NLP_scientific-text-generation/output/.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n",
            "01/26/2021 23:14:34 - INFO - __main__ -   Namespace(device=device(type='cuda'), fp16=False, k=50, length=100, model_name_or_path='/content/gdrive/MyDrive/NLP_scientific-text-generation/output/', model_type='gpt2', n_gpu=1, no_cuda=False, num_return_sequences=5, p=0.9, padding_text='', prefix='', prompt=\"Abstract:\\nCausal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1\\u2009km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\\nTL;DR:\\nClimate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\\n\\nAbstract:\\nThe effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\\nTL;DR:\\n\", repetition_penalty=1.0, seed=42, stop_token='<EOS>', temperature=0.7, xlm_language='')\n",
            "[WARNING|generation_utils.py:820] 2021-01-26 23:14:34,913 >> Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "=== GENERATED SEQUENCE 1 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "The climate change hypothesis has been widely accepted and promoted in the literature, but few studies have examined the mechanisms involved in the evolution of the hypothesis. This study provides a systematic review and meta-analysis of the literature on the hypothesis. The results show that the hypothesis is robust, but does not imply a causal relationship between temperature, precipitation and biodiversity loss. This study suggests that the hypothesis could be extended to consider other processes, such as changes in the environment and changes in the evolution of species. This i\n",
            "=== GENERATED SEQUENCE 2 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "The concept of ecological sustainability has been used for many decades to support a wide range of ecological systems and to mitigate risks to the environment. The ecological sustainability of a system depends on the degree of adherence to a set of values and the degree of sustainable development. This paper focuses on the concept of ecological sustainability by examining how the concept of ecological sustainability has been applied to a wide range of systems, including agriculture, fisheries, mining and other forms of environmental management. Ecological sustainability is defined by the concept o\n",
            "=== GENERATED SEQUENCE 3 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "In this paper, I present a simple and practical model of the relationship between temperature and biodiversity using the euro. The model is based on a three-stage model, with a 95% confidence interval (CI) of 0.20–0.75. The model is used to quantify the economic and social benefits of renewable energy, renewable energy subsidies, and carbon taxes. The results show that the economic and social benefits of renewable energy are estimated to be $1.5–2.0 trillion \n",
            "=== GENERATED SEQUENCE 4 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "The purpose of this paper is to highlight the literature on the effects of climate change on biodiversity. However, there are two main limitations. First, this paper focuses on the effect of climate change on the distribution of biodiversity, which is not included in the main analysis. Second, it contains only the most recent year of the current assessment. Given that there is little information on the impact of climate change on biodiversity, this analysis is not designed to provide a general assessment of the impact of climate change on th\n",
            "=== GENERATED SEQUENCE 5 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "A paper by Bockmann and colleagues (2015) in the Journal of the American Statistical Association (annual review) showed that the rate of global change is increasing faster than the rate of natural variability (R2). The authors argue that a simple regression model can be used to show that the rate of global change is increasing faster than the rate of natural variability, but that this is not the case for the global mean. A simple model that can be used to see the rate of global change i\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T63fJ9oSqCtL",
        "outputId": "84ed0035-ecaa-4f29-f366-c33a1aa1a40a"
      },
      "source": [
        "# Test Abstract 1 with Few-Shot TL;DR generation\r\n",
        "prompt = fs_prompt + test_abstract_5 + \"\\nTL;DR:\\n\"\r\n",
        "\r\n",
        "%run \"/content/gdrive/MyDrive/NLP_scientific-text-generation/transformers/examples/text-generation/run_generation.py\" \\\r\n",
        "--model_type gpt2 \\\r\n",
        "--model_name_or_path \"/content/gdrive/MyDrive/NLP_scientific-text-generation/output/\" \\\r\n",
        "--length 100 \\\r\n",
        "--prompt \"$prompt\" \\\r\n",
        "--stop_token \"<EOS>\" \\\r\n",
        "--temperature .7 \\\r\n",
        "--k 50 \\\r\n",
        "--num_return_sequences 5"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "01/26/2021 22:53:53 - WARNING - __main__ -   device: cuda, n_gpu: 1, 16-bits training: False\n",
            "[INFO|tokenization_utils_base.py:1685] 2021-01-26 22:53:53,989 >> Model name '/content/gdrive/MyDrive/NLP_scientific-text-generation/output/' not found in model shortcut name list (gpt2, gpt2-medium, gpt2-large, gpt2-xl, distilgpt2). Assuming '/content/gdrive/MyDrive/NLP_scientific-text-generation/output/' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
            "[INFO|tokenization_utils_base.py:1718] 2021-01-26 22:53:53,991 >> Didn't find file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/added_tokens.json. We won't load it.\n",
            "[INFO|tokenization_utils_base.py:1718] 2021-01-26 22:53:53,993 >> Didn't find file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/tokenizer.json. We won't load it.\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 22:53:53,995 >> loading file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/vocab.json\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 22:53:53,996 >> loading file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/merges.txt\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 22:53:53,996 >> loading file None\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 22:53:53,997 >> loading file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/special_tokens_map.json\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 22:53:53,998 >> loading file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:1764] 2021-01-26 22:53:53,999 >> loading file None\n",
            "[INFO|configuration_utils.py:443] 2021-01-26 22:53:54,088 >> loading configuration file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/config.json\n",
            "[INFO|configuration_utils.py:481] 2021-01-26 22:53:54,091 >> Model config GPT2Config {\n",
            "  \"_name_or_path\": \"gpt2\",\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.3.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|modeling_utils.py:1025] 2021-01-26 22:53:54,093 >> loading weights file /content/gdrive/MyDrive/NLP_scientific-text-generation/output/pytorch_model.bin\n",
            "[INFO|modeling_utils.py:1143] 2021-01-26 22:53:59,506 >> All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
            "\n",
            "[INFO|modeling_utils.py:1152] 2021-01-26 22:53:59,507 >> All the weights of GPT2LMHeadModel were initialized from the model checkpoint at /content/gdrive/MyDrive/NLP_scientific-text-generation/output/.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n",
            "01/26/2021 22:53:59 - INFO - __main__ -   Namespace(device=device(type='cuda'), fp16=False, k=50, length=100, model_name_or_path='/content/gdrive/MyDrive/NLP_scientific-text-generation/output/', model_type='gpt2', n_gpu=1, no_cuda=False, num_return_sequences=5, p=0.9, padding_text='', prefix='', prompt=\"Abstract:\\nCausal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1\\u2009km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\\nTL;DR:\\nClimate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\\n\\nAbstract:\\nSignificantly more carbon is stored in the world's soils—including peatlands, wetlands and permafrost—than is present in the atmosphere. Disagreement exists, however, regarding the effects of climate change on global soil carbon stocks. If carbon stored belowground is transferred to the atmosphere by a warming-induced acceleration of its decomposition, a positive feedback to climate change would occur. Conversely, if increases of plant-derived carbon inputs to soils exceed increases in decomposition, the feedback would be negative. Despite much research, a consensus has not yet emerged on the temperature sensitivity of soil carbon decomposition. Unravelling the feedback effect is particularly difficult, because the diverse soil organic compounds exhibit a wide range of kinetic properties, which determine the intrinsic temperature sensitivity of their decomposition. Moreover, several environmental constraints obscure the intrinsic temperature sensitivity of substrate decomposition, causing lower observed ‘apparent’ temperature sensitivity, and these constraints may, themselves, be sensitive to climate.\\nTL;DR:\\nSoil carbon decomposition may be sensitive to climate, but the amount of decomposition is constrained by other factors.\\n\\nAbstract:\\nThe effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\\nTL;DR:\\n\", repetition_penalty=1.0, seed=42, stop_token='<EOS>', temperature=0.7, xlm_language='')\n",
            "[WARNING|generation_utils.py:820] 2021-01-26 22:53:59,671 >> Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "=== GENERATED SEQUENCE 1 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "Significantly more carbon is stored in the world's soils—including peatlands, wetlands and permafrost—than is present in the atmosphere. Disagreement exists, however, regarding the effects of climate change on global soil carbon stocks. If carbon stored belowground is transferred to the atmosphere by a warming-induced acceleration of its decomposition, a positive feedback to climate change would occur. Conversely, if increases of plant-derived carbon inputs to soils exceed increases in decomposition, the feedback would be negative. Despite much research, a consensus has not yet emerged on the temperature sensitivity of soil carbon decomposition. Unravelling the feedback effect is particularly difficult, because the diverse soil organic compounds exhibit a wide range of kinetic properties, which determine the intrinsic temperature sensitivity of their decomposition. Moreover, several environmental constraints obscure the intrinsic temperature sensitivity of substrate decomposition, causing lower observed ‘apparent’ temperature sensitivity, and these constraints may, themselves, be sensitive to climate.\n",
            "TL;DR:\n",
            "Soil carbon decomposition may be sensitive to climate, but the amount of decomposition is constrained by other factors.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "The literature contains a large number of references to 'categorization' and 'organization' in the financial literature. The current effort is to identify and describe these concepts in the context of the general finance literature. However, there are many conceptual barriers to understanding the financial implications of financial categorization and categorization. The current paper aims at exploring the conceptual issues raised by categorization and the implications of this proposed system on the global financial crisis. We present an empirical assessment of the financial categorization i\n",
            "=== GENERATED SEQUENCE 2 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "Significantly more carbon is stored in the world's soils—including peatlands, wetlands and permafrost—than is present in the atmosphere. Disagreement exists, however, regarding the effects of climate change on global soil carbon stocks. If carbon stored belowground is transferred to the atmosphere by a warming-induced acceleration of its decomposition, a positive feedback to climate change would occur. Conversely, if increases of plant-derived carbon inputs to soils exceed increases in decomposition, the feedback would be negative. Despite much research, a consensus has not yet emerged on the temperature sensitivity of soil carbon decomposition. Unravelling the feedback effect is particularly difficult, because the diverse soil organic compounds exhibit a wide range of kinetic properties, which determine the intrinsic temperature sensitivity of their decomposition. Moreover, several environmental constraints obscure the intrinsic temperature sensitivity of substrate decomposition, causing lower observed ‘apparent’ temperature sensitivity, and these constraints may, themselves, be sensitive to climate.\n",
            "TL;DR:\n",
            "Soil carbon decomposition may be sensitive to climate, but the amount of decomposition is constrained by other factors.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "This paper focuses on the contribution of macroeconomic variables to biodiversity loss. In particular, it examines the impact macroeconomic variables have on biodiversity loss (i.e., the impact on biodiversity of the degree of global warming). The authors analyse the impact macroeconomic variables have on the biodiversity loss of countries in the world's richest and most developed countries. Their results show that the macroeconomic variables have a significant impact on biodiversity loss and that they are associated with the degree of global warming. The implications of thi\n",
            "=== GENERATED SEQUENCE 3 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "Significantly more carbon is stored in the world's soils—including peatlands, wetlands and permafrost—than is present in the atmosphere. Disagreement exists, however, regarding the effects of climate change on global soil carbon stocks. If carbon stored belowground is transferred to the atmosphere by a warming-induced acceleration of its decomposition, a positive feedback to climate change would occur. Conversely, if increases of plant-derived carbon inputs to soils exceed increases in decomposition, the feedback would be negative. Despite much research, a consensus has not yet emerged on the temperature sensitivity of soil carbon decomposition. Unravelling the feedback effect is particularly difficult, because the diverse soil organic compounds exhibit a wide range of kinetic properties, which determine the intrinsic temperature sensitivity of their decomposition. Moreover, several environmental constraints obscure the intrinsic temperature sensitivity of substrate decomposition, causing lower observed ‘apparent’ temperature sensitivity, and these constraints may, themselves, be sensitive to climate.\n",
            "TL;DR:\n",
            "Soil carbon decomposition may be sensitive to climate, but the amount of decomposition is constrained by other factors.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "In this paper, the term \"sustainability\" refers to the concept of a sustainable development that is based on the ability of the community to thrive and to sustain itself. The concept is applied to many areas of the economy, from education to health and beyond. The concept is considered to be a core component of sustainable development, i.e., a set of principles that should guide the development of the economy and society. However, it is not sufficient to just define the concept and understand its basi\n",
            "=== GENERATED SEQUENCE 4 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "Significantly more carbon is stored in the world's soils—including peatlands, wetlands and permafrost—than is present in the atmosphere. Disagreement exists, however, regarding the effects of climate change on global soil carbon stocks. If carbon stored belowground is transferred to the atmosphere by a warming-induced acceleration of its decomposition, a positive feedback to climate change would occur. Conversely, if increases of plant-derived carbon inputs to soils exceed increases in decomposition, the feedback would be negative. Despite much research, a consensus has not yet emerged on the temperature sensitivity of soil carbon decomposition. Unravelling the feedback effect is particularly difficult, because the diverse soil organic compounds exhibit a wide range of kinetic properties, which determine the intrinsic temperature sensitivity of their decomposition. Moreover, several environmental constraints obscure the intrinsic temperature sensitivity of substrate decomposition, causing lower observed ‘apparent’ temperature sensitivity, and these constraints may, themselves, be sensitive to climate.\n",
            "TL;DR:\n",
            "Soil carbon decomposition may be sensitive to climate, but the amount of decomposition is constrained by other factors.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "This paper presents the current state of knowledge regarding the impact of climate change on biodiversity in the tropics and subtropics. The paper covers a wide range of topics related to the management of biodiversity and the management of the environment, as well as the impact of climate change on agriculture and the environment itself. The paper analyzes the literature on the impact of climate change on biodiversity and how it can be applied to mitigate the impacts of climate change on agriculture. The paper concludes by considering the implications of th\n",
            "=== GENERATED SEQUENCE 5 ===\n",
            "Abstract:\n",
            "Causal attribution of recent biological trends to climate change is complicated because non-climatic influences dominate local, short-term biological changes. Any underlying signal from climate change is likely to be revealed by analyses that seek systematic trends across diverse species and geographic regions; however, debates within the Intergovernmental Panel on Climate Change (IPCC) reveal several definitions of a ‘systematic trend’. Here, we explore these differences, apply diverse analyses to more than 1,700 species, and show that recent biological trends match climate change predictions. Global meta-analyses documented significant range shifts averaging 6.1 km per decade towards the poles (or metres per decade upward), and significant mean advancement of spring events by 2.3 days per decade. We define a diagnostic fingerprint of temporal and spatial ‘sign-switching’ responses uniquely predicted by twentieth century climate trends. Among appropriate long-term/large-scale/multi-species data sets, this diagnostic fingerprint was found for 279 species. This suite of analyses generates ‘very high confidence’ (as laid down by the IPCC) that climate change is already affecting living systems.\n",
            "TL;DR:\n",
            "Climate change predictions are confirmed by the collective change in distribution of species, and the change in timing of biological events.\n",
            "\n",
            "Abstract:\n",
            "Significantly more carbon is stored in the world's soils—including peatlands, wetlands and permafrost—than is present in the atmosphere. Disagreement exists, however, regarding the effects of climate change on global soil carbon stocks. If carbon stored belowground is transferred to the atmosphere by a warming-induced acceleration of its decomposition, a positive feedback to climate change would occur. Conversely, if increases of plant-derived carbon inputs to soils exceed increases in decomposition, the feedback would be negative. Despite much research, a consensus has not yet emerged on the temperature sensitivity of soil carbon decomposition. Unravelling the feedback effect is particularly difficult, because the diverse soil organic compounds exhibit a wide range of kinetic properties, which determine the intrinsic temperature sensitivity of their decomposition. Moreover, several environmental constraints obscure the intrinsic temperature sensitivity of substrate decomposition, causing lower observed ‘apparent’ temperature sensitivity, and these constraints may, themselves, be sensitive to climate.\n",
            "TL;DR:\n",
            "Soil carbon decomposition may be sensitive to climate, but the amount of decomposition is constrained by other factors.\n",
            "\n",
            "Abstract:\n",
            "The effects of climate change on biodiversity are increasingly well documented, and many methods have been developed to assess species' vulnerability to climatic changes, both ongoing and projected in the coming decades. To minimize global biodiversity losses, conservationists need to identify those species that are likely to be most vulnerable to the impacts of climate change. In this Review, we summarize different currencies used for assessing species' climate change vulnerability. We describe three main approaches used to derive these currencies (correlative, mechanistic and trait-based), and their associated data requirements, spatial and temporal scales of application and modelling methods. We identify strengths and weaknesses of the approaches and highlight the sources of uncertainty inherent in each method that limit projection reliability. Finally, we provide guidance for conservation practitioners in selecting the most appropriate approach(es) for their planning needs and highlight priority areas for further assessments.\n",
            "TL;DR:\n",
            "The paper draws on the literature on the impact of climate change on ecosystems. The analysis was conducted in cooperation with the Global Change Research Institute (GCRI), a leading global research and education organization (GURI) with a focus on sustainable development. This paper focuses on the impact of climate change on the biodiversity of the world's major food crops (e.g., wheat, soybeans, corn, rice, wheat, maize, wheat berry, sorghum). The paper draws on th\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BLN2OQfTxD1n"
      },
      "source": [
        "## Appendix\r\n",
        "For creating a dataset based on all abstracts use this code for the third step in the datapreparation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3C9DP_BFZBdd"
      },
      "source": [
        "#3. Data preparation\n",
        "import os\n",
        "import json\n",
        "import pandas as pd\n",
        "\n",
        "dir = '/content/gdrive/MyDrive/NLP_scientific-text-generation/json'\n",
        "\n",
        "abstracts = []\n",
        "count = 7129 #7129 includes all abstracts in the dataset. Change to a smaller number to chose only a subset.\n",
        "i = 0\n",
        "\n",
        "for f_name in os.listdir(dir):\n",
        "  if f_name.endswith(\".json\"):\n",
        "    path = os.path.join(dir, f_name)\n",
        "    with open(path, \"r\", encoding=\"utf8\") as f:\n",
        "      data = json.load(f)\n",
        "      is_english = False\n",
        "      abstract = \"\"\n",
        "      for obj in data[\"metadata\"]:\n",
        "        if obj[\"key\"] == \"dc.language.iso\":\n",
        "          if obj[\"value\"] == \"eng\":\n",
        "            is_english = True\n",
        "          else:\n",
        "            break\n",
        "        elif obj[\"key\"] == \"dc.description.abstract\":\n",
        "          abstract = obj[\"value\"]\n",
        "          if is_english:\n",
        "            break\n",
        "      \n",
        "      if is_english:\n",
        "        abstracts.append(abstract)\n",
        "\n",
        "    # ensure we get exactly the amount of samples required\n",
        "    if len(abstracts) >= count:\n",
        "      break\n",
        "#put the list into a dataframe to print the abstracts with an index to count the number of abstracts\n",
        "df = pd.DataFrame(abstracts)\n",
        "df.columns = ['abstract']\n",
        "df = df.reset_index()\n",
        "print(df)      "
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}