Crawler_Corn_Soybeans.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Crawler_Corn_Soybeans.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rodrigonevest/web-scraping/blob/main/Crawler_Corn_Soybeans.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LIwtULu0Z5lx",
        "outputId": "0eb0612f-eee3-4d99-f1f6-ce975d96df4e"
      },
      "source": [
        "import urllib3\n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "import re\n",
        "import nltk\n",
        "\n",
        "nltk.download('book')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading collection 'book'\n",
            "[nltk_data]    | \n",
            "[nltk_data]    | Downloading package abc to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/abc.zip.\n",
            "[nltk_data]    | Downloading package brown to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/brown.zip.\n",
            "[nltk_data]    | Downloading package chat80 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/chat80.zip.\n",
            "[nltk_data]    | Downloading package cmudict to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/cmudict.zip.\n",
            "[nltk_data]    | Downloading package conll2000 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2000.zip.\n",
            "[nltk_data]    | Downloading package conll2002 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/conll2002.zip.\n",
            "[nltk_data]    | Downloading package dependency_treebank to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/dependency_treebank.zip.\n",
            "[nltk_data]    | Downloading package genesis to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/genesis.zip.\n",
            "[nltk_data]    | Downloading package gutenberg to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/gutenberg.zip.\n",
            "[nltk_data]    | Downloading package ieer to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ieer.zip.\n",
            "[nltk_data]    | Downloading package inaugural to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/inaugural.zip.\n",
            "[nltk_data]    | Downloading package movie_reviews to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/movie_reviews.zip.\n",
            "[nltk_data]    | Downloading package nps_chat to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/nps_chat.zip.\n",
            "[nltk_data]    | Downloading package names to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/names.zip.\n",
            "[nltk_data]    | Downloading package ppattach to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/ppattach.zip.\n",
            "[nltk_data]    | Downloading package reuters to /root/nltk_data...\n",
            "[nltk_data]    | Downloading package senseval to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/senseval.zip.\n",
            "[nltk_data]    | Downloading package state_union to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/state_union.zip.\n",
            "[nltk_data]    | Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data]    | Downloading package swadesh to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/swadesh.zip.\n",
            "[nltk_data]    | Downloading package timit to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/timit.zip.\n",
            "[nltk_data]    | Downloading package treebank to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/treebank.zip.\n",
            "[nltk_data]    | Downloading package toolbox to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/toolbox.zip.\n",
            "[nltk_data]    | Downloading package udhr to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr.zip.\n",
            "[nltk_data]    | Downloading package udhr2 to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/udhr2.zip.\n",
            "[nltk_data]    | Downloading package unicode_samples to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/unicode_samples.zip.\n",
            "[nltk_data]    | Downloading package webtext to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/webtext.zip.\n",
            "[nltk_data]    | Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet.zip.\n",
            "[nltk_data]    | Downloading package wordnet_ic to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/wordnet_ic.zip.\n",
            "[nltk_data]    | Downloading package words to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/words.zip.\n",
            "[nltk_data]    | Downloading package maxent_treebank_pos_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/maxent_treebank_pos_tagger.zip.\n",
            "[nltk_data]    | Downloading package maxent_ne_chunker to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping chunkers/maxent_ne_chunker.zip.\n",
            "[nltk_data]    | Downloading package universal_tagset to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/universal_tagset.zip.\n",
            "[nltk_data]    | Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data]    | Downloading package book_grammars to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping grammars/book_grammars.zip.\n",
            "[nltk_data]    | Downloading package city_database to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping corpora/city_database.zip.\n",
            "[nltk_data]    | Downloading package tagsets to /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping help/tagsets.zip.\n",
            "[nltk_data]    | Downloading package panlex_swadesh to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    | Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]    |     /root/nltk_data...\n",
            "[nltk_data]    |   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
            "[nltk_data]    | \n",
            "[nltk_data]  Done downloading collection book\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rg17slmPkOik",
        "outputId": "ec548c63-d744-4af4-f7e6-657443d31bed"
      },
      "source": [
        "!pip install -U pycountry\n",
        "\n",
        "import pycountry"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pycountry\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/76/73/6f1a412f14f68c273feea29a6ea9b9f1e268177d32e0e69ad6790d306312/pycountry-20.7.3.tar.gz (10.1MB)\n",
            "\u001b[K     |████████████████████████████████| 10.1MB 5.4MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pycountry\n",
            "  Building wheel for pycountry (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pycountry: filename=pycountry-20.7.3-py2.py3-none-any.whl size=10746863 sha256=7a8ee6b16901fc24559bb7305ff495fd3982b6b60dd7096a809602cdba5facc3\n",
            "  Stored in directory: /root/.cache/pip/wheels/33/4e/a6/be297e6b83567e537bed9df4a93f8590ec01c1acfbcd405348\n",
            "Successfully built pycountry\n",
            "Installing collected packages: pycountry\n",
            "Successfully installed pycountry-20.7.3\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6IiMo9uoaByF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "8df1e725-c505-4b39-a05b-bd7add362370"
      },
      "source": [
        " \n",
        "#print(cbot.head())\n",
        "def BuscaTexto(pagina):\n",
        "  \n",
        "  urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n",
        "\n",
        "  #Criando o dataframe para salvar os dados\n",
        "  df = pd.DataFrame(index=['Date','Link','Headlines','News'])#,'Cbot_corn','CepeaR$_corn','CepeaU$','Rótulo'\"\"\",'Pais'\"\"\"])  \n",
        "\n",
        "  http = urllib3.PoolManager()\n",
        " \n",
        "  try:\n",
        "    dados_paginas = http.request('GET',pagina)\n",
        " \n",
        "  except HTTPError as e:\n",
        "    print(e)\n",
        "  except URLError:\n",
        "    print(\"Server down or incorrect domain\")  \n",
        "   # Pegando os dados do html da pagina\n",
        "  index = BeautifulSoup(dados_paginas.data,'lxml')\n",
        "\n",
        "  #Procurando as tag \"a\" dentro da \"div\"\n",
        "  links = index.find(\"div\",{\"id\":\"content\"})\n",
        "  links = links.find_all('a')\n",
        "  #print(links)\n",
        "  \n",
        "  cont = 0\n",
        "  base = pagina.split('/News')[0]\n",
        "\n",
        "  #Percorrendo os link e salvando seu href(os links)    \n",
        "  for link in links:    \n",
        "    if cont == range(len(base)):\n",
        "    #if cont == 160:\n",
        "         \n",
        "      break;\n",
        " \n",
        "    else:\n",
        "      cont +=1   \n",
        "        \n",
        "    aux = str(link.get('href'))\n",
        "    url = base + '/' + aux\n",
        " \n",
        "    try:\n",
        "      dados = http.request('GET',url)\n",
        "    except:\n",
        "     print(\"Pagina nao encontrada\" )\n",
        "    \n",
        " \n",
        "    #Pegando os dados do html do link acessado\n",
        "    news = BeautifulSoup(dados.data,'lxml')\n",
        "\n",
        "    #Pegando o conteúdo da Manchete\n",
        "    conteudo = news.find(\"div\",{\"id\": \"content\"})       \n",
        "    #conteudo = news.find(\"div\",{\"id\":\"section\"})\n",
        "    #conteudo = news.find_all({\"h3\"})\n",
        "    #if conteudo != None:\n",
        "    #x2 = '' \n",
        "    #if conteudo != None:   \n",
        "     # for x in conteudo:   \n",
        "      #  x2 +=''.join(x.find_all(text=True))              \n",
        "       # x2 = re.sub(\"[\\n,\\t]\",\"\",x2) \n",
        "           \n",
        "    #Pegando o conteudo das noticias\n",
        "    noticias = news.find(\"div\",{\"id\": \"content\"}) \n",
        "    noticia = news.find(\"div\",{\"id\": \"content\"})\n",
        "         \n",
        "    #noticias = news.find(\"div\",{\"id\":\"section\"})\n",
        "    #noticias = news.find_all({\"p\"})\n",
        "    if noticias != None:\n",
        "      noticias = [y.get_text(strip=True) for y in news.find_all('p')]      \n",
        "      noticias = str(noticias)\n",
        "      noticias = noticias.replace(\"['\",'')\n",
        "      noticias = noticias.replace(\"']\",'')\n",
        "      noticias = noticias.replace('[\"','')\n",
        "      noticias = noticias.replace('\"]','')\n",
        "      noticias = noticias.replace(\"', '\",'') \n",
        "\n",
        "      conteudo = [x.get_text(strip=True) for x in news.find_all('h3')]\n",
        "      conteudo = str(conteudo)\n",
        "      conteudo = conteudo.replace(\"['\",'')\n",
        "      conteudo = conteudo.replace(\"']\",'')\n",
        "      conteudo = conteudo.replace('[\"','')\n",
        "      conteudo = conteudo.replace('\"]','')\n",
        "      conteudo = conteudo.replace(\"', '\",'')\n",
        "       \n",
        "      data = news.find({\"h5\"}).get_text(strip=True)\n",
        "      if noticia != None:    \n",
        "        noticia = [j for j in noticia.find_all('li',text=True)]      \n",
        "        noticia = str(noticia)\n",
        "        noticia = noticia.replace(\"['\",'')\n",
        "        noticia = noticia.replace(\"']\",'')\n",
        "        noticia = noticia.replace('[\"','')\n",
        "        noticia = noticia.replace('\"]','')\n",
        "        noticia = noticia.replace(\"', '\",'')\n",
        "        noticia = noticia.replace('<\"','')\n",
        "        noticia = noticia.replace('\">','') \n",
        "        noticia = noticia.replace('\"\\n','')\n",
        "        noticia = re.sub('[\\n,\\t,<li>,/]','',noticia)\n",
        "\n",
        "    \n",
        "\n",
        "   # y2 = '' \n",
        "    #if noticias != None:       \n",
        "     # for y in noticias :           \n",
        "      #  y2 +=' '.join(y.find_all(text=True))              \n",
        "       # y2 = re.sub(\"[\\n,\\t,\\r,\\f]\",\"\",y2)\n",
        "       \n",
        "\n",
        "\n",
        "    #Pegando o conteúdo da Data \n",
        "    #data = news.find(\"div\",{\"id\": \"content\"})\n",
        "    #data = news.find(\"div\",{\"id\":\"section\"})\n",
        "      data = news.find({\"h5\"}).get_text(strip=True)\n",
        "              \n",
        "    \n",
        "    #if data != None:\n",
        "     # txt_dt = data.get_text(strip=True)\n",
        "\n",
        "    \n",
        "    \n",
        "      df = df.append({'Date': data,\n",
        "                        'Link': url ,\n",
        "                        'Heallines' : conteudo,\n",
        "                         'News': noticias + noticia,\n",
        "                          }, #'Pais': j},                                   \n",
        "                          ignore_index=True)      \n",
        "  \n",
        "  return df\n",
        "\"\"\"\n",
        "    paises = list()\n",
        "    for y in pycountry.countries:\n",
        "      paises.append(y.name)    \n",
        "    j = list()\n",
        "    cont = 0\n",
        "    for p in paises:\n",
        "      if p in texto:        \n",
        "        j.append(p)\n",
        "\"\"\" "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'\\n   paises = list()\\n   for y in pycountry.countries:\\n     paises.append(y.name)    \\n   j = list()\\n   cont = 0\\n   for p in paises:\\n     if p in texto:        \\n       j.append(p)\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        },
        "id": "ycnr9DN1aGJB",
        "outputId": "f29868d0-5e18-479e-b4a0-f9f36bf19b9e"
      },
      "source": [
        "link2 = 'http://soybeansandcorn.com/News'\n",
        "\n",
        "df = BuscaTexto(link2)\n",
        "df.dropna(inplace=True)\n",
        "df['Date'] = pd.to_datetime(df['Date'],errors='coerce')\n",
        "df.dropna(subset=['Date'],inplace=True)\n",
        "\n",
        "\n",
        "df.head() \n",
        "\n",
        " \n",
        "#print(df.isnull().sum())\n",
        "#print(df.to_string()) \n",
        "#df.head(20) \n",
        "\n",
        "\n",
        "#print(df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Date</th>\n",
              "      <th>Heallines</th>\n",
              "      <th>Link</th>\n",
              "      <th>News</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2021-02-18</td>\n",
              "      <td>Congestion at Port of Miritituba not Expected ...</td>\n",
              "      <td>http://soybeansandcorn.com/news/Feb18_21-Conge...</td>\n",
              "      <td>Since last week, there have been thousands of ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>2021-02-18</td>\n",
              "      <td>Later Planted Soy in Eastern Parana Avoided We...</td>\n",
              "      <td>http://soybeansandcorn.com/news/Feb18_21-Later...</td>\n",
              "      <td>In the municipality of Castro, which is locate...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>2021-02-17</td>\n",
              "      <td>2020/21 Argentina Soybeans 23% Good/Excellent,...</td>\n",
              "      <td>http://soybeansandcorn.com/news/Feb17_21-20202...</td>\n",
              "      <td>The weather last week was OK in Argentina with...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>2021-02-17</td>\n",
              "      <td>2020/21 Argentina Corn 23% Good/Excellent, 16%...</td>\n",
              "      <td>http://soybeansandcorn.com/news/Feb17_21-20202...</td>\n",
              "      <td>Even though the weather in Argentina has been ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>2021-02-16</td>\n",
              "      <td>2020/21 Brazil Soybeans 9% Harvested vs. 24% L...</td>\n",
              "      <td>http://soybeansandcorn.com/news/Feb16_21-20202...</td>\n",
              "      <td>The 2020/21 Brazil soybean crop was 9% harvest...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        Date  ...                                               News\n",
              "4 2021-02-18  ...  Since last week, there have been thousands of ...\n",
              "5 2021-02-18  ...  In the municipality of Castro, which is locate...\n",
              "6 2021-02-17  ...  The weather last week was OK in Argentina with...\n",
              "7 2021-02-17  ...  Even though the weather in Argentina has been ...\n",
              "8 2021-02-16  ...  The 2020/21 Brazil soybean crop was 9% harvest...\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Pf8o6gj8K8qS",
        "outputId": "1bbcb0b5-3439-408a-cf08-85eaa6ca681e"
      },
      "source": [
        "df.isnull().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Date         0\n",
              "Heallines    0\n",
              "Link         0\n",
              "News         0\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qGbB3RMg5Tub"
      },
      "source": [
        "df.to_csv('Crawler_Corn_Soybeans.csv',index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}