Final_Project_4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from lxml import html\n",
    "import pandas as pd\n",
    "import scraperwiki\n",
    "import lxml.html\n",
    "import numpy as np\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pip install -U selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "browser = webdriver.Chrome('C:/Users/MikasaServer/Downloads/chromedriver.exe')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "colnames = ['Company']\n",
    "companies_df = pd.read_csv('Companies_names.csv', encoding = \"ISO-8859-1\", names=colnames)\n",
    "\n",
    "#We import as 'companies' the companies names.\n",
    "\n",
    "companies = companies_df['Company']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Pages to avoid :\n",
    "\n",
    "#https://beta.companieshouse.gov.uk\n",
    "#https://en.wikipedia.org\n",
    "#https://www.theconstructionindex.co.uk\n",
    "#https://www.bloomberg.com\n",
    "#https://www.linkedin.com\n",
    "#https://companycheck.co.uk\n",
    "\n",
    "#We store partial links of the wrong websites in a list:\n",
    "wrong_websites=['Officers','Overview', 'LinkedIn','Wikipedia', 'Free business summary', 'Endole','Bloomberg', 'theconstructionindex', 'SoloCheck', 'Vision-Net']\n",
    "\n",
    "#We store the possible partial links of the Modern Slavery statements in a list:\n",
    "statements_names=['Modern', 'Slavery', 'modern', 'slavery', 'MODERN', 'SLAVERY']\n",
    "\n",
    "len(wrong_websites)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#First scraper. \n",
    "\n",
    "webpage=[]\n",
    "statements=[]\n",
    "no_statements=[]\n",
    "\n",
    "for i in np.arange(1612):\n",
    "    browser.get(\"https://www.google.com/\")\n",
    "    from selenium.webdriver.common.keys import Keys\n",
    "    input=browser.find_elements_by_tag_name('input')[3].send_keys(companies[i], Keys.RETURN)\n",
    "    from selenium.common.exceptions import NoSuchElementException\n",
    "    \n",
    "    try :\n",
    "        website= browser.find_element_by_partial_link_text('.co.uk')\n",
    "        try:\n",
    "            for i in np.arange(10):\n",
    "                wrong=browser.find_element_by_partial_link_text(wrong_websites[i])\n",
    "        except NoSuchElementException:\n",
    "            wrong=None\n",
    "        if not website is wrong:                                    \n",
    "            webpage.append(website.text)\n",
    "            website.click()\n",
    "            \n",
    "    except NoSuchElementException:\n",
    "        website= browser.find_element_by_partial_link_text('.com')\n",
    "        try:\n",
    "            for i in np.arange(10):\n",
    "                wrong=browser.find_element_by_partial_link_text(wrong_websites[i])\n",
    "        except NoSuchElementException:\n",
    "            wrong=None\n",
    "        if not website is wrong:\n",
    "            try:\n",
    "                webpage.append(website.text)\n",
    "                website.click()   \n",
    "            except NoSuchElementException:\n",
    "                next\n",
    "                \n",
    "    time.sleep(3)\n",
    "    #This is to find the statement inside the webpage: \n",
    "    \n",
    "    try:\n",
    "        for r in statements_names:\n",
    "            statement=browser.find_element_by_partial_link_text(r)\n",
    "            statement=statement.get_attribute(\"href\")\n",
    "            statements.append(statement)\n",
    "\n",
    "    except NoSuchElementException:\n",
    "        no_statement='No statement found'\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "colnames = ['Company']\n",
    "companies_df = pd.read_csv('Companies_names_2_for_scraper.csv', encoding = \"ISO-8859-1\", names=colnames)\n",
    "\n",
    "#We import as 'companies' the companies names.\n",
    "\n",
    "companies_2 = companies_df['Company']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "df = pd.DataFrame(data=webpage)\n",
    "df.to_csv(\"2_webpages_result.csv\", sep=',',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data=statements)\n",
    "df.to_csv(\"2_statements_result.csv\", sep=',',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data=no_statements)\n",
    "df.to_csv(\"1_no_statements_result.csv\", sep=',',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(companies_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#First scraper. 2nd part. \n",
    "webpage=[]\n",
    "statements=[]\n",
    "\n",
    "\n",
    "for i in np.arange(521):\n",
    "    browser.get(\"https://www.google.com/\")\n",
    "    from selenium.webdriver.common.keys import Keys\n",
    "    input=browser.find_elements_by_tag_name('input')[3].send_keys(companies_2[i], Keys.RETURN)\n",
    "    from selenium.common.exceptions import NoSuchElementException\n",
    "    \n",
    "    try :\n",
    "        website= browser.find_element_by_partial_link_text('.co.uk')\n",
    "        try:\n",
    "            for i in np.arange(10):\n",
    "                wrong=browser.find_element_by_partial_link_text(wrong_websites[i])\n",
    "        except NoSuchElementException:\n",
    "            wrong=None\n",
    "        if not website is wrong:                                    \n",
    "            webpage.append(website.text)\n",
    "            website.click()\n",
    "            \n",
    "    except NoSuchElementException:\n",
    "        website= browser.find_element_by_partial_link_text('.com')\n",
    "        try:\n",
    "            for i in np.arange(10):\n",
    "                wrong=browser.find_element_by_partial_link_text(wrong_websites[i])\n",
    "        except NoSuchElementException:\n",
    "            wrong=None\n",
    "        if not website is wrong:\n",
    "            try:\n",
    "                webpage.append(website.text)\n",
    "                website.click()   \n",
    "            except NoSuchElementException:\n",
    "                next\n",
    "                \n",
    "    time.sleep(3)\n",
    "    #This is to find the statement inside the webpage: \n",
    "    \n",
    "    try:\n",
    "        for r in statements_names:\n",
    "            statement=browser.find_element_by_partial_link_text(r)\n",
    "            statement=statement.get_attribute(\"href\")\n",
    "            statements.append(statement)\n",
    "\n",
    "    except NoSuchElementException:\n",
    "        no_statement='No statement found'\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2nd scraper for invalid websites returned on the 1st scraper. \n",
    "\n",
    "import pandas as pd\n",
    "colnames = ['Name']\n",
    "companies_invalid_websites_1st_scraper = pd.read_csv('companies_with_Invalid_websites_for_2nd_scraper.csv', encoding = \"ISO-8859-1\", names=colnames)\n",
    "\n",
    "#We import as the companies names.\n",
    "\n",
    "companies_for_2nd_scraper_1 = companies_invalid_websites_1st_scraper['Name']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    " #2nd scraper for invalid websites returned by the 1st scraper. \n",
    "\n",
    "webpages_3=[]\n",
    "statements_3=[]\n",
    "\n",
    "\n",
    "for i in np.arange(628):\n",
    "    browser.get(\"https://www.google.com/\")\n",
    "    from selenium.webdriver.common.keys import Keys\n",
    "    input=browser.find_elements_by_tag_name('input')[3].send_keys(companies_for_2nd_scraper_1[i], Keys.RETURN)\n",
    "    from selenium.common.exceptions import NoSuchElementException\n",
    "\n",
    "    try :\n",
    "        website= browser.find_element_by_partial_link_text('.com')\n",
    "        webpage_3.append(website.text)\n",
    "        website.click()\n",
    "   \n",
    "    except NoSuchElementException:\n",
    "        website= browser.find_element_by_partial_link_text('.co.uk')\n",
    "        try:\n",
    "            for i in np.arange(10):\n",
    "                wrong=browser.find_element_by_partial_link_text(wrong_websites[i])\n",
    "        except NoSuchElementException:\n",
    "            wrong=None\n",
    "        if not website is wrong:\n",
    "            try:\n",
    "                webpages_3.append(website.text)\n",
    "                website.click()   \n",
    "            except NoSuchElementException:\n",
    "                next\n",
    "                \n",
    "    time.sleep(3)\n",
    "    #This is to find the statement inside the webpage: \n",
    "    \n",
    "    try:\n",
    "        for r in statements_names:\n",
    "            statement=browser.find_element_by_partial_link_text(r)\n",
    "            statement=statement.get_attribute(\"href\")\n",
    "            statements_3.append(statement)\n",
    "\n",
    "    except NoSuchElementException:\n",
    "        no_statement='No statement found'\n",
    "      \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "df = pd.DataFrame(data=webpages_3)\n",
    "df.to_csv(\"2ndscraper_1stpart_webpages_result.csv\", sep=',',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data=statements_3)\n",
    "df.to_csv(\"2ndscraper_1srpart_statements_result.csv\", sep=',',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Valid websites. 2nd scraper. \n",
    "\n",
    "import pandas as pd\n",
    "colnames = ['Name']\n",
    "companies_valid_websites_2nd_scraper = pd.read_csv('valid_websites_for2nd_scraper.csv', encoding = \"ISO-8859-1\", names=colnames)\n",
    "\n",
    "\n",
    "companies_for_2nd_scraper_2 = companies_valid_websites_2nd_scraper['Name']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#2nd scraper 2nd part (valid wbesites) using Selenium. \n",
    "\n",
    "statements_5=[]\n",
    "\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "    \n",
    "\n",
    "for i in np.arange(616):\n",
    "    browser.get(companies_for_2nd_scraper_2[i])\n",
    "   \n",
    "    time.sleep(3)\n",
    "    #This is to find the statement inside the webpage: \n",
    "    \n",
    "    try:\n",
    "        for r in statements_names:\n",
    "            statement=browser.find_element_by_partial_link_text(r)\n",
    "            statement=statement.get_attribute(\"href\")\n",
    "            statements_5.append(statement)\n",
    "\n",
    "    except NoSuchElementException:\n",
    "            no_statement='No statement found'\n",
    "        \n",
    "                \n",
    "  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}