9_29ソート機能付STYLE4Design_Scraping.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"9/29ソート機能付STYLE4Design_Scraping.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"HDfMpa_EQF6h","colab_type":"code","colab":{}},"source":["import pandas as pd\n","import requests\n","from google.colab import files\n","from bs4 import BeautifulSoup\n","import time\n","\n","\n","# ソートしたいcategoryを選択\n","category = {0: 'アート/デザイン',\n","            1: 'インテリア',\n","            2: 'ガジェット',\n","            3: 'グルメ/キッチン',\n","            4: 'ゲーム',\n","            5: 'ファッション',\n","            6: 'フォト',\n","            7: 'ミュージック',\n","            8: '動画/ムービー',\n","            9: '広告',\n","            10: '建築/スポット',\n","            11: '本',\n","            12: '雑貨'\n","           }\n","\n","# ソートする際に引っ張ってくるURLに組み合わせるtag名\n","category_url = {0: 'art-design',\n","                1: 'interior',\n","                2: 'gadgets',\n","                3: 'gourmet',\n","                4: 'games',\n","                5: 'fashion',\n","                6: 'photo',\n","                7: 'music',\n","                8: 'video',\n","                9: 'ad',\n","                10: 'architecture',\n","                11: 'books',\n","                12: 'groceries'\n","               }\n","\n","print(category)\n","a = int(input('見たいカテゴリーを選んでください！'))\n","print(category[a] + 'ですね。ソートして表示します')\n","\n","# スクレイプするサイトの指定\n","base_url = 'https://design.style4.info/category/'\n","\n","# BeautifulSoupで解析＆カテゴリソート済の全ページ数と最終ページ数の取得\n","res = BeautifulSoup(requests.get(base_url + category_url[a]).text, 'html.parser')\n","all_pages = res.find('div', {'class': 'pagination clearfix'}).span.text\n","max_page = int(all_pages.replace('Page 1 of ', ''))\n","print('ページ数は' + str(max_page) + 'です')\n","\n","# データフレーム指定\n","cols = ['item', 'url']\n","df = pd.DataFrame(columns = cols)\n","\n","# ここから本番\n","# 「タイトルとURL取得→dfに入れる」を最終ページまで繰り返し処理\n","for i in range(1, max_page + 1):\n","    print('-' * 70)\n","    \n","    # 文字化け防止とデータ解析\n","    response = requests.get(base_url + category_url[a] + '/page/' + str(i))\n","    response.encoding = response.apparent_encoding\n","    soup = BeautifulSoup(response.text, 'html.parser')\n","    \n","    # 記事の１つのカタマリをリスト化でもってくる\n","    item_list = soup.find_all('div', {'class': 'item-main'})\n","    # 記事のカタマリを１つずつ持ってくる\n","    for item in item_list:\n","        \n","        # GoogleAdsの回避\n","        if item.a is None:\n","            pass\n","        else:\n","            url = item.h2.a.get('href')\n","            name = item.h2.a.string\n","            print(name)\n","            print(url)\n","            se = pd.Series([name, url], cols)\n","            df = df.append(se, ignore_index=True)\n","    time.sleep(1)\n","\n","df.info()\n","filename = 'Fun_article.csv'\n","df.to_csv(filename, encoding = 'utf-8-sig')\n","files.download(filename)\n"],"execution_count":0,"outputs":[]}]}