notebook/crawl_tweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# [GetOldTweets](https://github.com/Mottl/GetOldTweets3) 사용법\n",
    "- `.tweetCriteria()` 설정 : 날짜 설정 시 마지막 날짜 포함되지 않음에 주의.\n",
    "- `.tweetManager` 호출 후 `getTweets()` 메소드 실행."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# install GOT3 package\n",
    "!pip install GetOldTweets3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ver 1 : 날짜 수기 입력\n",
    "\n",
    "- 날짜 입력 에러 커스텀. \n",
    "- `getJsonResponse` 함수 수정.\n",
    "    - 디버그 옵션 수정 : 디버그 시 헤더 정보, json 정보 전부 출력하지 않도록 변경.\n",
    "    - HTTP 429 에러: `time.sleep`. 10 -> 5 -> 3 -> 2 -> 1.\n",
    "    - TimeOut Error: 한 번 더 기다리고 안 되면 pass.\n",
    "    - Url 에러, Json 파싱 에러: 바로 pass."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# module import\n",
    "import GetOldTweets3 as got\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import sys\n",
    "import urllib\n",
    "import json\n",
    "import datetime\n",
    "import time\n",
    "import os\n",
    "from random import uniform\n",
    "from tqdm import tqdm\n",
    "import csv\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getJsonResponse(tweetCriteria, refreshCursor, cookieJar, proxy, useragent=None, debug=False):\n",
    "    \"\"\"\n",
    "    Invoke an HTTP query to Twitter.\n",
    "    Should not be used as an API function. A static method.\n",
    "    \"\"\"\n",
    "    url = \"https://twitter.com/i/search/timeline?\"\n",
    "\n",
    "    if not tweetCriteria.topTweets:\n",
    "        url += \"f=tweets&\"\n",
    "\n",
    "    url += (\"vertical=news&q=%s&src=typd&%s\"\n",
    "            \"&include_available_features=1&include_entities=1&max_position=%s\"\n",
    "            \"&reset_error_state=false\")\n",
    "\n",
    "    urlGetData = ''\n",
    "\n",
    "    if hasattr(tweetCriteria, 'querySearch'):\n",
    "        urlGetData += tweetCriteria.querySearch\n",
    "\n",
    "    if hasattr(tweetCriteria, 'excludeWords'):\n",
    "        urlGetData += ' -'.join([''] + tweetCriteria.excludeWords)\n",
    "\n",
    "    if hasattr(tweetCriteria, 'username'):\n",
    "        if not hasattr(tweetCriteria.username, '__iter__'):\n",
    "            tweetCriteria.username = [tweetCriteria.username]\n",
    "\n",
    "        usernames_ = [u.lstrip('@') for u in tweetCriteria.username if u]\n",
    "        tweetCriteria.username = {u.lower() for u in usernames_ if u}\n",
    "\n",
    "        usernames = [' from:'+u for u in sorted(tweetCriteria.username)]\n",
    "        if usernames:\n",
    "            urlGetData += ' OR'.join(usernames)\n",
    "\n",
    "    if hasattr(tweetCriteria, 'within'):\n",
    "        if hasattr(tweetCriteria, 'near'):\n",
    "            urlGetData += ' near:\"%s\" within:%s' % (tweetCriteria.near, tweetCriteria.within)\n",
    "        elif hasattr(tweetCriteria, 'lat') and hasattr(tweetCriteria, 'lon'):\n",
    "            urlGetData += ' geocode:%f,%f,%s' % (tweetCriteria.lat, tweetCriteria.lon, tweetCriteria.within)\n",
    "\n",
    "    if hasattr(tweetCriteria, 'since'):\n",
    "        urlGetData += ' since:' + tweetCriteria.since\n",
    "\n",
    "    if hasattr(tweetCriteria, 'until'):\n",
    "        urlGetData += ' until:' + tweetCriteria.until\n",
    "\n",
    "    if hasattr(tweetCriteria, 'minReplies'):\n",
    "        urlGetData += ' min_replies:' + tweetCriteria.minReplies\n",
    "\n",
    "    if hasattr(tweetCriteria, 'minFaves'):\n",
    "        urlGetData += ' min_faves:' + tweetCriteria.minFaves\n",
    "\n",
    "    if hasattr(tweetCriteria, 'minRetweets'):\n",
    "        urlGetData += ' min_retweets:' + tweetCriteria.minRetweets\n",
    "\n",
    "    if hasattr(tweetCriteria, 'lang'):\n",
    "        urlLang = 'l=' + tweetCriteria.lang + '&'\n",
    "    else:\n",
    "        urlLang = ''\n",
    "    url = url % (urllib.parse.quote(urlGetData.strip()), urlLang, urllib.parse.quote(refreshCursor))\n",
    "    useragent = useragent or TweetManager.user_agents[0]\n",
    "\n",
    "    headers = [\n",
    "        ('Host', \"twitter.com\"),\n",
    "        ('User-Agent', useragent),\n",
    "        ('Accept', \"application/json, text/javascript, */*; q=0.01\"),\n",
    "        ('Accept-Language', \"en-US,en;q=0.5\"),\n",
    "        ('X-Requested-With', \"XMLHttpRequest\"),\n",
    "        ('Referer', url),\n",
    "        ('Connection', \"keep-alive\")\n",
    "    ]\n",
    "\n",
    "    if proxy:\n",
    "        opener = urllib.request.build_opener(urllib.request.ProxyHandler({'http': proxy, 'https': proxy}), urllib.request.HTTPCookieProcessor(cookieJar))\n",
    "    else:\n",
    "        opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookieJar))\n",
    "    opener.addheaders = headers\n",
    "\n",
    "    # 디버그 시 헤더 정보 출력 수정\n",
    "    if debug:\n",
    "        print(url)\n",
    "        # print('\\n'.join(h[0]+': '+h[1] for h in headers))\n",
    "\n",
    "    # HTTP Request 429 에러 방지\n",
    "    time.sleep(3)\n",
    "\n",
    "    try:\n",
    "        response = opener.open(url)\n",
    "        jsonResponse = response.read()\n",
    "\n",
    "    except TimeoutError as e:\n",
    "        print(\"Timeout error\")\n",
    "        print('sleep 30')\n",
    "        time.sleep(30)\n",
    "\n",
    "        # 이 부분 나중에 retry 매직 메소드 알아보기 / 강사님 질문\n",
    "        try:\n",
    "            response = opener.open(url)\n",
    "            jsonResponse = response.read()\n",
    "        except TimeoutError as e:\n",
    "            print(\"Timeout Error again.\")\n",
    "            pass\n",
    "\n",
    "    except Exception as e:\n",
    "        print(\"An error occured during an HTTP request:\", str(e))\n",
    "        print(\"Error URL:\", url)\n",
    "        print(\"Try to open in browser: https://twitter.com/search?q=%s&src=typd\" % urllib.parse.quote(urlGetData))\n",
    "\n",
    "        # 이 부분 나중에 확인하기\n",
    "        print(\"sleep 30!!\")\n",
    "        pass\n",
    "        # sys.exit()\n",
    "        \n",
    "\n",
    "    try:\n",
    "        s_json = jsonResponse.decode()\n",
    "    except:\n",
    "        print(\"Invalid response from Twitter\")\n",
    "        print(\"Error URL:\", url)\n",
    "        pass\n",
    "        # sys.exit()\n",
    "    else:\n",
    "        try:\n",
    "            dataJson = json.loads(s_json)\n",
    "        except:\n",
    "            print(\"Error parsing JSON: %s\" % s_json)\n",
    "            print(\"Error URL:\", url)\n",
    "        pass\n",
    "        # sys.exit()\n",
    "        \n",
    "#     if debug:\n",
    "#         print(s_json)\n",
    "#         print(\"---\\n\")\n",
    "\n",
    "\n",
    "    return dataJson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 날짜 다시 입력해야 할 때 에러 발생시키자.\n",
    "class NotValidEndDateError(Exception):\n",
    "    def __init__(self):\n",
    "        super().__init__('마지막 검색 날짜를 다시 설정하십시오.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GetOldTweets3 패키지의 setUntil 메소드는 마지막 날짜 포함하지 않으므로, 주의.\n",
    "def set_crawl_date(start_date, end_date):\n",
    "    \n",
    "    start_date = datetime.datetime.strptime(str(start_date), \"%Y%m%d\")\n",
    "    end_date = datetime.datetime.strptime(str(end_date), \"%Y%m%d\") - datetime.timedelta(days=1)\n",
    "    \n",
    "    if end_date == start_date:\n",
    "        raise NotValidEndDateError\n",
    "    \n",
    "    else:   \n",
    "        print(\"트윗 수집 날짜 설정: {0}부터 {1}까지\".format(start_date, end_date))    \n",
    "        return start_date.strftime(\"%Y-%m-%d\"), end_date.strftime(\"%Y-%m-%d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crawl_tweets(start_date, end_date, debug=False):    \n",
    "    got.manager.TweetManager.getJsonResponse = getJsonResponse # 함수에 이 부분 넣어주고\n",
    "    \n",
    "    print(\"========== 트윗 수집 시작: {0} ~ {1} ==========\".format(start_date, end_date))\n",
    "    start_time = time.time()\n",
    "    tweet_criteria = got.manager.TweetCriteria().setQuerySearch('Elon Musk')\\\n",
    "                                                .setSince(start_date)\\\n",
    "                                                .setUntil(end_date)\\\n",
    "                                                .setLang('en')\n",
    "    tweets = got.manager.TweetManager.getTweets(tweet_criteria, debug=debug)\n",
    "    \n",
    "    elapsed_time = time.time()-start_time\n",
    "    \n",
    "    print(\"수집 완료 : {}\".format(time.strftime(\"%H:%M:%S\", time.gmtime(elapsed_time))))\n",
    "    print(\"총 수집 트윗 개수 : {0}\".format(len(tweets)))\n",
    "    \n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results(tweet_data):\n",
    "    results = []\n",
    "    for tweet in tqdm(tweet_data):\n",
    "        results.append({'url': tweet.permalink,\n",
    "                        'date': tweet.date,\n",
    "                        'text': tweet.text,\n",
    "                        'user': tweet.username,\n",
    "                        'mentions': tweet.mentions,\n",
    "                        'retweets': tweet.retweets,\n",
    "                        'favorites': tweet.favorites,\n",
    "                        'hashtags': tweet.hashtags})\n",
    "        \n",
    "    return results        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_tweets(tweet_lists):\n",
    "    base_file_dir = \"tweets\"\n",
    "\n",
    "    if not os.path.exists(base_file_dir):\n",
    "        os.makedirs(base_file_dir)\n",
    "        \n",
    "    with open(f\"{base_file_dir}/tweets_{crawl_start}_{crawl_end}.csv\", \"a\", -1, encoding=\"utf-8\") as f:    \n",
    "        writer = csv.writer(f)\n",
    "        writer.writerow(['url', 'date', 'text', 'user', 'mentions', 'retweets', 'favorites', 'hashtags'])        \n",
    "        for tweet_list in tqdm(tweet_lists):\n",
    "            writer.writerow(list(tweet_list.values()))\n",
    "            \n",
    "    return "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "트윗 수집 날짜 설정: 2015-01-01 00:00:00부터 2015-01-02 00:00:00까지\n",
      "========== 트윗 수집 시작: 2015-01-01 ~ 2015-01-02 ==========\n"
     ]
    }
   ],
   "source": [
    "# 크롤링 진행\n",
    "crawl_start, crawl_end = set_crawl_date(20150101, 20150103)\n",
    "tweet_results = crawl_tweets(crawl_start, crawl_end)\n",
    "tweet_results_lists = get_results(tweet_results)\n",
    "save_tweets(tweet_results_lists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ver 2\n",
    "\n",
    "- date_range 추가: 날짜 입력 by 반복문 (날짜 exception class 없음)\n",
    "- url 설정 부분 간소화: 필요한 부분만 url 돌리도록 설정.\n",
    "- crawl 함수 변경: 검색어 변경."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# module import\n",
    "import GetOldTweets3 as got\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import sys\n",
    "import urllib\n",
    "import json\n",
    "from time import sleep\n",
    "import pandas as pd\n",
    "\n",
    "import datetime\n",
    "import time\n",
    "import os\n",
    "from random import uniform\n",
    "from tqdm import tqdm\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GOT 스태틱 메소드 수정\n",
    "def getJsonResponse(tweetCriteria, refreshCursor, cookieJar, proxy, useragent=None, debug=False):\n",
    "    url = \"https://twitter.com/i/search/timeline?\"\n",
    "\n",
    "    if not tweetCriteria.topTweets:\n",
    "        url += \"f=tweets&\"\n",
    "\n",
    "    url += (\"vertical=news&q=%s&src=typd&%s\"\n",
    "            \"&include_available_features=1&include_entities=1&max_position=%s\"\n",
    "            \"&reset_error_state=false\")\n",
    "\n",
    "    urlGetData = ''\n",
    "    \n",
    "    # url + query search, since, since, until, lang    \n",
    "    urlGetData += tweetCriteria.querySearch\n",
    "    urlGetData += ' since:' + tweetCriteria.since\n",
    "    urlGetData += ' until:' + tweetCriteria.until\n",
    "    urlLang = 'l=' + tweetCriteria.lang + '&'\n",
    "    \n",
    "    # url 설정\n",
    "    url = url % (urllib.parse.quote(urlGetData.strip()), urlLang, urllib.parse.quote(refreshCursor))\n",
    "    useragent = useragent or TweetManager.user_agents[0]\n",
    "\n",
    "    headers = [\n",
    "        ('Host', \"twitter.com\"),\n",
    "        ('User-Agent', useragent),\n",
    "        ('Accept', \"application/json, text/javascript, */*; q=0.01\"),\n",
    "        ('Accept-Language', \"en-US,en;q=0.5\"),\n",
    "        ('X-Requested-With', \"XMLHttpRequest\"),\n",
    "        ('Referer', url),\n",
    "        ('Connection', \"keep-alive\")\n",
    "    ]\n",
    "\n",
    "    if proxy:\n",
    "        opener = urllib.request.build_opener(urllib.request.ProxyHandler({'http': proxy, 'https': proxy}), urllib.request.HTTPCookieProcessor(cookieJar))\n",
    "    else:\n",
    "        opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookieJar))\n",
    "    opener.addheaders = headers        \n",
    "    \n",
    "    time.sleep(1) # HTTP Request 429 에러 방지.\n",
    "\n",
    "    ########################################## 에러 핸들링 수정 ##########################################\n",
    "    \n",
    "    # TimeOut 에러\n",
    "    try:\n",
    "        response = opener.open(url)\n",
    "        jsonResponse = response.read()\n",
    "    except TimeoutError as e:\n",
    "        \n",
    "        if debug: # 디버그 옵션\n",
    "            print(\"에러 url 주소:\", url) \n",
    "            \n",
    "        print(\"Timeout error\")\n",
    "        print(\"30초 정지\")\n",
    "        time.sleep(30)\n",
    "\n",
    "        # 한 번 더 시도\n",
    "        try:\n",
    "            response = opener.open(url)\n",
    "            jsonResponse = response.read()\n",
    "        except TimeoutError as e:\n",
    "            print(\"Timeout error again. 패스.\")\n",
    "            pass\n",
    "\n",
    "    # UrlParse 에러: 다시 시도 X\n",
    "    except Exception as e:\n",
    "        if debug: # 디버그 옵션\n",
    "            print(\"에러 url 주소:\", url)\n",
    "            \n",
    "        print(\"HTTP 요청 오류\", str(e))        \n",
    "        print(\"브라우저 오픈: https://twitter.com/search?q=%s&src=typd\" % urllib.parse.quote(urlGetData))\n",
    "        print(\"30초 정지\")\n",
    "        \n",
    "        pass\n",
    "\n",
    "    # Json 데이터 오류\n",
    "    try:\n",
    "        s_json = jsonResponse.decode()\n",
    "    except:\n",
    "        print(\"올바르지 못한 응답\")\n",
    "        if debug: # 디버그 옵션\n",
    "            print(\"에러 url 주소:\", url)\n",
    "        pass\n",
    "    else:\n",
    "        try:\n",
    "            dataJson = json.loads(s_json)\n",
    "        except: # json 데이터 파싱 오류\n",
    "            if debug: # 디버그 옵션\n",
    "                print(\"에러 url 주소:\", url)                \n",
    "            print(\"JSON: %s\" % s_json)\n",
    "        pass\n",
    "\n",
    "    return dataJson"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setUntil : 마지막 날짜 배제되므로 주의.\n",
    "def set_crawl_date(start_date, freq=1):    \n",
    "    end_date = start_date + datetime.timedelta(days=freq)\n",
    "    \n",
    "    # timestamp to string format\n",
    "    start_date = start_date.strftime(\"%Y-%m-%d\")\n",
    "    end_date = end_date.strftime(\"%Y-%m-%d\")\n",
    "    \n",
    "    # check\n",
    "    print(\"트윗 수집 날짜 설정: {0}부터 {1}까지\".format(start_date, end_date)) \n",
    "    \n",
    "    return start_date, end_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def crawl_tweets(start_date, end_date, query, lang='en', debug=True):    \n",
    "    '''\n",
    "    query: 검색할 트윗 검색어\n",
    "    lang: 검색할 트윗 언어\n",
    "    debug: 설정 시 에러 url 표시\n",
    "    '''\n",
    "    \n",
    "    print(\"========== 트윗 수집 시작: {0} ~ {1} ==========\".format(start_date, end_date))\n",
    "    start_time = time.time()\n",
    "    tweet_criteria = got.manager.TweetCriteria().setQuerySearch(query)\\\n",
    "                                                .setSince(start_date)\\\n",
    "                                                .setUntil(end_date)\\\n",
    "                                                .setLang(lang)\n",
    "    tweets = got.manager.TweetManager.getTweets(tweet_criteria, debug=debug)\n",
    "    \n",
    "    elapsed_time = time.time()-start_time\n",
    "    \n",
    "    print(\"수집 완료 : {}\".format(time.strftime(\"%H:%M:%S\", time.gmtime(elapsed_time))))\n",
    "    print(\"총 수집 트윗 개수 : {0}\".format(len(tweets)))\n",
    "    \n",
    "    return tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# got tweet 객체로부터 결과 추출\n",
    "def get_results(tweet_data):\n",
    "    results = []\n",
    "    for tweet in tqdm(tweet_data):\n",
    "        results.append({'url': tweet.permalink,\n",
    "                        'date': tweet.date,\n",
    "                        'text': tweet.text,\n",
    "                        'user': tweet.username,\n",
    "                        'mentions': tweet.mentions,\n",
    "                        'retweets': tweet.retweets,\n",
    "                        'favorites': tweet.favorites,\n",
    "                        'hashtags': tweet.hashtags})\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 추출한 결과 저장\n",
    "def save_tweets(tweet_lists, base_file_dir=\"tweets\"):\n",
    "    \n",
    "    if not os.path.exists(base_file_dir):\n",
    "        os.makedirs(base_file_dir)\n",
    "        \n",
    "    with open(f\"{base_file_dir}/tweets_{crawl_start}_{crawl_end}.csv\", \"a\", -1, encoding=\"utf-8\") as f:    \n",
    "        writer = csv.writer(f)\n",
    "        writer.writerow(['url', 'date', 'text', 'user', 'mentions', 'retweets', 'favorites', 'hashtags'])        \n",
    "        for tweet_list in tqdm(tweet_lists):\n",
    "            writer.writerow(list(tweet_list.values()))\n",
    "            \n",
    "    return "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "트윗 수집 날짜 설정: 2015-01-01부터 2015-01-02까지\n",
      "========== 트윗 수집 시작: 2015-01-01 ~ 2015-01-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 1654/1654 [00:00<00:00, 334379.85it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1654/1654 [00:00<00:00, 87261.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수집 완료 : 00:02:05\n",
      "총 수집 트윗 개수 : 1654\n",
      "트윗 수집 날짜 설정: 2015-02-01부터 2015-02-02까지\n",
      "========== 트윗 수집 시작: 2015-02-01 ~ 2015-02-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 530/530 [00:00<?, ?it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████| 530/530 [00:00<00:00, 75921.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수집 완료 : 00:00:44\n",
      "총 수집 트윗 개수 : 530\n",
      "트윗 수집 날짜 설정: 2015-03-01부터 2015-03-02까지\n",
      "========== 트윗 수집 시작: 2015-03-01 ~ 2015-03-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|████████████████████████████████████████████████████████████████████████████████████████| 526/526 [00:00<?, ?it/s]\n",
      "100%|█████████████████████████████████████████████████████████████████████████████| 526/526 [00:00<00:00, 65939.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수집 완료 : 00:00:39\n",
      "총 수집 트윗 개수 : 526\n",
      "트윗 수집 날짜 설정: 2015-04-01부터 2015-04-02까지\n",
      "========== 트윗 수집 시작: 2015-04-01 ~ 2015-04-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████████████████████████████████████████████████████████████████████████████████| 1659/1659 [00:00<?, ?it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1659/1659 [00:00<00:00, 69255.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수집 완료 : 00:02:06\n",
      "총 수집 트윗 개수 : 1659\n",
      "트윗 수집 날짜 설정: 2015-05-01부터 2015-05-02까지\n",
      "========== 트윗 수집 시작: 2015-05-01 ~ 2015-05-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|████████████████████████████████████████████████████████████████████████| 18055/18055 [00:00<00:00, 724151.65it/s]\n",
      "100%|████████████████████████████████████████████████████████████████████████| 18055/18055 [00:00<00:00, 127908.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수집 완료 : 00:22:33\n",
      "총 수집 트윗 개수 : 18055\n",
      "트윗 수집 날짜 설정: 2015-06-01부터 2015-06-02까지\n",
      "========== 트윗 수집 시작: 2015-06-01 ~ 2015-06-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████████████████████████████████████████████████████████████████████| 2032/2032 [00:00<00:00, 514073.57it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 2032/2032 [00:00<00:00, 88565.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수집 완료 : 00:02:30\n",
      "총 수집 트윗 개수 : 2032\n",
      "트윗 수집 날짜 설정: 2015-07-01부터 2015-07-02까지\n",
      "========== 트윗 수집 시작: 2015-07-01 ~ 2015-07-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████████████████████████████████████████████████████████████████████| 1092/1092 [00:00<00:00, 369012.24it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1092/1092 [00:00<00:00, 73008.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수집 완료 : 00:01:22\n",
      "총 수집 트윗 개수 : 1092\n",
      "트윗 수집 날짜 설정: 2015-08-01부터 2015-08-02까지\n",
      "========== 트윗 수집 시작: 2015-08-01 ~ 2015-08-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████████████████████████████████████████████████████████████████████| 1180/1180 [00:00<00:00, 141679.18it/s]\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1180/1180 [00:00<00:00, 65740.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수집 완료 : 00:01:29\n",
      "총 수집 트윗 개수 : 1180\n",
      "트윗 수집 날짜 설정: 2015-09-01부터 2015-09-02까지\n",
      "========== 트윗 수집 시작: 2015-09-01 ~ 2015-09-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|███████████████████████████████████████████████████████████████████████████| 1132/1132 [00:00<00:00, 88224.02it/s]\n",
      "100%|██████████████████████████████████████████████████████████████████████████| 1132/1132 [00:00<00:00, 169964.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "수집 완료 : 00:01:26\n",
      "총 수집 트윗 개수 : 1132\n",
      "트윗 수집 날짜 설정: 2015-10-01부터 2015-10-02까지\n",
      "========== 트윗 수집 시작: 2015-10-01 ~ 2015-10-02 ==========\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-f1408a24e77c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mdate\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdateRange\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mcrawl_start\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcrawl_end\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mset_crawl_date\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdate\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# freq 변경 가능: 며칠치 데이터 가져올 지 설정.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m     \u001b[0mtweet_results\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcrawl_tweets\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcrawl_start\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcrawl_end\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mquery\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'Elon Musk'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m     \u001b[0mtweet_results_lists\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_results\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtweet_results\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m     \u001b[0msave_tweets\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtweet_results_lists\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-11-e346b167632c>\u001b[0m in \u001b[0;36mcrawl_tweets\u001b[1;34m(start_date, end_date, query, lang, debug)\u001b[0m\n\u001b[0;32m     12\u001b[0m                                                 \u001b[1;33m.\u001b[0m\u001b[0msetUntil\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mend_date\u001b[0m\u001b[1;33m)\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m                                                 \u001b[1;33m.\u001b[0m\u001b[0msetLang\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlang\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 14\u001b[1;33m     \u001b[0mtweets\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgot\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmanager\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTweetManager\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetTweets\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtweet_criteria\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdebug\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     16\u001b[0m     \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mstart_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\cpu_env\\lib\\site-packages\\GetOldTweets3\\manager\\TweetManager.py\u001b[0m in \u001b[0;36mgetTweets\u001b[1;34m(tweetCriteria, receiveBuffer, bufferLength, proxy, debug)\u001b[0m\n\u001b[0;32m     63\u001b[0m             \u001b[0mactive\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m             \u001b[1;32mwhile\u001b[0m \u001b[0mactive\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 65\u001b[1;33m                 \u001b[0mjson\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTweetManager\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetJsonResponse\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtweetCriteria\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrefreshCursor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcookieJar\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mproxy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muser_agent\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdebug\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdebug\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     66\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'items_html'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     67\u001b[0m                     \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\cpu_env\\lib\\site-packages\\GetOldTweets3\\manager\\TweetManager.py\u001b[0m in \u001b[0;36mgetJsonResponse\u001b[1;34m(tweetCriteria, refreshCursor, cookieJar, proxy, useragent, debug)\u001b[0m\n\u001b[0;32m    342\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    343\u001b[0m             \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopener\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 344\u001b[1;33m             \u001b[0mjsonResponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    345\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    346\u001b[0m             \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"An error occured during an HTTP request:\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\cpu_env\\lib\\http\\client.py\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, amt)\u001b[0m\n\u001b[0;32m    468\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    469\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 470\u001b[1;33m                     \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_safe_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlength\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    471\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mIncompleteRead\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    472\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_close_conn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\cpu_env\\lib\\http\\client.py\u001b[0m in \u001b[0;36m_safe_read\u001b[1;34m(self, amt)\u001b[0m\n\u001b[0;32m    618\u001b[0m         \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    619\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[0mamt\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 620\u001b[1;33m             \u001b[0mchunk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mamt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mMAXAMOUNT\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    621\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mchunk\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    622\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mIncompleteRead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mb''\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ms\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mamt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\cpu_env\\lib\\socket.py\u001b[0m in \u001b[0;36mreadinto\u001b[1;34m(self, b)\u001b[0m\n\u001b[0;32m    587\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    588\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 589\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    590\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    591\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_timeout_occurred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\cpu_env\\lib\\ssl.py\u001b[0m in \u001b[0;36mrecv_into\u001b[1;34m(self, buffer, nbytes, flags)\u001b[0m\n\u001b[0;32m   1069\u001b[0m                   \u001b[1;34m\"non-zero flags not allowed in calls to recv_into() on %s\"\u001b[0m \u001b[1;33m%\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1070\u001b[0m                   self.__class__)\n\u001b[1;32m-> 1071\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnbytes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1072\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1073\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrecv_into\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbuffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnbytes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mflags\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\cpu_env\\lib\\ssl.py\u001b[0m in \u001b[0;36mread\u001b[1;34m(self, len, buffer)\u001b[0m\n\u001b[0;32m    927\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    928\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mbuffer\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 929\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbuffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    930\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    931\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sslobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# 크롤링 진행\n",
    "dateRange = pd.date_range(start='20150101', end='20200601', freq='MS') # freq 변경 가능: 며칠치 데이터 가져올 지 설정.\n",
    "\n",
    "for date in dateRange:\n",
    "    crawl_start, crawl_end = set_crawl_date(date)  # freq 변경 가능: 며칠치 데이터 가져올 지 설정.\n",
    "    tweet_results = crawl_tweets(crawl_start, crawl_end, query='Elon Musk', debug=False)\n",
    "    tweet_results_lists = get_results(tweet_results)\n",
    "    save_tweets(tweet_results_lists)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}