2019-02-10_tepco/denki_kakeibo_every30min.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import configparser\n",
    "import json\n",
    "import requests\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "config = configparser.ConfigParser()\n",
    "config.read('.password', 'UTF-8')\n",
    " \n",
    "username = config.get(\"settings\", 'username')\n",
    "password = config.get(\"settings\", 'password')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "session = requests.Session()\n",
    "\n",
    "# くらしtepcoからでんき家計簿にログイン\n",
    "param = {\n",
    "        'ACCOUNTUID': username,\n",
    "        'PASSWORD': password,\n",
    "        'HIDEURL': '/ls/pf/ja/pco/mypage/redirect-sso.page?sitekbn=kakeibo',\n",
    "        'LOGIN': 'EUAS_LOGIN',\n",
    "        }\n",
    "\n",
    "header = {\n",
    "        'Referer': 'https://www.kakeibo.tepco.co.jp/dk/com/menu/'\n",
    "        }\n",
    "\n",
    "login = session.post(\n",
    "        'https://www.kurashi.tepco.co.jp/kpf-login', data=param, headers=header)\n",
    "login.encoding = login.apparent_encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***my officeCd***\n",
      "***my visitNum***\n",
      "amount\n",
      "/dk/com/menu/goElectricUsageAmount\n"
     ]
    }
   ],
   "source": [
    "# 月毎データページへの移動リンクで発火するjs関数から、必要な情報を抜き出す\n",
    "html = BeautifulSoup(login.text, \"html.parser\")\n",
    "link_js = html.find(id=\"frame3\").find(class_=\"box01 box firstBox\").find(\"a\")[\"onclick\"]\n",
    "#print(link_js)\n",
    "key_officeCd = link_js.split(\"'\")[3]\n",
    "key_visitNum = link_js.split(\"'\")[5]\n",
    "key_display = link_js.split(\"'\")[7]\n",
    "post_to = link_js.split(\"'\")[17]\n",
    "\n",
    "#print(key_officeCd)\n",
    "print(\"***my officeCd***\")\n",
    "#print(key_visitNum)\n",
    "print(\"***my visitNum***\")\n",
    "print(key_display)\n",
    "print(post_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 月毎データの表示ページ に移動\n",
    "param = {\n",
    "        'key.officeCd': key_officeCd,\n",
    "        'key.visitNum': key_visitNum,\n",
    "        'key.display': key_display\n",
    "}\n",
    "\n",
    "header = {\n",
    "        'Referer':login.url\n",
    "        }\n",
    "\n",
    "data_page = session.post(\n",
    "        'https://www.kakeibo.tepco.co.jp'+post_to, data=param, headers=header)\n",
    "data_page.encoding = data_page.apparent_encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***my officeCd***\n",
      "***my visitNum***\n",
      "/dk/syo/electricUsageAmount/goElectricUsage30minGraph\n"
     ]
    }
   ],
   "source": [
    "# 30分毎データページへの移動リンクで発火するjs関数から、必要な情報を抜き出す\n",
    "html = BeautifulSoup(data_page.text, \"html.parser\")\n",
    "link_js = html.find(id=\"motion_area1\").find(\"a\")[\"onclick\"]\n",
    "#print(link_js)\n",
    "\n",
    "key_officeCd = link_js.split(\"'\")[3]\n",
    "key_visitNum = link_js.split(\"'\")[5]\n",
    "post_to = link_js.split(\"'\")[13]\n",
    "\n",
    "print(\"***my officeCd***\")\n",
    "#print(key_officeCd)\n",
    "print(\"***my visitNum***\")\n",
    "#print(key_visitNum)\n",
    "print(post_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 30分毎データの表示ページ に移動\n",
    "param = {\n",
    "        'officeVisitCd': \"%s/%s\"%(key_officeCd, key_visitNum),\n",
    "        'key.officeCd': key_officeCd,\n",
    "        'key.visitNum': key_visitNum\n",
    "}\n",
    "\n",
    "header = {\n",
    "        'Referer':data_page.url\n",
    "        }\n",
    "\n",
    "data_page_30min = session.post(\n",
    "        'https://www.kakeibo.tepco.co.jp'+post_to, data=param, headers=header)\n",
    "data_page_30min.encoding = data_page_30min.apparent_encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 30分値表示ページから、数値データを抜き出す\n",
    "def getData(html_text):\n",
    "    def getCDATA(text):\n",
    "        ret = \"\"\n",
    "        isInCDATAArea = False\n",
    "        for l in text.split(\"\\n\"):\n",
    "            l = l.strip()\n",
    "            if (l == \"// <![CDATA[\"):\n",
    "                isInCDATAArea = True\n",
    "            elif (l.strip() == \"// ]]>\"):\n",
    "                isInCDATAArea = False\n",
    "        \n",
    "            if(isInCDATAArea):\n",
    "                ret += l + \"\\n\"\n",
    "        \n",
    "        return ret\n",
    "    \n",
    "    d = {}\n",
    "    function = \"\"\n",
    "    for l in getCDATA(html_text).split(\"\\n\"):\n",
    "        \n",
    "        if(l.startswith(\"function\")):\n",
    "            function = l.split(\"function\")[1].split(\"()\")[0].strip()\n",
    "            if(function.startswith(\"vbar_usage_grp\")):\n",
    "                d[function] = {}\n",
    "            \n",
    "        if(function.startswith(\"vbar\")):\n",
    "            if(l.startswith(\"var items = \")):\n",
    "                d[function][\"items\"] = json.loads(l.split(\"=\")[-1].strip(\";\"))\n",
    "    \n",
    "    html = BeautifulSoup(html_text, \"html.parser\")\n",
    "    date_str = html.find(id=\"motion_area1\").find_all(\"td\")[1].contents[0]\n",
    "    \n",
    "    date = datetime.datetime.strptime(date_str.split(\"\\u3000\")[0], '%a %b %d %H:%M:%S JST %Y')\n",
    "    \n",
    "    return (date, d[\"vbar_usage_grp\"][\"items\"][0][1:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 前日移動に必要な情報\n",
    "def getParamsToMovePrev(html_text):\n",
    "    html = BeautifulSoup(html_text, \"html.parser\")\n",
    "    link_js = html.find(id=\"motion_area1\").find(\"a\")[\"onclick\"]\n",
    "    #print(link_js)\n",
    "\n",
    "    key_d = link_js.split(\"'\")[3]\n",
    "    key_date = link_js.split(\"'\")[5]\n",
    "    post_to = link_js.split(\"'\")[13]\n",
    "\n",
    "    return(key_d, key_date, post_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 前日の30分データ表示ページ を取得\n",
    "def getPrev(key_d, key_date, post_to):\n",
    "    param = {\n",
    "        'key.d': key_d,\n",
    "        'key.date': key_date\n",
    "    }\n",
    "\n",
    "    header = {\n",
    "        'Referer':data_page.url\n",
    "    }\n",
    "\n",
    "    data_page_30min = session.post(\n",
    "            'https://www.kakeibo.tepco.co.jp'+post_to, data=param, headers=header)\n",
    "    data_page_30min.encoding = data_page_30min.apparent_encoding\n",
    "    \n",
    "    return data_page_30min.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#取得可能な30分値データを、全て取得\n",
    "\n",
    "current_page = data_page_30min.text\n",
    "\n",
    "with open(\"./usage30min.csv\", \"w\") as f:\n",
    "\n",
    "    while True:\n",
    "        d, date, post_to = getParamsToMovePrev(current_page)\n",
    "        current_page = getPrev(d, date, post_to)\n",
    "        data = getData(current_page)\n",
    "        if(len(data[1]) != 48):\n",
    "            break\n",
    "    \n",
    "        f.write(data[0].strftime('%Y-%m-%d'))\n",
    "        for usage in data[1]:\n",
    "            f.write(\" ,\" + str(usage))\n",
    "        f.write(\"\\n\")\n",
    "        f.flush()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}