Merge branch 'cse'

Conflicts: main.py
nsmgr8 · Feb 24, 2010 · 5310b1e · 5310b1e
2 parents af67b72 + 8f2a7d9
commit 5310b1e
Showing 1 changed file with 93 additions and 11 deletions.
diff --git a/main.py b/main.py
@@ -20,25 +20,30 @@
 
 dseroot = "http://www.dsebd.org/"
 dselatest = dseroot + "latest_share_price_all.php"
-dsesanere = re.compile(r'<body[^>]*>')
-datere = re.compile(r'[a-zA-Z]{3}\s*\d{2},\s*\d{4}\s*at\s*\d{2}:\d{2}:\d{2}')
+
+cselatest = "http://www.csebd.com/trade/top.htm"
 
 fetch_error_message = 'Sorry, there was an error fetching data from main server.'
 
-time_key = 'timekey'
-data_key = 'csvdata'
-cache_time = 2 * 60
+time_key = 'dsedate'
+data_key = 'dsedata'
+cse_key = 'csedata'
+csedate_key = 'csedate'
+cache_time = 1 * 60
 
 class DSEHandler(webapp.RequestHandler):
 
+    dsesanere = re.compile(r'<body[^>]*>')
+    datere = re.compile(r'[a-zA-Z]{3}\s*\d{2},\s*\d{4}\s*at\s*\d{2}:\d{2}:\d{2}')
+
     def _get_time(self):
         last_update = memcache.get(time_key)
-        if last_update:
+        if last_update is not None:
             return last_update
 
         response = urlfetch.fetch(dseroot)
         if response.status_code == 200:
-            last_update = datere.search(response.content).group()
+            last_update = self.datere.search(response.content).group()
             last_update = last_update.replace(' ', '')
             last_update = datetime.datetime.strptime(last_update, "%b%d,%Yat"
                                                      "%H:%M:%S")
@@ -66,7 +71,7 @@ def get(self):
             return
 
         dsecontent = dseresult.content
-        dsecontent = dsesanere.sub('<body>', dsecontent)
+        dsecontent = self.dsesanere.sub('<body>', dsecontent)
 
         soup = BeautifulSoup(dsecontent)
         headtr = soup.body.table.tr.findAll('b')
@@ -83,8 +88,8 @@ def get(self):
         data = soup.body.table.findAll('tr')[1:]
         for row in data:
             row = row.findAll('td')[1:]
-            d = [row[0].a.contents[0],]
-            d.append(last_update)
+
+            d = [row[0].a.contents[0], last_update]
             for col in row[1:]:
                 d.append(col.find(text=True))
 
@@ -101,10 +106,87 @@ def get(self):
         logging.info('fetched real data')
 
 
+class CSEHandler(webapp.RequestHandler):
+
+    csedatere = re.compile(r'Date: '
+                           r'([a-zA-Z]{3})\s*(\d{2})\s*(\d{4})\s*(\d{1,2}):(\d{1,2})(AM|PM)')
+    csedatare = re.compile(r'^\s*(\w{3,5}).*?'
+                           '(\d+\.{0,1}\d*)\s+'
+                           '(\d+\.{0,1}\d*)\s+'
+                           '(\d+\.{0,1}\d*)\s+'
+                           '(\d+\.{0,1}\d*)\s+'
+                           '(\d+\.{0,1}\d*)\s+'
+                           '(-{0,1}\d+\.{0,1}\d*)\s+'
+                           '(\d+\.{0,1}\d*)\s+'
+                           '(\d+\.{0,1}\d*)\s+', re.MULTILINE)
+
+    def get(self):
+        last_update = memcache.get(csedate_key)
+        csvdata = memcache.get(cse_key)
+        if csvdata and last_update is not None:
+            csvname = 'cse-%s.csv' % last_update.isoformat()
+
+            self.response.headers.add_header('content-disposition',
+                                             'attachment', filename=csvname)
+            self.response.headers['Content-Type'] = 'text/csv'
+            self.response.out.write(csvdata)
+
+            logging.info('retrieved from cache')
+            return
+
+        cseresult = urlfetch.fetch(cselatest)
+        if not cseresult.status_code == 200:
+            self.response.out.write(fetch_error_message)
+            return
+
+        content = cseresult.content
+        soup = BeautifulSoup(content)
+        precontents = soup.body.findAll('pre')
+
+        sdate = list(self.csedatere.search(precontents[0].contents[0]).groups())
+        for i in [1, 3, 4]:
+            if len(sdate[i]) == 1:
+                sdate[i] = '0' + sdate[i]
+        sdate = ' '.join(sdate)
+        last_update = datetime.datetime.strptime(sdate,
+                                                 '%b %d %Y %I %M %p')
+        csvname = 'cse-%s.csv' % last_update.isoformat()
+
+        output = StringIO.StringIO()
+        csvfile = csv.writer(output)
+        heads = ['Company', 'Date Time', 'Open', 'High', 'Low', 'Close',
+                 'Prev. Close', 'Difference', 'Trades', 'Volume',]
+        csvfile.writerow(heads)
+
+        contents = precontents[1].contents[0].split('\n')
+        for content in contents:
+            try:
+                data = list(self.csedatare.search(content).groups())
+                if data[-1] == '0':
+                    continue
+                data.insert(1, last_update)
+                csvfile.writerow(data)
+            except:
+                pass
+
+        self.response.headers.add_header('content-disposition',
+                                         'attachment', filename=csvname)
+        self.response.headers['Content-Type'] = 'text/csv'
+
+        csvdata = output.getvalue()
+        output.close()
+
+        self.response.out.write(csvdata)
+
+        memcache.set(cse_key, csvdata, cache_time)
+        memcache.set(csedate_key, last_update, cache_time)
+
+        logging.info('fetched real data')
 
 
 def main():
-  application = webapp.WSGIApplication([('/', DSEHandler)],
+  application = webapp.WSGIApplication([('/dse', DSEHandler),
+                                        ('/cse', CSEHandler)],
                                        debug=True)
   util.run_wsgi_app(application)