added BoersennewsEndpoint

lib/importer.js

@@ -4,19 +4,27 @@
 
     Equity:
     ---------
-    An equity object must contain the following attributes:
+    An equity object should contain the following attributes:
     {
         isin: The ISIN unique equity id
         name: The name of the equity
-        currentPrice: The current price (as realtime as possible)
+        latestPrice: The current price (as realtime as possible)
         monthlyPrices: [...] Array of monthly prices of the first day in each month of the last 12 months starting with the price at the beginning of the current month.
         dailyPrices: [...] Array of daily prices at the beginning of each day in the last 30 trading(!) days starting with the price of the last ended trading day.
+        latestFacts: {
+            year: The year of the facts
+            pbRatio: P/B ratio (german: KBV)
+            peRatio: P/E ratio (german: KGV)
+        }
+        historicFacts: [{},...] Array containing the same objects as latestFacts but with facts of the last years, latestFacts is not included in this list
     }
 
     The following attributes are optional:
     {
         wkn: The WKN unique equity id
     }
+
+    If some values cannot be retrieved they will be null.
 */
 
 

src/lib/endpoints/boersennews.coffee

@@ -0,0 +1,120 @@
+Endpoint = require '../endpoint.js'
+Crawler = require('crawler').Crawler
+
+###
+Fetch stock data from boersennews.de
+###
+class BoersennewsEndpoint extends Endpoint
+    constructor: ->
+        this.baseUrl = 'http://www.boersennews.de'
+        this.crawler = new Crawler
+            forceUTF8: true
+            #debug: true
+            maxConnections: 10
+            # Means if we request the same URI twice it is not crawled again.
+            # But callback will be called!
+            # This means you have to create a new endpoint if you want to flush the cash!
+            cache: true
+
+    ###
+    Retrieve a single equity by ISIN
+    ###
+    getEquityByIsin: (isin, cb) ->
+        isin = isin.toLowerCase()
+
+        # search supports GET requests
+        url = 'http://www.boersennews.de/markt/search/simple/key/' + encodeURIComponent(isin) + '/category/sto'
+        this.crawler.queue [
+            uri: url
+            callback: (error, result, $) =>
+                if error
+                    cb new Error('Could not load boersennews.de!'), null
+                    return
+
+                url = null
+                for row in $('table.tabList.lastLine tr:has(td)')
+                    tds = $(row).find('td')
+                    if $(tds[2]).text().toLowerCase() == isin
+                        url = this.baseUrl + $(tds[0]).find('a').attr('href').replace("/profile", "/fundamental")
+                        break
+
+                if not url
+                    cb null, null
+                    return
+
+                this.crawlEquity url, cb
+            ]
+        this
+
+    ###
+    Crawl an equity by its URL on boersennews.de
+    This methods crawls only stock facts page.
+
+    @param {String} url URL of an equity "Fundamentale Daten" page on boersennews.de
+    @param {Function} cb Callback is called with Error|null and Object, the crawled equity
+    ###
+    crawlEquity: (url, cb) ->
+        this.crawler.queue [
+            uri: url
+            callback: (error, result, $) =>
+                if error
+                    cb new Error('Could not load boersennews.de!'), null
+                    return
+
+                if not this._isValidEquityUrl(result.request.href)
+                    cb new Error('Not a valid equity URL!'), null
+                    return
+
+                equity =
+                    name: $('h2').text().replace(' Fundamentale Daten', '')
+
+                # WKN and ISIN
+                matches = /ISIN:\s([^\s]+)\s\|\sWKN:\s([^\s]+)/.exec($('.instrumentInfos h3').text())
+                equity.isin = matches[1]
+                equity.wkn = matches[2]
+
+                # find available years
+                colsMap = {}
+                for col, i in $('.tabList tr').eq(0).find('th')
+                    if /^\d+$/.test $(col).text()
+                        colsMap[parseInt $(col).text()] = i
+
+                # create facts per year with null values
+                factsPerYear = {}
+                for year, i in colsMap
+                    factsPerYear[year] =
+                        year: year
+                        pbRatio: null
+                        peRatio: null
+
+                # helper function
+                fillFacts = (cols, key) =>
+                    for year, colIndex in colsMap
+                        # parse number
+                        matches = /([0-9,\.-]+)/.exec(cols.eq(colIndex).text())
+                        if matches and matches[1] != '-'
+                            num = parseFloat matches[1].replace('.', '').replace(',', '.')
+                            factsPerYear[year][key] = num
+
+                #iterate over table rows and fill as much facts as possible
+                for row in $('.tabList tr:has(td)')
+                    cols = $(row).find('td')
+                    if /^KGV/.test cols.eq(0).text()
+                        fillFacts cols, 'peRatio'
+                        continue
+
+                facts = []
+                for obj in factsPerYear
+                    facts.push obj
+                facts.sort (a,b) -> b.year - a.year
+
+                equity.latestFacts = facts.shift()
+                equity.historicFacts = facts
+
+                cb null, equity
+            ]
+        this
+
+    _isValidEquityUrl: (url) -> /^http:\/\/www\.boersennews\.de\/markt\/aktien\/[^\/]+\/[^\/]+\/fundamental$/.test(url)
+
+module.exports = BoersennewsEndpoint

src/lib/endpoints/finanzennet.coffee

@@ -35,7 +35,7 @@ class FinanzennetEndpoint extends Endpoint
                 if this._isValidIndexUrl(result.request.href)
                     this.crawlIndex(url, cb)
                 else
-                    url = $('.main').first().find('table tr').eq(1).find('a').attr('href')
+                    url = $('.main table tr').eq(1).find('a').attr('href')
                     if not url
                         cb(null, null)
                     else
@@ -69,7 +69,7 @@ class FinanzennetEndpoint extends Endpoint
                 if this._isValidEquityUrl result.request.href
                     this.crawlEquity url, stockMarket, cb
                 else
-                    url = $('.main').first().find('table tr').eq(1).find('a').attr('href')
+                    url = $('.main table tr').eq(1).find('a').attr('href')
                     if not url
                         cb null, null
                     else
@@ -162,7 +162,7 @@ class FinanzennetEndpoint extends Endpoint
                 equity.isin = matches[2]
 
                 # current price
-                equity.currentPrice = parseFloat($('.pricebox .content table').eq(0).find('th:first-child').text().replace('.', '').replace(',','.'))
+                equity.latestPrice = parseFloat($('.pricebox .content table').eq(0).find('th:first-child').text().replace('.', '').replace(',','.'))
 
                 # find finanzen.net equity ID
                 finanzennetId = null
@@ -254,7 +254,7 @@ class FinanzennetEndpoint extends Endpoint
                     url: result.request.href
 
                 # current price
-                index.currentPrice = parseFloat($('.pricebox .content table').eq(0).find('th:first-child').text().replace('.', '').replace(',','.'))
+                index.latestPrice = parseFloat($('.pricebox .content table').eq(0).find('th:first-child').text().replace('.', '').replace(',','.'))
 
                 now = new Date
                 this.crawler.queue [