Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

We’re showing branches in this repository, but you can also compare across forks.

base fork: LOVEATWORK/storiesofswedishlapland
base: 46642ff981
...
head fork: LOVEATWORK/storiesofswedishlapland
compare: 08441b1831
  • 6 commits
  • 6 files changed
  • 0 commit comments
  • 1 contributor
2  .gitignore
View
@@ -1,2 +1,4 @@
.DS_Store
node_modules
+scraper/data
+scraper/config.coffee
8 scraper/config-defaults.coffee
View
@@ -0,0 +1,8 @@
+### copy me to config.coffee ###
+
+config = {}
+
+config.instagram_id = ''
+config.instagram_secret = ''
+
+module.exports = config
0  scraper/data/.keep
View
No changes.
98 scraper/index.coffee
View
@@ -0,0 +1,98 @@
+
+async = require 'async'
+config = require './config'
+instagram = require 'instagram-node-lib'
+{Model} = require './model'
+
+instagram.set 'client_id', config.instagram_id
+instagram.set 'client_secret', config.instagram_secret
+
+mapRange = (value, inputRange, outputRange=[0, 1]) ->
+ ((value - inputRange[0]) / (inputRange[1] - inputRange[0]) * (outputRange[1] - outputRange[0]) + outputRange[0])
+
+randomLocation = ->
+ return {
+ latitude: mapRange Math.random(), [0, 1], [-90, 90]
+ longitude: mapRange Math.random(), [0, 1], [-180, 180]
+ }
+
+resolveReach = (media, callback) ->
+ ### Resolve instagram *media* likes and comments to users with geodata. ###
+ # TODO: need a oauth server to access user info randomly create data for now
+ rv = []
+ for like in media.likes
+ rv.push
+ username: like.username
+ followers: ~~(Math.random() * 100)
+ location: randomLocation()
+ for comment in media.comments
+ rv.push
+ username: comment.from.username
+ followers: ~~(Math.random() * 100)
+ location: randomLocation()
+ callback null, rv
+
+
+# points of interest, posts collected 5km around point
+poi = [
+ {lat: 65.5843, lng: 22.1467, name: 'Luleå'}
+ {lat: 65.825282, lng: 21.665039, name: 'Bodn'}
+]
+
+class DataModel extends Model
+ constructor: (@id, @data) ->
+ serialize: -> {@id, @data}
+
+DataModel.deserialize = (id, data, callback) ->
+ callback null, new User id, data
+
+class Node extends DataModel
+ ### A node is a instagram post and its reach ###
+
+Node.new = (id, callback) ->
+ ### Fetch instagram post *id* and resolve likes to users/positions ###
+ async.waterfall [
+ (callback) ->
+ # fetch media
+ instagram.media.info
+ media_id: id
+ error: callback
+ complete: (data) -> callback null, data
+ (media, callback) ->
+ resolveReach media, (error, result) ->
+ # TODO: handle media without location
+ callback error,
+ date: media.created_time
+ location: media.location
+ images: media.images
+ reach: result
+ (result, callback) ->
+ callback null, new Node(id, result)
+ ], callback
+
+Node.nodesForLocation = (location, callback) ->
+ instagram.media.search
+ lat: location.lat
+ lng: location.lng
+ distance: 5000
+ complete: (data) ->
+ async.map data, (node, callback) ->
+ Node.load node.id, callback
+ , callback
+ error: (message) -> callback new Error message
+
+class User extends DataModel
+
+User.new = (id, callback) ->
+ instagram.users.recent
+ user_id: id
+ error: callback
+ complete: (data) ->
+ callback null, new User id, dat
+
+if require.main is module
+ poi.forEach (point) ->
+ Node.nodesForLocation point, (error, result) ->
+ throw error if error?
+ console.log "Reach nodes for #{ point.name }:"
+ console.dir result
56 scraper/model.coffee
View
@@ -0,0 +1,56 @@
+
+levelup = require 'levelup'
+path = require 'path'
+
+throwop = (error) -> throw error if error?
+
+safeGet = (db, key, callback) ->
+ db.get key, (error, result) ->
+ if error? and error.name isnt 'NotFoundError'
+ callback error
+ else
+ callback null, result or null
+
+class Model
+
+ constructor: (@id) ->
+
+ serialize: -> {@id}
+
+ save: (callback) ->
+ db = @constructor.db()
+ db.put @id, @serialize(), callback
+
+Model.dblocation = path.join __dirname, './data'
+
+Model.db = ->
+ if not @_db?
+ dbpath = path.join this.dblocation, this.name.toLowerCase()
+ console.log "Opening database at #{ dbpath }"
+ @_db = levelup dbpath,
+ keyEncoding: 'utf8'
+ valueEncoding: 'json'
+ process.on 'exit', => @_db.close()
+ return @_db
+
+Model.load = (id, ignoreCache, callback) ->
+ if arguments.length isnt 3
+ callback = arguments[1] or throwop
+ ignoreCache = false
+
+ safeGet @db(), id, (error, result) =>
+ return callback error if error?
+ if not result? or ignoreCache
+ this.new id, (error, result) ->
+ return callback error, result if error? or not result?
+ result.save (error) -> callback error, result
+ else
+ this.deserialize id, result, callback
+
+Model.deserialize = (id, data, callback) ->
+ throw new Error 'Not implemented'
+
+Model.new = (id, callback) ->
+ throw new Error 'Not implemented'
+
+module.exports = {Model}
12 scraper/package.json
View
@@ -0,0 +1,12 @@
+{
+ "name": "lapland-scraper",
+ "version": "1.0.0",
+ "private": true,
+ "dependencies": {
+ "async": "*",
+ "coffee-script": "*",
+ "express": "*",
+ "instagram-node-lib": "*",
+ "levelup": "*"
+ }
+}

No commit comments for this range

Something went wrong with that request. Please try again.