Fix out of memory errors by adding "filesInDatabase" option

Add filesInDatabase option which can be disabled to put uploaded files on-disk only - applies to both minidumps and symbol files.
Also add maximum uploaded file size limit.

README.md

@@ -111,6 +111,8 @@ customFields:
     - name: 'customfile2'
   params: ['customparam']
 dataDir: '/home/myuser/.simple-breakpad-server'
+fileMaxUploadSize: 100000000
+filesInDatabase: true
 ```
 
 ### Database configuration
@@ -129,6 +131,12 @@ For now, if you change this configuration after the database is initialized, you
 
 Simple breakpad server caches symbols on the disk within the directory specified by `dataDir`. The default location is `$HOME/.simple-breakpad-server`.
 
+## Uploaded Files
+
+By default, there is no enforced limit to uploaded file size (limited by Node.js heap size and database size), and uploaded files (minidumps or custom files) are stored directly in the database.  The maximum allowed file size can be specified with `fileMaxUploadSize` (in bytes).
+
+The server can be directed to store all uploaded files on disk (instead of in the database) with `filesInDatabase: false`, however the dumps may be unable to be read after switching, so the database should be recreated (manually delete the `database.sqlite` file from your data directory).  Old symbols files (already on disk) should still work fine after changing this setting, even if they don't show up in the web interface's Symfiles list.  Note: if `filesInDatabase` is set to `false`, and you are doing backups, you should back up your entire data directory (or, at least, the symbols/ directory) in addition to your database file.
+
 ## Contributing
 
 Simple Breakpad Server is a work in progress and there is a lot to do. Send pull requests and issues on the project's [Github page](https://github.com/acrisci/simple-breakpad-server).

src/app.coffee

@@ -15,6 +15,7 @@ busboy = require 'connect-busboy'
 streamToArray = require 'stream-to-array'
 Sequelize = require 'sequelize'
 addr = require 'addr'
+fs = require 'fs-promise'
 
 crashreportToApiJson = (crashreport) ->
   json = crashreport.toJSON()
@@ -31,16 +32,15 @@ crashreportToViewJson = (report) ->
     id: report.id
     props: {}
 
-  for name, value of Crashreport.attributes
-    if value.type instanceof Sequelize.BLOB
-      fields.props[name] = { path: "/crashreports/#{report.id}/files/#{name}" }
-
   json = report.toJSON()
   for k,v of json
     if k in hidden
       # pass
+    else if config.get("customFields:filesById:#{k}")
+      # a file
+      fields.props[k] = { path: "/crashreports/#{report.id}/files/#{k}" }
     else if Buffer.isBuffer(json[k])
-      # already handled
+      # shouldn't happen, should hit line above
     else if k == 'created_at'
       # change the name of this key for display purposes
       fields.props['created'] = moment(v).fromNow()
@@ -51,11 +51,11 @@ crashreportToViewJson = (report) ->
 
   return fields
 
-symfileToViewJson = (symfile) ->
+symfileToViewJson = (symfile, contents) ->
   hidden = ['id', 'updated_at', 'contents']
   fields =
     id: symfile.id
-    contents: symfile.contents
+    contents: contents
     props: {}
 
   json = symfile.toJSON()
@@ -77,7 +77,23 @@ symfileToViewJson = (symfile) ->
 db.sync()
   .then ->
     Symfile.findAll().then (symfiles) ->
-      Promise.all(symfiles.map((s) -> Symfile.saveToDisk(s))).then(run)
+      pruneSymfilesFromDB = not config.get('filesInDatabase')
+      # TODO: This is really, really slow when you have a lot of symfiles, and
+      #   config.get('filesInDatabase') is true - only write those which do not
+      #   already exist on disk?  User can delete the on-disk cache if needed.
+      Promise.all(symfiles.map((s) -> Symfile.saveToDisk(s, pruneSymfilesFromDB)))
+  .then ->
+    console.log 'Symfile loading finished'
+    if Symfile.didPrune
+      # One-time vacuum of sqllite data to free up all of the data that was just deleted
+      console.log 'One-time compacting and syncing database after prune...'
+      db.query('VACUUM').then ->
+        db.sync().then ->
+          console.log 'Database compaction finished'
+    else
+      return
+  .then ->
+    run()
   .catch (err) ->
     console.error err.stack
     process.exit 1
@@ -120,7 +136,11 @@ run = ->
     console.trace err
     res.status(500).send "Bad things happened:<br/> #{err.message || err}"
 
-  breakpad.use(busboy())
+  breakpad.use(busboy(
+    limits:
+      fileSize: config.get 'fileMaxUploadSize'
+  ))
+  lastReportId = 0
   breakpad.post '/crashreports', (req, res, next) ->
     props = {}
     streamOps = []
@@ -129,18 +149,32 @@ run = ->
     # Fixed list of just localhost as trusted reverse-proxy, we can add
     #   a config option if needed
     props.ip = addr(req, ['127.0.0.1', '::ffff:127.0.0.1'])
+    reportUploadGuid = moment().format('YYYY-MM-DD.HH.mm.ss') + '.' +
+      process.pid + '.' + (++lastReportId)
 
     req.busboy.on 'file', (fieldname, file, filename, encoding, mimetype) ->
-      streamOps.push streamToArray(file).then((parts) ->
-        buffers = []
-        for i in [0 .. parts.length - 1]
-          part = parts[i]
-          buffers.push if part instanceof Buffer then part else new Buffer(part)
-
-        return Buffer.concat(buffers)
-      ).then (buffer) ->
+      if config.get('filesInDatabase')
+        streamOps.push streamToArray(file).then((parts) ->
+          buffers = []
+          for i in [0 .. parts.length - 1]
+            part = parts[i]
+            buffers.push if part instanceof Buffer then part else
+              new Buffer(part)
+
+          return Buffer.concat(buffers)
+        ).then (buffer) ->
+          if fieldname of Crashreport.attributes
+            props[fieldname] = buffer
+      else
+        # Stream file to disk, record filename in database
         if fieldname of Crashreport.attributes
-          props[fieldname] = buffer
+          saveFilename = path.join reportUploadGuid, fieldname
+          props[fieldname] = saveFilename
+          saveFilename = path.join config.getUploadPath(), saveFilename
+          fs.mkdirs(path.dirname(saveFilename)).then ->
+            file.pipe fs.createWriteStream(saveFilename)
+        else
+          file.close()
 
     req.busboy.on 'field', (fieldname, val, fieldnameTruncated, valTruncated) ->
       if fieldname == 'prod'
@@ -245,13 +279,18 @@ run = ->
 
       if 'raw' of req.query
         res.set 'content-type', 'text/plain'
-        res.send(symfile.contents.toString())
-        res.end()
+        if symfile.contents?
+          res.send(symfile.contents.toString())
+          res.end()
+        else
+          fs.createReadStream(Symfile.getPath(symfile)).pipe(res)
+
       else
-        res.render 'symfile-view', {
-          title: 'Symfile'
-          symfile: symfileToViewJson(symfile)
-        }
+        Symfile.getContents(symfile).then (contents) ->
+          res.render 'symfile-view', {
+            title: 'Symfile'
+            symfile: symfileToViewJson(symfile, contents)
+          }
 
   breakpad.get '/crashreports/:id', (req, res, next) ->
     Crashreport.findById(req.params.id).then (report) ->
@@ -281,20 +320,38 @@ run = ->
 
   breakpad.get '/crashreports/:id/files/:filefield', (req, res, next) ->
     # download the file for the given id
+    field = req.params.filefield
+    if !config.get("customFields:filesById:#{field}")
+      return res.status(404).send 'Crash report field is not a file'
+
     Crashreport.findById(req.params.id).then (crashreport) ->
       if not crashreport?
         return res.status(404).send 'Crash report not found'
 
-      field = req.params.filefield
       contents = crashreport.get(field)
 
-      if not Buffer.isBuffer(contents)
-        return res.status(404).send 'Crash report field is not a file'
-
       # Find appropriate downloadAs file name
       filename = config.get("customFields:filesById:#{field}:downloadAs") || field
       filename = filename.replace('{{id}}', req.params.id)
 
+      if !config.get('filesInDatabase')
+        # If this is a string, or a string stored as a blob in an old database,
+        # stream the on-disk file instead
+        onDiskFilename = contents
+        if Buffer.isBuffer(contents)
+          if contents.length > 128
+            # Large, must be an old actual dump stored in the database
+            onDiskFilename = null
+          else
+            onDiskFilename = contents.toString('utf8')
+        if onDiskFilename
+          # stream
+          res.setHeader('content-disposition', "attachment; filename=\"#{filename}\"")
+          return fs.createReadStream(path.join(config.getUploadPath(), onDiskFilename)).pipe(res)
+
+      if not Buffer.isBuffer(contents)
+        return res.status(404).send 'Crash report field is an unknown type'
+
       res.setHeader('content-disposition', "attachment; filename=\"#{filename}\"")
       res.send(contents)
 

src/config.coffee

@@ -35,6 +35,8 @@ nconf.defaults
     files: []
     params: []
   dataDir: SBS_HOME
+  filesInDatabase: true
+  fileMaxUploadSize: Infinity
 
 # Post-process custom files and params
 customFields = nconf.get('customFields')
@@ -71,6 +73,7 @@ for field, idx in customFields.params
 nconf.set('customFields', customFields)
 
 nconf.getSymbolsPath = -> path.join(nconf.get('dataDir'), 'symbols')
+nconf.getUploadPath = -> path.join(nconf.get('dataDir'), 'uploads')
 
 fs.mkdirsSync(nconf.getSymbolsPath())
 

src/model/crashreport.coffee

@@ -29,13 +29,30 @@ for field in customFields.params
   schema[field.name] = Sequelize.STRING
 
 for field in customFields.files
-  schema[field.name] = Sequelize.BLOB
+  schema[field.name] = if config.get('filesInDatabase') then Sequelize.BLOB else Sequelize.STRING
 
 Crashreport = sequelize.define('crashreports', schema, options)
 
 Crashreport.getStackTrace = (record, callback) ->
   return callback(null, cache.get(record.id)) if cache.has record.id
 
+  if !config.get('filesInDatabase')
+    # If this is a string, or a string stored as a blob in an old database,
+    # just use the on-disk file instead
+    onDiskFilename = record.upload_file_minidump
+    if Buffer.isBuffer(record.upload_file_minidump)
+      if record.upload_file_minidump.length > 128
+        # Large, must be an old actual dump stored in the database
+        onDiskFilename = null
+      else
+        onDiskFilename = record.upload_file_minidump.toString('utf8')
+    if onDiskFilename
+      # use existing file, do not delete when done!
+      use_filename = path.join(config.getUploadPath(), onDiskFilename)
+      return minidump.walkStack use_filename, [symbolsPath], (err, report) ->
+        cache.set record.id, report unless err?
+        callback err, report
+
   tmpfile = tmp.fileSync()
   fs.writeFile(tmpfile.name, record.upload_file_minidump).then ->
     minidump.walkStack tmpfile.name, [symbolsPath], (err, report) ->

src/model/symfile.coffee

@@ -35,19 +35,54 @@ options =
 
 Symfile = sequelize.define('symfiles', schema, options)
 
-Symfile.saveToDisk = (symfile) ->
+Symfile.getPath = (symfile) ->
   symfileDir = path.join(symbolsPath, symfile.name, symfile.code)
-  fs.mkdirs(symfileDir).then ->
-    # From https://chromium.googlesource.com/breakpad/breakpad/+/master/src/processor/simple_symbol_supplier.cc#179:
-    # Transform the debug file name into one ending in .sym.  If the existing
-    #   name ends in .pdb, strip the .pdb.  Otherwise, add .sym to the non-.pdb
-    #   name.
-    symbol_name = symfile.name
-    if path.extname(symbol_name).toLowerCase() == '.pdb'
-      symbol_name = symbol_name.slice(0, -4)
-    symbol_name += '.sym'
-    filePath = path.join(symfileDir, symbol_name)
-    fs.writeFile(filePath, symfile.contents)
+  # From https://chromium.googlesource.com/breakpad/breakpad/+/master/src/processor/simple_symbol_supplier.cc#179:
+  # Transform the debug file name into one ending in .sym.  If the existing
+  #   name ends in .pdb, strip the .pdb.  Otherwise, add .sym to the non-.pdb
+  #   name.
+  symbol_name = symfile.name
+  if path.extname(symbol_name).toLowerCase() == '.pdb'
+    symbol_name = symbol_name.slice(0, -4)
+  symbol_name += '.sym'
+  path.join(symfileDir, symbol_name)
+
+Symfile.saveToDisk = (symfile, prune) ->
+  filePath = Symfile.getPath(symfile)
+
+  # Note: this code will migrate symbol files between filesInDatabase: true/false,
+  #   or from an older version where filesInDatabase: true only referred to
+  #   dump files, however dump files have no similar migration - the database
+  #   must be wiped, so this is only actually useful for upgrading versions,
+  #   not switching between filesInDatabase modes, though it can be used
+  #   to import/export symbol files if you don't care about old dumps.
+
+  if not symfile.contents
+    if not prune
+      # If at startup, and the option was set back to "filesInDatabase", read them back from disk?
+      return fs.exists(filePath).then (exists) ->
+        if (exists)
+          console.log "Restoring contents to database from symfile #{symfile.id}, #{filePath}"
+          fs.readFile(filePath, 'utf8').then (contents) ->
+            Symfile.didPrune = true
+            Symfile.update({ contents: contents }, { where: { id: symfile.id }, fields: ['contents']})
+    else
+      # At startup, pruning, already no contents, great!
+      return
+
+  fs.mkdirs(path.dirname(filePath)).then ->
+    fs.writeFile(filePath, symfile.contents).then ->
+      if prune
+        console.log "Pruning contents from database for symfile #{symfile.id}, file saved at #{filePath}"
+        delete symfile.contents
+        Symfile.didPrune = true
+        Symfile.update({ contents: null }, { where: { id: symfile.id }, fields: ['contents']})
+
+Symfile.getContents = (symfile) ->
+  if config.get('filesInDatabase')
+    Promise.reslove(symfile.contents)
+  else
+    fs.readFile(Symfile.getPath(symfile), 'utf8')
 
 Symfile.createFromRequest = (req, res, callback) ->
   props = {}
@@ -98,8 +133,10 @@ Symfile.createFromRequest = (req, res, callback) ->
             else
               Promise.resolve()
           p.then ->
-            Symfile.create(props, {transaction: t}).then (symfile) ->
-              Symfile.saveToDisk(symfile).then ->
+            Symfile.saveToDisk(props, false).then ->
+              if not config.get('filesInDatabase')
+                delete props.contents
+              Symfile.create(props, {transaction: t}).then (symfile) ->
                 cache.clear()
                 callback(null, symfile)