From fdfa714ca4822ec960ab8790c61934d730809596 Mon Sep 17 00:00:00 2001 From: chrisweb Date: Sat, 15 Oct 2016 23:57:16 +0200 Subject: [PATCH] refactoring --- .gitattributes | 5 ++ .gitignore | 21 +++--- README.md | 63 ++++++++++------- client/TODO | 4 -- downloads/.gitignore | 11 +++ package.json | 14 ++-- server/bootstrap.js | 68 +++++++++++++------ server/library/audioDataAnalyzer.js | 73 ++++++++++---------- server/library/directoryManager.js | 4 +- server/library/fileDownloader.js | 4 +- server/library/fileManager.js | 2 +- server/library/waveformData.js | 101 +++++++++++++--------------- waveform-data-generator.njsproj | 78 +++++++++++++++++++++ waveform-data-generator.sln | 22 ++++++ 14 files changed, 312 insertions(+), 158 deletions(-) create mode 100644 .gitattributes delete mode 100644 client/TODO create mode 100644 downloads/.gitignore create mode 100644 waveform-data-generator.njsproj create mode 100644 waveform-data-generator.sln diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b7ca95b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# Auto detect text files and perform LF normalization +* text=auto + +# JS files must always use LF for tools to work +*.js eol=lf diff --git a/.gitignore b/.gitignore index 19728da..974f79b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,17 +12,22 @@ lib-cov pids results +# npm npm-debug.log + +# windows Thumbs.db desktop.ini +# third party libraries node_modules -downloads -yeoman -nbproject -.sass-cache - -downloads -client_build +vendor -configuration.js \ No newline at end of file +# IDEs +nbproject/private +.vs +*.user +.ntvs_analysis.dat +.ntvs_analysis.dat.tmp +*.njsperf +Output-Npm.txt \ No newline at end of file diff --git a/README.md b/README.md index 935bf39..1e9e55a 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,71 @@ -waveform-data-generator -======================= +# waveform-data-generator -Generates waveform data (peaks) that can then get visualized using: https://github.com/chrisweb/waveform-visualizer +Generates waveform data (peaks) that can then get visualized using: https://github.com/chrisweb/waveform-visualizer ![](https://github.com/chrisweb/waveform-visualizer/blob/master/examples/images/waveform.png) -Waveform created using the visualizer with data from waveform data generator +Waveform created using the visualizer with data from waveform data generator -Getting started ---------------- +## Getting started * Install git and then do a local checkout of this project * First, start by installing nodejs (http://nodejs.org/) (which includes npm) to run the server or use the cli tool +* update npm to ensure you have the latest version installed +```npm install npm -g``` * Now install ffmeg package based on your operating system (https://www.ffmpeg.org/download.html) (and if you develop in windows add it to your path: http://www.wikihow.com/Install-FFmpeg-on-Windows) * Do a local checkout of this project using git * Use your command line tool and go to the root of this project (type: cd /LOCAL_PROJECT_PATH) * type: npm install, to install the required nodejs modules (dependencies) -Launch the server ------------------ +## Launch the server * Use your command line tool and go to the root of this project (type: cd /LOCAL_PROJECT_PATH) * To lauch the server type: node server * Open your browser and type the following address: 127.0.0.1:35000 -Use the command line tool -------------------------- +## Using the command line tool * Use your command line tool and go to the root of this project (type: cd /LOCAL_PROJECT_PATH) * Type: node cli PARAMETER_1 PARAMETER_2 (...) * For example: node cli ./downloads 1100511 ogg 200 local json false -Out of memory: --------------- +### Cli Options (Parameters) + +* The first parameter "./downloads" is the repository where you want the audio files to get stored +* The second parameter "1100511" is the ID (also filename) of the track +* The third parameter "ogg" is the audio file format (also file exntension) of the track (ogg / mp3) +* The fourth parameter "200" is the amount of peaks you want to get +* The fifth parameter "local" tells the script if the file is already on your local machine, use "jamendo" to download the file from jamendo and store it the downloads directory +* The sixth parameter "json" is the type of output you want, the peaks can get outputted in either json or as a string +* The seventh parameter tells the script if it should use ffprobe to detect the track format (number of channels, the sampling frequency, ...) or use default values + +## Using the web interface -Error "FATAL ERROR: JS Allocation failed - process out of memory" +* Use the following url to fetch a song from a remote source (currently only support for jamendo, fork me of you want to add another serive) and get a json containing the peaks +```http://127.0.0.1:35000/getwavedata?service=jamendo&trackId=1321406``` -If the file you want to parse is too big, try increasing the memory limit of your node process, like this: +### web interface Options (Parameters) -node --max-old-space-size=1900 cli ./downloads 1100511 ogg 200 local json false +* trackId: the ID of a track [required / numeric track ID] +* trackFormat: the audio file format (file exntension) of the track (ogg or mp3) [default ogg] +* peaksAmount: the amount of peaks you want to get [default 200] +* method: the http request method to get the remote file [default GET] +* serverDirectory: the repository where you want the audio files to get stored on the server [default ./downloads] +* service: the audio file provider you want to get the track from (you can use 'local' if the file is already in your server directory) [default jamendo] +* detectFormat: tells the script if it should use ffprobe to detect the track format (number of channels, the sampling frequency, ...) or use default values (true or false) [default false] -If you still run out of memory try to reduce the sample rate by passing custom values as seventh parameter, for example if the song sample rate is 44100 but runs out of memory, then try again with 22050, like this: +Out of memory: +-------------- -node --max-old-space-size=1900 cli ./downloads 1100511 ogg 200 local json sr=22050 +Error "FATAL ERROR: JS Allocation failed - process out of memory" -Parameters: +If the file you want to parse is too big, try increasing the memory limit of your node process, like this: -* The first parameter "./downloads" is the repository where you want the audio files to get stored -* The second parameter "1100511" is the name of the track -* The third parameter "ogg" is the format of the track (ogg / mp3) -* The fourth parameter "200" is the amount of peaks you want to get -* The fifth parameter "local" tells the script if the file is already on your local machine, use "jamendo" to download the file from jamendo and store it the downloads directory -* The sixth parameter "json" is the type of output you want, the peaks can get outputted in either json or as a string -* The seventh parameter tells the script if it should use ffprobe to detect the track format (number of channels, the sampling frequency, ...) or use default values +node --max-old-space-size=1900 cli ./downloads 1100511 ogg 200 local json false + +If you still run out of memory try to reduce the sample rate by passing custom values as seventh parameter, for example if the song sample rate is 44100 but runs out of memory, then try again with 22050, like this: + +node --max-old-space-size=1900 cli ./downloads 1100511 ogg 200 local json sr=22050 TODOs ----- diff --git a/client/TODO b/client/TODO deleted file mode 100644 index be2338d..0000000 --- a/client/TODO +++ /dev/null @@ -1,4 +0,0 @@ -TODO -==== - -Client side waveform data generation \ No newline at end of file diff --git a/downloads/.gitignore b/downloads/.gitignore new file mode 100644 index 0000000..3ae4e5f --- /dev/null +++ b/downloads/.gitignore @@ -0,0 +1,11 @@ +*.mp3 +*.ogg +*.wav +*.aiff +*.wma +*.webm +*.oga +*.opus +*.m4a +*.aac +*.flac \ No newline at end of file diff --git a/package.json b/package.json index b87bdb9..832ac3c 100644 --- a/package.json +++ b/package.json @@ -4,11 +4,12 @@ "description": "nodejs (ffmpeg) waveform data (peaks) generator", "main": "server.js", "scripts": { - "test": "none yet" + "test": "", + "start": "node server.js" }, "repository": { "type": "git", - "url": "git@github.com:chrisweb/waveform-data-generator.git" + "url": "git+ssh://git@github.com/chrisweb/waveform-data-generator.git" }, "keywords": [ "waveform", @@ -17,12 +18,15 @@ "data", "generator", "peaks", - "ffmpeg" + "ffmpeg" ], - "author": "Chris Weber (chrisweb) ", + "author": "Chris Weber (chrisweb)", "license": "MIT", "bugs": { "url": "https://github.com/chrisweb/waveform-data-generator/issues" }, - "homepage": "https://github.com/chrisweb/waveform-data-generator" + "homepage": "https://github.com/chrisweb/waveform-data-generator", + "dependencies": { + "marked": "0.3.5" + } } diff --git a/server/bootstrap.js b/server/bootstrap.js index 7408a83..2ab17a0 100644 --- a/server/bootstrap.js +++ b/server/bootstrap.js @@ -8,8 +8,10 @@ var querystring = require('querystring'); var waveformData = require('./library/waveformData'); -var serverPort = 35000; -var serverIp = '127.0.0.1'; +var marked = require('marked'); + +var serverPort = process.env.PORT || 35000; +var serverIp = process.env.HOSTNAME || '127.0.0.1'; /** * @@ -18,20 +20,18 @@ var serverIp = '127.0.0.1'; * @param {type} request * @param {type} response */ -http.createServer(function (request, response) { +var server = http.createServer(function (request, response) { // parse the url var urlParts = url.parse(request.url); - //console.log(urlParts); - // check if its is the url of a javascript file if (urlParts.pathname.split('.').pop() === 'js') { // if the file exists send it to the client // not really secure but this is a prototype // TODO: filter the file request - fs.readFile('client' + urlParts.pathname, function(error, fileContent) { + fs.readFile('client' + urlParts.pathname, function (error, fileContent) { if (!error) { @@ -52,14 +52,14 @@ http.createServer(function (request, response) { }); } else { - + // handle the "routes" - switch(urlParts.pathname) { + switch (urlParts.pathname) { case '/': - fs.readFile('client/index.html', function(error, html) { + /*fs.readFile('client/index.html', function (error, html) { if (!error) { - + // send the main html page to the client response.writeHead(200, { 'Content-Type': 'text/html' }); response.write(html); @@ -75,15 +75,37 @@ http.createServer(function (request, response) { } + });*/ + + fs.readFile('README.md', 'utf-8', function (error, document) { + + if (!error) { + + // send the main html page to the client + response.writeHead(200, { 'Content-Type': 'text/html' }); + response.write(marked(document)); + response.end(); + + } else { + + // the main page could not be found return a page not + // found message + response.writeHead(404, { 'Content-Type': 'text/html' }); + response.write('page not found'); + response.end(); + + } + }); + break; case '/getwavedata': var queryObject = querystring.parse(urlParts.query); if (typeof queryObject !== 'undefined') { - - waveformData.getRemoteWaveData(queryObject, function(error, peaks) { + + waveformData.getRemoteWaveData(queryObject, function (error, peaks) { if (!error) { @@ -113,11 +135,11 @@ http.createServer(function (request, response) { //console.log(queryObject); if (typeof queryObject !== 'undefined' && queryObject.trackId !== 'undefined' && queryObject.trackFormat !== 'undefined') { - + var trackName = queryObject.trackId + '.' + queryObject.trackFormat; if (queryObject.serverDirectory === undefined) { - + queryObject.serverDirectory = './downloads'; } @@ -125,10 +147,10 @@ http.createServer(function (request, response) { var trackPath = queryObject.serverDirectory + '/' + trackName; var fileStat = fs.statSync(trackPath); - + var mimeType; - - switch(queryObject.trackFormat) { + + switch (queryObject.trackFormat) { case 'ogg': mimeType = 'audio/ogg'; break; @@ -136,9 +158,9 @@ http.createServer(function (request, response) { mimeType = 'audio/mpeg'; break; } - + response.writeHead(200, { 'Content-Type': mimeType, 'Content-Length': fileStat.size }); - + var readStream = fs.createReadStream(trackPath); readStream.pipe(response); @@ -154,6 +176,10 @@ http.createServer(function (request, response) { } } -}).listen(serverPort, serverIp); +}); + +server.listen(serverPort, serverIp, function () { + + console.log('server is listening, ip: ' + serverIp + ', port: ' + serverPort); -console.log('server is listening, ip: ' + serverIp + ', port: ' + serverPort); \ No newline at end of file +}); \ No newline at end of file diff --git a/server/library/audioDataAnalyzer.js b/server/library/audioDataAnalyzer.js index f19cbcf..138635b 100644 --- a/server/library/audioDataAnalyzer.js +++ b/server/library/audioDataAnalyzer.js @@ -171,7 +171,7 @@ analyzer.prototype.getFormat = function getFormatFunction(trackPath, callback) { // if the trackdata object isnt empty if (Object.keys(that.trackData).length > 0) { - callback(false, that.trackData); + callback(null, that.trackData); } else { @@ -205,7 +205,7 @@ analyzer.prototype.getFormat = function getFormatFunction(trackPath, callback) { } else { - callback(false, null); + callback(null, this.trackData); } @@ -242,11 +242,9 @@ analyzer.prototype.getPeaks = function getValuesFunction(trackPath, peaksAmountR that.printMemory({'file': 'audioDataAnalyzer', 'line': '258'}); - if (trackData === null) { - - trackData = {}; - - if(typeof that.detectFormat === 'number'){ + if (Object.keys(trackData).length === 0 && trackData.constructor === Object) { + + if (typeof that.detectFormat === 'number') { trackData.sampleRate = that.detectFormat; @@ -310,8 +308,6 @@ analyzer.prototype.getPeaks = function getValuesFunction(trackPath, peaksAmountR ffmpegSpawn.stdout.on('end', function(data) { - //console.log('ffmpegSpawn stdout end'); - var samplesLength = that.samples.length; // check if we got enough samples to put at least one sample @@ -392,42 +388,49 @@ analyzer.prototype.getPeaks = function getValuesFunction(trackPath, peaksAmountR }); ffmpegSpawn.on('exit', function(code) { - - if (code > 0) { - - if (that.stderrFfmpegOuputString === '') { - - that.stderrFfmpegOuputString = 'unknown ffmpeg error'; - - } - - callback(that.stderrFfmpegOuputString); - - } else { - - var peaksInPercentLength = that.peaksInPercent.length; + + // under heavy load it seems that sometimes ffmpegSpawn.on('exit') gets called + // before ffmpegSpawn.stdout.on('end') got called + // so we now wait 200ms before trying to read the peaks + setTimeout(function () { - if (peaksInPercentLength > 0) { + if (code > 0) { + + if (that.stderrFfmpegOuputString === '') { + + that.stderrFfmpegOuputString = 'unknown ffmpeg error'; - callback(false, that.peaksInPercent); + } + + callback(that.stderrFfmpegOuputString); } else { + + var peaksInPercentLength = that.peaksInPercent.length; + + if (peaksInPercentLength > 0) { + + callback(null, that.peaksInPercent); - var samplesLength = that.samples.length; - - if (samplesLength === 0) { - - callback('no output recieved'); + } else { + + var samplesLength = that.samples.length; + + if (samplesLength === 0) { + + callback('no output recieved'); - } else if (samplesLength < peaksAmount) { + } else if (samplesLength < peaksAmount) { + + callback('not enough peaks in this song for a full wave'); - callback('not enough peaks in this song for a full wave'); + } } - - } - } + } + + }, 200); }); diff --git a/server/library/directoryManager.js b/server/library/directoryManager.js index 3bfbf27..645c095 100644 --- a/server/library/directoryManager.js +++ b/server/library/directoryManager.js @@ -30,7 +30,7 @@ directoryManager.prototype.exists = function directoryExistsFunction(directory, // async exists fs.exists(directory, function(exists) { - callback(false, exists); + callback(null, exists); }); @@ -74,7 +74,7 @@ directoryManager.prototype.create = function createDirectoryFunction(directory, if (!error) { - callback(false); + callback(null); } else { diff --git a/server/library/fileDownloader.js b/server/library/fileDownloader.js index d41d7e8..e5b643a 100644 --- a/server/library/fileDownloader.js +++ b/server/library/fileDownloader.js @@ -113,7 +113,7 @@ downloader.prototype.downloadIfNotExists = function downloadIfNotExists(options, } else { - callback(false, filePath); + callback(null, filePath); } @@ -209,7 +209,7 @@ downloader.prototype.downloadFile = function downloadFileFunction(downloadOption // close the write stream writeStream.end(); - callback(false, serverFilePath); + callback(null, serverFilePath); }); diff --git a/server/library/fileManager.js b/server/library/fileManager.js index b531f52..be2672d 100644 --- a/server/library/fileManager.js +++ b/server/library/fileManager.js @@ -30,7 +30,7 @@ fileManager.prototype.exists = function fileExistsFunction(file, callback) { // async exists fs.exists(file, function(exists) { - callback(false, exists); + callback(null, exists); }); diff --git a/server/library/waveformData.js b/server/library/waveformData.js index a6659f6..6df0680 100644 --- a/server/library/waveformData.js +++ b/server/library/waveformData.js @@ -10,33 +10,16 @@ var queryObjectToOptions = function queryObjectToOptionsFunction(queryObject) { var options = { trackId: queryObject.trackId, - trackFormat: queryObject.trackFormat, - peaksAmount: queryObject.peaksAmount, + trackFormat: queryObject.trackFormat || 'ogg', + peaksAmount: queryObject.peaksAmount || 200, method: 'GET', - serverDirectory: queryObject.serverDirectory, - fileName: queryObject.trackId + '.' + queryObject.trackFormat, - service: queryObject.service, - detectFormat: queryObject.detectFormat + serverDirectory: queryObject.serverDirectory || './downloads', + service: queryObject.service || 'jamendo', + detectFormat: queryObject.detectFormat || false }; - if (options.serverDirectory === undefined) { - - options.serverDirectory = './downloads'; - - } - - if (options.service === undefined) { - - options.service = 'jamendo'; - - } - - if (options.detectFormat === undefined) { - - options.detectFormat = false; - - } - + options.fileName = options.trackId + '.' + options.trackFormat; + return options; }; @@ -54,7 +37,7 @@ var analyzeAudio = function analyzeAudioFunction(filePath, options, callback) { // if there was no error analyzing the track if (!error) { - callback(false, peaks); + callback(null, peaks); } else { @@ -79,45 +62,53 @@ var getRemoteWaveData = function getRemoteWaveDataFunction(queryObject, callback // track options var options = queryObjectToOptions(queryObject); - // service options - switch(queryObject.service) { + if (typeof options.trackId !== 'undefined') { + + // service options + switch (queryObject.service) { + + case 'jamendo': + default: + + // track format + switch (queryObject.trackFormat) { + case 'ogg': + options.formatCode = 'ogg1'; + break; + default: + options.formatCode = 'mp31'; + } + + options.remoteHost = 'storage-new.newjamendo.com'; + options.remotePath = '/download/track/' + options.trackId + '/' + options.formatCode; + options.remotePort = 80; + + } - case 'jamendo': - default: + // initialize the track downloader + var fileDownloader = new FileDownloader(); - // track format - switch(queryObject.trackFormat) { - case 'ogg': - options.formatCode = 'ogg1'; - break; - default: - options.formatCode = 'mp31'; - } - - options.remoteHost = 'storage-new.newjamendo.com'; - options.remotePath = '/download/track/' + options.trackId + '/' + options.formatCode; - options.remotePort = 80; + // download the track and write it on the disc of it does not already exist + fileDownloader.writeToDisc(options, function writeFileCallback(error, filePath) { - } - - // initialize the track downloader - var fileDownloader = new FileDownloader(); - - // download the track and write it on the disc of it does not already exist - fileDownloader.writeToDisc(options, function writeFileCallback(error, filePath) { + // if there was no error downloading and writing the track + if (!error) { + + analyzeAudio(filePath, options, callback); - // if there was no error downloading and writing the track - if (!error) { + } else { + + callback(error); - analyzeAudio(filePath, options, callback); + } - } else { + }); - callback(error); + } else { - } + callback('please specify at least a trackId'); - }); + } }; diff --git a/waveform-data-generator.njsproj b/waveform-data-generator.njsproj new file mode 100644 index 0000000..e29e7fe --- /dev/null +++ b/waveform-data-generator.njsproj @@ -0,0 +1,78 @@ + + + + Debug + 2.0 + {dfeb48ac-81ed-4db0-90bd-4052c0e46770} + + ShowAllFiles + server.js + . + . + {3AF33F2E-1136-4D97-BBB7-1795711AC8B8};{349c5851-65df-11da-9384-00065b846f21};{9092AA53-FB77-4645-B42D-1CCCA6BD08BD} + 11.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + False + True + 0 + / + http://localhost:48022/ + False + True + http://localhost:1337 + False + + + + + + + CurrentPage + True + False + False + False + + + + + + + + + False + False + + + + + \ No newline at end of file diff --git a/waveform-data-generator.sln b/waveform-data-generator.sln new file mode 100644 index 0000000..22f5f42 --- /dev/null +++ b/waveform-data-generator.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.24720.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9092AA53-FB77-4645-B42D-1CCCA6BD08BD}") = "waveform-data-generator", "waveform-data-generator.njsproj", "{DFEB48AC-81ED-4DB0-90BD-4052C0E46770}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {DFEB48AC-81ED-4DB0-90BD-4052C0E46770}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {DFEB48AC-81ED-4DB0-90BD-4052C0E46770}.Debug|Any CPU.Build.0 = Debug|Any CPU + {DFEB48AC-81ED-4DB0-90BD-4052C0E46770}.Release|Any CPU.ActiveCfg = Release|Any CPU + {DFEB48AC-81ED-4DB0-90BD-4052C0E46770}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal