์ฐธ๊ณ ) ํ๋ก ํธ ์ํธ ์น ์ฌ์ดํธ์ ํ๋ก ํธ์๋ ์ฝ๋๋ ์ฌ๊ธฐ๋ฅผ ํด๋ฆญํ๋ฉด ํ์ธ ๊ฐ๋ฅํฉ๋๋ค.
-
node ๋ฒ์ : v12.13.1
-
npm ๋ฒ์ : 6.12.1
-
master ๋ธ๋์น์์ npm i๋ก ๋ ธ๋ ๋ชจ๋์ ์๋ก ๋ค์ด๋ฐ์ต๋๋ค
- package-lock.json ํ์ผ ๋ค์ด๋ก๋ ์ ์ฐธ๊ณ ์๋ฃ: package-lock.josn์ ๊ดํ์ฌ
-
package-lock.json ์ ์ญํ : ๋ฐฑ์๋ ์๋์ ์ํด ํ์ํ ๋ชจ๋ ๋ฆฌ์คํธ๋ก
npm install (๋ชจ๋์ด๋ฆ)
๋ฅผ ํฐ๋ฏธ๋ํด์ ์๋ํ๋ฉด ์ํ๋ ๋ชจ๋๋ค์ ๋ก์ปฌ ์ปดํจํฐ์ ์ค์น ๊ฐ๋ฅํฉ๋๋ค.
- app.js : ๋๋ผ๋ณ ํฌ๋กค๋ง์ ์คํ์ํค๋ javascript ํ์ผ
- public/main_json.js : ๊ตฌ๊ธ ํธ๋ ๋ ์ฌ์ดํธ๋ฅผ ๊ฐ ๋๋ผ๋ณ๋ก ํฌ๋กค๋งํด์ค๊ณ Google Translate API๋ฅผ ์ด์ฉํด์ ๋ฒ์ญํ ๊ฒฐ๊ณผ๋ฅผ json ํ์ผ ํ์์ผ๋ก ์ ์ฅํ๋ javascript ํ์ผ
- public/json_result : ๋ฐฑ์๋์์ ์์ฑํ jsonํ์ผ์ ํฌํจํ๋ ํด๋. ํ๋ก ํธ์๋์์ ๋ฐ์ดํฐ๋ฅผ ๋ณด์ฌ์ฃผ๊ธฐ ์ํด ์ฐธ์กฐ.
Express ์ฑ์ ์ฌ์ฉํ๊ธฐ ์ํด ๋ฏธ๋ค์จ์ด ์์ฑํฉ๋๋ค. ๋ฏธ๋ค์จ์ด ํจ์๋ req ์ค๋ธ์ ํธ, res ์ค๋ธ์ ํธ ์ค ๋ฏธ๋ค์จ์ด ํจ์์ ๋ํ ๊ถํ์ ๊ฐ๋ ํจ์์ ๋๋ค.
๋ฏธ๋ค์จ์ด ํจ์ ํธ์ถ์ get HTTP ๋ฉ์๋๋ฅผ ์ด์ฉํด ๊ฒฝ๋ก๋ก ๋ผ์ฐํ ํด์ ๋ฏธ๋ค์จ์ด ํจ์๋ฅผ ์๋์ํต๋๋ค. Res ์ค๋ธ์ ํธ๋ ๋ฏธ๋ค์จ์ด ํจ์์ ๋ํ HTTP ์๋ต ์ธ์์ด๋ฉฐ req ์ค๋ธ์ ํธ๋ ๋ฏธ๋ค์จ์ด ํจ์์ ๋ํ HTTP ์์ฒญ ์ธ์์ด๋ค.
์ฑ์ ๋ฃจํธ์ ๋ํ ์์ฒญ์ ์คํํ ๋, ์ฑ์ ์ด์ ์์ฒญ์ด ์คํ์ํค๋ ๋ฏธ๋ค์จ์ด ํจ์๋ฅผ ํจ์๋ฅผ ์๋์ํต๋๋ค.
var express = require('express');
var app = express();
app.listen(3001,function(){
console.log('start! express server on port 3001');
});
app.use(express.static('public'));
// url routing
app.get('/',function(req,res){
res.sendFile(__dirname + "/public/main.html");
});
app.get('/result',function(req,res){
res.sendFile(__dirname+'/public/result/result_' + req.query.geo+'.html');
});
app.get('/json_result',function(req,res){
res.sendFile(__dirname+'/public/json_result/result_' + req.query.geo+'.json');
});
๋ชจ๋ ๊ตญ๊ฐ์ ๋ํ ํค์๋ ํฌ๋กค๋ง์ ์ํด ๊ตญ๊ฐ ์ฝ๋์ ๋ฒ์ญ ์ฝ๋๋ฅผ ๋ฆฌ์คํธ ํ์์ผ๋ก ์ ์ฅํฉ๋๋ค.
app.js ํ์ผ์ ์๋์ํค๋ฉด ์ง์ ํด๋ ๋๋ผ๋ณ ๊ตญ๊ฐ์ฝ๋์ ๊ทธ ๋๋ผ์ ์ธ์ด๋ฅผ ํ๊ตญ์ด๋ก ๋ฒ์ญํ์ฌ json ํ์ผ์ ๋ง๋ญ๋๋ค.
/*๊ตญ๊ฐ๋ช
= "๋ฏธ๊ตญ, ์๋ฅดํจํฐ๋, ์ค์คํธ๋ฆฌ์, ์ค์คํธ๋ ์ผ๋ฆฌ์, ๋ฒจ๊ธฐ์, ๋ธ๋ผ์ง, ์บ๋๋ค, ์ค์์ค, ์น ๋ , ์ฝ๋กฌ๋น์, ์ฒด์ฝ, ๋
์ผ, ๋ด๋งํฌ, ์ด์งํธ, ํ๋๋, ํ๋์ค, ์๊ตญ, ๊ทธ๋ฆฌ์ค, ํ์ฝฉ, ํ๊ฐ๋ฆฌ, ์ธ๋๋ค์์, ์์ผ๋๋, ์ด์ค๋ผ์, ์ธ๋, ์ดํ๋ฆฌ์, ์ผ๋ณธ, ์ผ๋, ํ๊ตญ, ๋ฉ์์ฝ, ๋ง๋ ์ด์์, ๋์ด์ง๋ฆฌ์, ๋ค๋๋๋, ๋
ธ๋ฅด์จ์ด, ๋ด์ง๋๋, ํ๋ฆฌํ, ํด๋๋, ํฌ๋ฅดํฌ๊ฐ, ๋ฃจ๋ง๋์, ๋ฌ์์, ์ฌ์ฐ๋์๋ผ๋น์, ์ค์จ๋ด, ์ฑ๊ฐํฌ๋ฅด, ํ๊ตญ, ํฐํค, ๋๋ง, ์ฐํฌ๋ผ์ด๋, ๋ฒ ํธ๋จ */
// ๊ตญ๊ฐ ์ฝ๋
var countries = ["US", "AR", "AT", "AU", "BE", "BR", "CA", "CH", "CL", "CO", "CZ", "DE", "DK", "EG","FI", "FR", "GB", "GR", "HK", "HU", "ID", "IE", "IL", "IN", "IT", "JP", "KE", "KR", "MX", "MY", "NG", "NL", "NO", "NZ", "PH", "PL", "PT", "RO", "RU", "SA", "SE", "SG", "TH", "TR", "TW", "UA", "VN"];
// ๊ตญ๊ฐ ์ธ์ด
var transLangs = ["en", "es", "de", "en", "fr", "pt", "en", "de", "es", "es", "cs", "de", "da", "ar", "fi", "fr", "en", "el", "zh-TW", "hu", "id",
"ga", "he", "hi", "it", "ja", "en", "ko", "es", "ms", "en", "nl", "no", "en", "tl", "pl", "pt", "ro", "ru", "ar", "sv", "en","th", "tr", "zh-TW", "uk", "en", "vi"];
1์๊ฐ์ ํ ๋ฒ์ฉ ๊ตญ๊ฐ๋ณ ํธ๋ ๋ ํค์๋๋ฅผ ์
๋ฐ์ดํธํฉ๋๋ค.
์ด๋ฅผ ์ํด get command๋ฅผ ์ด์ฉํ๋ฉฐ ์ฐ์ ์ด๋ฏธ ๋ง๋ค์ด์ง json ํ์ผ๋ค์ ์ญ์ ํฉ๋๋ค. ๊ทธ๋ฆฌ๊ณ get command๋ฅผ ์ด์ฉํด์ main_json.js ํ์ผ์ ์คํ์ํต๋๋ค. geo
๋ณ์๋ ๊ตญ๊ฐ์ฝ๋์ด๋ฉฐ transLanguage
๋ ๊ตญ๊ฐ์์ ์ฌ์ฉํ๋ ์ธ์ด๋ก main_json.js์ ์ต์
์ผ๋ก ์ ๋ฌํ์ฌ GoogleTranslate API ๋ฅผ ์ด์ฉํด ํ๊ตญ์ด๋ก ๋ฒ์ญํ๋ ๋ฐ ์ฌ์ฉํฉ๋๋ค.
setInterval(function(){
for (var i=0;i<countries.length;i++) { // ๋ชจ๋ ๊ตญ๊ฐ์ ๋ํ ํค์๋ ํฌ๋กค๋ง var geo = countries[i];
var transLanguage = transLangs[i];
cmd.get(
"rm public/json_result/result_"+ geo + ".json"
);
cmd.get(
"node public/main_json.js "+ geo + " "+ transLanguage
);
}
}, 360000); // 1์๊ฐ์ 1๋ฒ ์คํ
Google Trends ์น ์ฌ์ดํธ์์ ์ํ๋ ๋ฐ์ดํฐ ์น ํฌ๋กค๋งํฉ๋๋ค.
Node.js ๋ด์ฅ ๋ชจ๋์ธ Request ๋ชจ๋์ ์ด์ฉํด ์ธํฐ๋ท์ ์์ฒญ์ ๋ณด๋ด๊ณ ์์ฒญ์ ํด๋นํ๋ ํ์ด์ง๋ฅผ ๊ฐ์ ธ์จ๋ค. Cheerio ๋ชจ๋์ ์ด์ฉํด ๋ฐ์์จ ํ์ด์ง๋ฅผ ํ์ฑํ์ฌ ์ ์ฒด ํ์ด์ง ์ค์์ ํ์ํ ๋ถ๋ถ์ ์ ๋ณด๋ง ๊ฐ์ ธ์ต๋๋ค.
var cheerio = require('cheerio');
var request = require('request');
๊ตฌ๊ธ ํธ๋ ๋ ์น ํ์ด์ง์์ ๊ฐ์ ธ์ฌ ๊ตญ๊ฐ ์ฝ๋ ๋ฐ ๋ฒ์ญํ ์ธ์ด ์ฝ๋ ๋ณ์์ ์ ์ฅํฉ๋๋ค.
cmd๋ก ์คํํ ๋ ์ฌ์ฉํ geo
๊ฐ US, transLanguage
๊ฐ en๋ผ๋ฉด sourceUrl
์ ๋ณ์ ๊ฐ์ https://trends.google.com/trends/trendingsearches/daily?geo=US์ด ๋ ๊ฒ์
๋๋ค.
// cmd๋ก ์คํํ ๋ ์ฌ์ฉํ argument ๊ฐ์ ธ์ค๊ธฐ
var geo = process.argv.slice(2)[0]; // ๊ตฌ๊ธ ํธ๋๋์์ ๊ฒ์ํ ๊ตญ๊ฐ ์ฝ๋
var transLanguage = process.argv.slice(2)[1]; // ๋ฒ์ญํ ์ธ์ด ์ฝ๋
var sourceUrl = 'https://trends.google.co.kr/trends/trendingsearches/daily/rss?geo=' + geo;
var title = new Array(),
description = new Array(),
date = new Array(),
title_trans = new Array();
์น์ฌ์ดํธ์์ ํฌ๋กค๋งํด์ค๋ ํธ๋ ๋ ๋ด์ค์ ๋ ์ง์ ์์ผ์ ํ๊ตญ์ด๋ก ๋ฒ์ญํ์ฌ ๋ฐํํ๋ convertWeekDaysByKorean()
ํจ์ getDataFormat()
ํจ์๋ฅผ ์ ์ํฉ๋๋ค.
// ์์ผ ๋ฒ์ญ
function convertWeekDaysByKorean(weekday) {
if (weekday.length < 1) {
return "";
}
weekday = weekday.substring(0, weekday.length-1);
switch (weekday) {
case "Mon" :
return "์์์ผ";
case "Tue" :
return "ํ์์ผ";
case "Wed" :
return "์์์ผ";
case "Thu" :
return "๋ชฉ์์ผ";
case "Fri" :
return "๊ธ์์ผ";
case "Sat" :
return "ํ ์์ผ";
case "Sun" :
return "์ผ์์ผ";
}
return "";
}
function getDataFormat(date) {
let MONTHS = {Jan : "1", Feb : "2", Mar : "3", Apr : "4", May : "5", Jun : "6", Jul : "7", Aug : "8", Sep : "9", Oct : "10", Nov : "11", Dec : "12"}
let arr = date.split(' ');
if (arr.length >= 4) {
return (arr[3] + "๋
" + MONTHS[arr[2]] + "์ " + arr[1] + "์ผ " + convertWeekDaysByKorean(arr[0]));
}
else {
return "";
}
}
request ๋ชจ๋์ ์ด์ฉํด์ cheerio๋ก ๋ถ๋ฌ์ค๊ณ $์ cheerio.load๋ฅผ ํฉ๋๋ค.
// ํฌ๋กค๋งํ๊ธฐ
request(sourceUrl, function(error, response, html){
if (!error) {
var $ = cheerio.load(html, {xmlMode : true});
// ํค์๋
$('item').each(function () {
title.push($(this).children('title').text());
});
// ๋ ์ง
$('item').each(function () {
let pubDate = $(this).children('pubDate').text();
date.push(getDataFormat(pubDate));
});
// ํค์๋ ์ค๋ช
$('item').each(function () {
description.push($(this).children('ht\\:news_item').children('ht\\:news_item_title').text());
});
๋ฒ์ญํ ๋ฌธ์ฅ์ ์์ฑํฉ๋๋ค.
// ๋ฒ์ญ ๋ฐ ์ถ๋ ฅ ํจ์ ์ ์
async function logData(title, date, description){
let tempDate, rank = 1;
let dateCount = 0;
var allContents = "";
var transContents = "";
for (let i = 0; i < 10; i++) { // ๋ฒ์ญํ ์ ์ฒด ๋ฌธ์ฅ ์์ฑ
if (tempDate != date[i]) { // ์ค๋์ keyword 10์๊น์ง๋ง ๋ณด์ด๊ฒ ์ ํ
if(dateCount<2) dateCount++;
if(dateCount==2) break; // ๋ค๋ฅธ ๋ ์ง keyword์ธ ๊ฒฝ์ฐ ์ค์ง
tempDate = date[i];
}
allContents = allContents + title[i] + "^" + description[i] + "^";
}
Google Translate Api ๋ชจ๋์ ์ด์ฉํด ํฌ๋กค๋ง ๊ฒฐ๊ณผ๋ฅผ ํ๊ตญ์ด๋ก ๋ฒ์ญํฉ๋๋ค.
const translate = require('@vitalets/google-translate-api')
// ๋ฒ์ญ ํจ์
async function translateFn(text){
var res = await translate(text, {from: transLanguage, to: 'ko'});
return res.text;
}
ํฌ๋กค๋ง ๊ฒฐ๊ณผ ๋ฒ์ญํ๋ translateFn
ํจ์ ํธ์ถํ์ฌ ๋ฐฐ์ด์ ์ ์ฅํฉ๋๋ค.
// transContents = allContents;
if (geo == "KR") // ํ๊ตญ ํธ๋๋๋ ๋ฒ์ญ ์ํจ
transContents = allContents;
else // ํด์ธ ํธ๋๋ ๋ฒ์ญํ๊ธฐ
transContents = await translateFn(allContents);
// ๋ฒ์ญ๋ ๋ฌธ์ฅ ๋ฐฐ์ด๋ก ์ ์ฅ
var translatedContents = new Array();
translatedContents = transContents.split('^');
const fs = require('fs')
// json ํ์ผ๋ช
์ค์
var fileName = './public/json_result/result_'+geo+'.json';
const resultJson = JSON.stringify(all);
fs.writeFileSync(fileName, resultJson);
ํฌ๋กค๋ง ๋ฐ ๋ฒ์ญ ๊ฒฐ๊ณผ๋ฅผ JSON ํ์ผํ์์ผ๋ก ์ ์ฅํ๊ธฐ ์ํด size, date, keyword๋ผ๋ ์ด๋ฆ์ json array ๋ฐ json object ์ ์ํฉ๋๋ค.
// ์ต์์ json array ์ ์
const all = {
sizes: new Array(),
dates: new Array(),
keywords: new Array()
}
var sizeArray = new Array();
var dateArray = new Array();
var keywordArray = new Array();
// size json object ์ ์
const size = {
size: 0
}
// date json object ์ ์
const date = {
date: tempDate
}
dateArray.push(date);
// keyword json object ์ ์
const keyword = {
title: "",
content: ""
}
๊ฐ ํค์๋์ ๋ํด์ ๋ฒ์ญํ์ฌ ๋ฐฐ์ด์ ์ ์ฅํด ๋์๋ ๊ฒฐ๊ณผ๋ฅผ keywordArray
๋ฐฐ์ด์ ์ถ๊ฐํฉ๋๋ค.
// ๊ฐ ํค์๋์ ๋ํด
for (let rank = 0; rank<(translatedContents.length/2 - 1); rank++) {
var keywordJSON = new Object();
keywordJSON.title = translatedContents[rank*2];
keywordJSON.content = translatedContents[rank*2+1];
keywordArray.push(keywordJSON); // keyword array์ ํค์๋ ์ถ๊ฐ
}
๊ฐ ๋๋ผ๋ณ ํธ๋ ๋์ ๊ฐ์๋ฅผ ์๋ฏธํ๋ sizeArray
๋ฐฐ์ด๊ณผ, ๋ ์ง๋ฅผ ์๋ฏธํ๋ dateArray'๋ฐฐ์ด,
keywordArray๋ฐฐ์ด์ json ๋ฐฐ์ด
all`์ ์ ์ฅํฉ๋๋ค.
size.size = keywordArray.length;
sizeArray.push(size);
all.sizes = sizeArray;
all.dates = dateArray;
all.keywords = keywordArray;
resultJson
์ JSON.stringify()
๋ฅผ ์ด์ฉํด all ๊ฐ์ฒด๋ฅผ Json ๋ฌธ์์ด๋ก ๋ณํํ ๊ฒฐ๊ณผ๊ฐ์ด๋ค. fileName
์ด๋ฆ์ json ํ์ผ๋ก resultJson ๊ฐ์ ์ ์ฅํด์ค๋๋ค.
const resultJson = JSON.stringify(all);
fs.writeFileSync(fileName, resultJson);
ํฌ๋กค๋ง ๋ฐ ๋ฒ์ญ ๊ฒฐ๊ณผ ์์ฑ๋ ํ์ผ๋ค์ด result_(๊ตญ๊ฐ์ฝ๋).json
ํ์์ผ๋ก ์ ์ฅ๋์ด ์์ต๋๋ค.