forked from myui/hivemall
-
Notifications
You must be signed in to change notification settings - Fork 0
news20 multiclass dataset
Makoto YUI edited this page Jun 23, 2014
·
5 revisions
Get the news20 dataset. http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass.html#news20
$ cat conv.awk
BEGIN{ FS=" " }
{
label=$1;
features=$2;
for(i=3;i<=NF;i++)
{
features = features "," $i;
}
print NR "\t" label "\t" features;
}
END{}
$ gawk -f conv.awk news20.scale > news20_scale.train
$ gawk -f conv.awk news20.t.scale > news20_scale.test
hadoop fs -mkdir /dataset/news20-multiclass/train
hadoop fs -mkdir /dataset/news20-multiclass/test
hadoop fs -copyFromLocal news20_scale.train /dataset/news20-multiclass/train
hadoop fs -copyFromLocal news20_scale.test /dataset/news20-multiclass/test
use news20;
delete jar /home/myui/tmp/hivemall.jar;
add jar /home/myui/tmp/hivemall.jar;
source /home/myui/tmp/define-all.hive;
Create external table news20mc_train (
rowid int,
label int,
features ARRAY<STRING>
) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' COLLECTION ITEMS TERMINATED BY "," STORED AS TEXTFILE LOCATION '/dataset/news20-multiclass/train';
Create external table news20mc_test (
rowid int,
label int,
features ARRAY<STRING>
) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' COLLECTION ITEMS TERMINATED BY "," STORED AS TEXTFILE LOCATION '/dataset/news20-multiclass/test';
create or replace view news20mc_train_x3
as
select
*
from (
select
amplify(3, *) as (rowid, label, features)
from
news20mc_train
) t
CLUSTER BY rand();
create table news20mc_test_exploded as
select
rowid,
label,
cast(split(feature,":")[0] as int) as feature,
cast(split(feature,":")[1] as float) as value
from
news20mc_test LATERAL VIEW explode(addBias(features)) t AS feature;