Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Tree: 4fcff4ae49
Fetching contributors…

Cannot retrieve contributors at this time

executable file 52 lines (48 sloc) 2.308 kB
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if [ "$1" = '' ]; then
echo "Usage: compute_stats '<table_name>(<comma separated column list>)'"
exit 1
fi
d=`mktemp -d compute_stats_XXXX`
echo -n "Removing hive temp tables from previous runs if any..."
test=`/mnt/vol/hive/stable/cluster/bin/hadoop dfs -ls compute_stats_test 2>>$d/err`
if [ "$test" != '' ]; then
/mnt/vol/hive/stable/cluster/bin/hadoop dfs -rmr compute_stats_test >>$d/err 2>&1
/mnt/vol/hive/stable/cluster/bin/hadoop dfs -rmr /Trash/Current/compute_stats_test >> $d/err 2>&1
fi
echo "Done"
echo -n "Project out columns to test table using hive..."
/mnt/vol/hive/stable/cluster/bin/hadoop jar /mnt/vol/hive/stable/lib/hive/fb_hive.jar com.facebook.hive.streaming.HiveStreaming -select $1 -sample 1/1024 -output compute_stats_test -numReduceTasks 0 >> $d/err 2>&1
echo "Done"
test=`/mnt/vol/hive/stable/cluster/bin/hadoop dfs -ls compute_stats_test 2>>$d/err`
if [ "$test" != '' ]; then
echo -n "Generate single local file out of hdfs table..."
/mnt/vol/hive/stable/cluster/bin/hadoop dfs -cat /user/`whoami`/compute_stats_test/* 2>>$d/err | gzip > $d/sample.gz
echo "Done"
n=`zcat $d/sample.gz | head -1 | tr ',' '\n' | wc -l`
echo "Generate stats for each of the projected columns..."
for ((i=1;i<=$n;i+=1)); do
echo " column $i"
zcat $d/sample.gz | cut -f$i -d',' | sort | uniq -c > $d/$i.stats;
count=`cat $d/$i.stats | wc -l`;
echo $i $count >> $d/overall.stats;
done
echo "Done"
echo "Stats in $d"
else
echo "Error extracting columns from table!";
fi
Jump to Line
Something went wrong with that request. Please try again.