hive数据导出成csv文件shell脚本

#!/bin/bash 
tab_nm=$1
etldt=`date -d $2 + "%Y%m%d"`
dt=`date -d $2 + "%Y-%m-%d"`
echo "${tab_nm}_${dt}"

#数据存放目录
data_hdfs_dir ="/tmp/idap/export_data/${tab_nm}"
#数据导出路経
local_path ="/gpfsetl/jcp_dw/script/idap/test/${tab_nm}_${dt}"
#scp将数据传到目标服务器上
scp_cmd ="user@xxx.xxx.xx.xxx:/home/dsadm/ycy/export_data"

#删除历史数据
rm -rf ${local_path}
if [ $? -ne 0 ]; then 
 echo "rm -rf ${local_path} ERROR !" 
 exit 2
else 
 echo "删除1ocal数据成功！"
fi

export HADOOP_USER_NAME = dsadm 
hadoop fs -rm -r -f ${data_hdfs_dir} 
if [ $? -ne 0 ]; then 
	echo " hadoop fs -rm -r -f ${data_hdfs_dir} ERROR !"
	exit 2
else 
	echo "删除 hdfs 数据成功！"
fi

#创趣 hdfs 目录
echo "创建 hdfs 目录：${data_hdfs_dir}" 
hadoop fs -mkdir -ps ${data_hdfs_dir}
if [ $? -ne 0 ] ; then 
	echo " hadoop fs -mkdir -p ${local_path} ERROR !"
	exit
e1se
 echo "创建 hdfs 日录：${local_path} 成功！"
fi

echo "创建1ocal目录： ${local_path}" 
mkdir ${local_path}
if [ $? -ne 0 ]; then 
 echo " hadoop fs -mkdir -p ${local_path} ERROR!"
 exit 2
else 
 echo "创建1ocal目录： ${local_path}成功！"
fi
 hadoop fs -chmod -R 777 ${data_hdfs_dir}
 
 

#导出语句
export_sql="select * from idap.${tab_nm} where last_etl_acg_dt = ${dt} "


echo "--将数据导出到 HDFS 上--"
beeline -u jdbc:hive2://olap14:10000 -e 
"set mapred.reduce.tasks=-1; insert overwrite directory ${data_hdfs_dir}
row format serde org.apache.hadoop.hive.serde2.OpenCSVSerde ${export_sql} distribute by 1;"
 
if [ $? -ne 0 ]; then 
	 echo "将数据导出到 HDFS 上 failed !"
	 exit 2
else 
	echo "将数据导出到 HDFS 上 success !"

echo "--将数据从 HDFS 导出到本地------"
fi

hadoop fs -getmerge ${data_hdfs_dir} ${local_path}/${tab_nm}_${etldt}.del 

if [ $? -ne 0 ]; then 
 echo "将数据从 HDFS 导出到本地失败！"
 exit 2
else 
 echo "特数据从 HDFS 导出到本地成功！"
fi

# Scp 将数据传到目标服务器上
scp ${local_path}/${tab_nm}_${etldt}.del ${scp_cmd} 
if [ $? -ne 0 ]; then 
	echo "scp 将数据传到目标服务器上失败！"
	exit 2
else 
	echo "scp 将数据传到目标服务器上成功！"
fi 

#删除临时数据文件 
echo "----删除临时数据文件----"
hadoop fs -rm -r -f ${data_hdfs_dir} 
rm -rf ${local_path}