insertintomain
#!/bin/sh
#Keep 7days data into main table and drop older partition.
set -x
if [ $# -ne 2 ]; then
echo 'Invalid number of arguments -- <1.ENV> <2.Load_type>'
exit 2
else
echo Argument passed correctly. Script Now Running for Environment: $1
fi
DIRNAME=`dirname $0`
PWD=`pwd`
echo PWD is $PWD
echo DIRNAME is $DIRNAME
if [ "$DIRNAME" = "." ]; then
export SCRIPT=$PWD ;
else
export SCRIPT=$DIRNAME ;
fi
hive_table_name=creb_timer_full
echo hive table name is : $hive_table_name
script_edh=${SCRIPT%/*}
echo script path is : $script_edh
DB_DIR=${script_edh%/*}
echo DB_DIR is : $DB_DIR
db=${DB_DIR##*/}
echo database is :$db
z2db=edh_50060_PSH_rb_$1
echo z2 db is : $z2db
echo Selecting First three bytes of argument to match environment name...
ENV=`echo $1 | cut -c 1-3`
case "$ENV" in
"sit") export connect_string="!connect jdbc:hive2://edh2dev.cloud1.cibc.com:10005/;principal=hive/_HOST@ADP1.CIBC.PTE"
;;
"uat") export connect_string="!connect jdbc:hive2://edh2uat.cloud1.cibc.com:10005/;principal=hive/_HOST@ADP1.CIBC.PTE"
;;
"pdi") export connect_string="!connect jdbc:hive2://edh2uat.cloud1.cibc.com:10005/;principal=hive/_HOST@ADP1.CIBC.PTE"
;;
"prd") export connect_string="!connect jdbc:hive2://edh2prod.cloud1.cibc.com:10005/;principal=hive/_HOST@AD.CIBC.COM"
;;
esac
# Oldest LoadDate allowed = delete all partitions having LoadDate < $oldestLoadDateAllowed
connect_string1=`echo $connect_string |cut -d" " -f2`
#select max proc_date from below query
proc_dt_temp=`beeline -u $connect_string1 verbose=true --silent=true -e "select cast(regexp_replace(date_sub(from_unixtime(unix_timestamp(cast(proc_dt_drvd as string),'yyyyMMdd'),'yyyy-MM-dd'),7),'-','') as int) as proc_dt_drvd from $z2db.${hive_table_name}_ts order by proc_dt_drvd desc limit 1"`
LoadDateAllowed=`echo $proc_dt_temp|cut -d"|" -f4 | tr -d ' '`
echo Load Date Allowed is: $LoadDateAllowed
if [ $2 = "initial" ]; then
oldestLoadDateAllowed=$LoadDateAllowed
elif [ $2 = "daily" ]; then
oldestLoadDateAllowed_temp=`beeline -u $connect_string1 verbose=true --silent=true -e "select proc_dt_drvd from $z2db.${hive_table_name}_ts order by proc_dt_drvd desc limit 1"`
oldestLoadDateAllowed=`echo $oldestLoadDateAllowed_temp|cut -d"|" -f4 | tr -d ' '`
fi
echo Inserting 7 days data into $hive_table_name
beeline -u $connect_string1 verbose=true -f $script_edh/base24_rb_futr/ddls/populateFromZ2_$hive_table_name.sql --hivevar OLDEST_LOADDATE_ALLOWED=$oldestLoadDateAllowed --hivevar db=$db --hivevar z2db=$z2db --hivevar LoadDateAllowed=$LoadDateAllowed
if [ $? -eq 0 ]; then
echo " ------------------------------------------------------------------------"
echo " SUCCESSFULLY populated" $db.base24_rb_futr from Z2 {$hive_table_name}_ts
echo " ------------------------------------------------------------------------"
else
exit 4
fi
Comments
Post a Comment