Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix hardcoded /tmp and /opt/cloudera to get the value of the parameter -b backupfolder only for hdp #45

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Fix hardcoded /tmp and /opt/cloudera to get the value of the parameter -b backupfolder only for hdp #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ee1c3d7
Delete hdp_log4j_jndi_removal.sh
akhadhri Dec 29, 2021
3abf309
Add files via upload
akhadhri Dec 29, 2021
70ac653
Delete hdp_support_scripts directory
akhadhri Dec 29, 2021
5761620
Add files via upload
akhadhri Dec 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions hdp_log4j_jndi_removal.sh
100755 → 100644
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
BASEDIR=$(dirname "$0")
echo $BASEDIR
platform=${3:-common}
backup_dir=$2

if ! command -v zip &> /dev/null; then
echo "zip not found. zip is required to run this script."
Expand All @@ -30,7 +31,7 @@ fi

if [ -z "$SKIP_JAR" ]; then
echo "Removing JNDI from jar files"
$BASEDIR/hdp_support_scripts/delete_jndi.sh "$1" $2
$BASEDIR/hdp_support_scripts/delete_jndi.sh "$1" $backup_dir
else
echo "Skipped patching .jar"
fi
Expand All @@ -41,8 +42,8 @@ if [ -z "$SKIP_HDFS" ]; then
echo "Found an HDFS namenode on this host, removing JNDI from HDFS tar.gz files for platform='$platform'"
keytab_file="hdfs.headless.keytab"
keytab=$(find /etc/security/keytabs/ -type f -iname $keytab_file |tail -1)
$BASEDIR/hdp_support_scripts/patch_hdfs_tgz.sh "/hdp/apps/" $keytab
$BASEDIR/hdp_support_scripts/patch_hdfs_tgz.sh "/user/" $keytab
$BASEDIR/hdp_support_scripts/patch_hdfs_tgz.sh "/hdp/apps/" $keytab $backup_dir
$BASEDIR/hdp_support_scripts/patch_hdfs_tgz.sh "/user/" $keytab $backup_dir
fi
elif [ $platform == "dell" ]; then
if ps -efww | grep org.apache.hadoop.yarn.server.resourcemanager.ResourceManager | grep -v grep 1>/dev/null 2>&1; then
Expand All @@ -52,8 +53,8 @@ if [ -z "$SKIP_HDFS" ]; then
if [[ -z "$keytab" || ! -s $keytab ]]; then
echo "If this is a secure cluster, please ensure that /etc/security/keytabs/hdfs.headless.keytab is present for DELL."
fi
$BASEDIR/hdp_support_scripts/patch_hdfs_tgz.sh "/hdp/apps/" $keytab
$BASEDIR/hdp_support_scripts/patch_hdfs_tgz.sh "/user/" $keytab
$BASEDIR/hdp_support_scripts/patch_hdfs_tgz.sh "/hdp/apps/" $keytab $backup_dir
$BASEDIR/hdp_support_scripts/patch_hdfs_tgz.sh "/user/" $keytab $backup_dir
fi
fi
else
Expand All @@ -62,6 +63,6 @@ fi

if [ -n "$RUN_SCAN" ]; then
echo "Running scan for missed JndiLookup classes. This may take a while."
$BASEDIR/hdp_support_scripts/scan_jndi.sh "$1" $2
$BASEDIR/hdp_support_scripts/scan_jndi.sh "$1" $backup_dir
fi

36 changes: 19 additions & 17 deletions hdp_support_scripts/delete_jndi.sh
100755 → 100644
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@
set -eu -o pipefail

BASEDIR=$(dirname "$0")
tmpdir=${TMPDIR:-/tmp}
mkdir -p $tmpdir
echo "Using tmp directory '$tmpdir'"
backup_dir=$2

#tmpdir=${TMPDIR:-/tmp}
mkdir -p $backup_dir
echo "Using tmp directory '$backup_dir'"

patch_tgz=$BASEDIR/patch_tgz.sh
if [ ! -f "$patch_tgz" ]; then
Expand All @@ -32,18 +34,18 @@ do
if [ -d $targetdir ]; then
echo "Running on '$targetdir'"

backupdir=${2:-/opt/cloudera/log4shell-backup}
mkdir -p "$backupdir"
echo "Backing up files to '$backupdir'"
#backupdir=${2:-/opt/cloudera/log4shell-backup}
#mkdir -p "$backupdir"
echo "Backing up files to '$backup_dir'"

for archivefile in $(find -L $targetdir -name "*.[wnj]ar"); do
if [ -L "$archivefile" ]; then
continue
fi
if grep -q JndiLookup.class $archivefile; then
# Backup file only if backup doesn't already exist
mkdir -p "$backupdir/$(dirname $archivefile)"
targetbackup="$backupdir/$archivefile.backup"
mkdir -p "$backup_dir/$(dirname $archivefile)"
targetbackup="$backup_dir/$archivefile.backup"
if [ ! -f "$targetbackup" ]; then
echo "Backing up to '$targetbackup'"
cp -f "$archivefile" "$targetbackup"
Expand All @@ -57,20 +59,20 @@ do
if unzip -l $archivefile | grep -v 'Archive:' | grep '\.jar$' >/dev/null; then
doZip=0

rm -r -f $tmpdir/unzip_target
mkdir $tmpdir/unzip_target
rm -r -f $backup_dir/unzip_target
mkdir $backup_dir/unzip_target
set +e
unzip -qq $archivefile -d $tmpdir/unzip_target
unzip -qq $archivefile -d $backup_dir/unzip_target
set -e

for jarfile in $(find -L $tmpdir/unzip_target/ -name "*.jar"); do
for jarfile in $(find -L $backup_dir/unzip_target/ -name "*.jar"); do
if [ -L "$jarfile" ]; then
continue
fi
if grep -q JndiLookup.class $jarfile; then
# Backup file only if backup doesn't already exist
mkdir -p "$backupdir/$(dirname $jarfile)"
targetbackup="$backupdir/$jarfile.backup"
mkdir -p "$backup_dir/$(dirname $jarfile)"
targetbackup="$backup_dir/$jarfile.backup"
if [ ! -f "$targetbackup" ]; then
echo "Backing up to '$targetbackup'"
cp -f "$jarfile" "$targetbackup"
Expand All @@ -86,7 +88,7 @@ do
if [ 1 -eq $doZip ]; then
tempfile=$(mktemp -u)
echo "Updating '$archivefile'"
pushd $tmpdir/unzip_target >/dev/null
pushd $backup_dir/unzip_target >/dev/null
zip -r -q $tempfile .
popd >/dev/null

Expand All @@ -97,7 +99,7 @@ do
rm -f $tempfile
fi

rm -r -f $tmpdir/unzip_target
rm -r -f $backup_dir/unzip_target
fi
done

Expand All @@ -106,7 +108,7 @@ do
continue
fi
if zgrep -q JndiLookup.class $tarfile; then
$patch_tgz $tarfile
$patch_tgz $tarfile $backup_dir
fi
done
else
Expand Down
19 changes: 10 additions & 9 deletions hdp_support_scripts/patch_hdfs_tgz.sh
100755 → 100644
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ BASEDIR=$(dirname "$0")

hdfs_path=$1
keytab=$2
backup_dir=$3

if [ ! "$#" -eq 2 ]; then
if [ ! "$#" -eq 3 ]; then
echo "Invalid arguements. The argument must be an HDFS directory and valid keytab file."
exit 1
fi
Expand Down Expand Up @@ -51,9 +52,9 @@ else
kinit -kt $keytab $principal
fi

tmpdir=${TMPDIR:-/tmp}
mkdir -p $tmpdir
echo "Using tmp directory '$tmpdir'"
#tmpdir=${TMPDIR:-/tmp}
mkdir -p $backup_dir
echo "Using tmp directory '$backup_dir'"

for hdfs_file_path in $($user_option hdfs dfs -ls -R $hdfs_path | awk 'BEGIN {LAST=""} /^d/ {LAST=$8} /^-.*(jar|tar.gz)/ {if (LAST) { print LAST; } LAST=""}')
do
Expand All @@ -62,7 +63,7 @@ do
current_time=$(date "+%Y.%m.%d-%H.%M.%S")
echo "Current Time : $current_time"

local_path="$tmpdir/hdfs_tar_files.${current_time}"
local_path="$backup_dir/hdfs_tar_files.${current_time}"

rm -r -f $local_path
mkdir -p $local_path
Expand All @@ -78,7 +79,7 @@ do
touch -d "$d" $local_path/mark
touch -d "$d" $local_path/*

$delete_jndi $local_path
$delete_jndi $local_path $backup_dir

changed=()
for f in $(ls $local_path); do
Expand All @@ -99,7 +100,7 @@ do
echo "No files found. Skipping directory"
fi

local_path="$tmpdir/hdfs_tar_files.${current_time}"
local_path="$backup_dir/hdfs_tar_files.${current_time}"

rm -r -f $local_path
mkdir -p $local_path
Expand All @@ -111,7 +112,7 @@ do
set -e

if [ $ec -eq 0 ]; then
hdfs_bc_path="$tmpdir/backup.${current_time}"
hdfs_bc_path="$backup_dir/backup.${current_time}"

echo "Taking a backup of HDFS dir $hdfs_file_path to $hdfs_bc_path"
$user_option hdfs dfs -mkdir -p $hdfs_bc_path
Expand All @@ -127,7 +128,7 @@ do
local_full_path="${local_path}/${f}"

echo "Executing the log4j removal script"
$patch_tgz $local_full_path
$patch_tgz $local_full_path $backup_dir

echo "Completed executing log4j removal script and uploading $f to $hdfs_file_path"
$user_option hdfs dfs -copyFromLocal -f $local_full_path $hdfs_file_path/$f
Expand Down
7 changes: 4 additions & 3 deletions hdp_support_scripts/patch_tgz.sh
100755 → 100644
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
set -eu -o pipefail

BASEDIR=$(dirname "$0")
backup_dir=$2

delete_jndi=$BASEDIR/delete_jndi.sh
if [ ! -f "$delete_jndi" ]; then
Expand All @@ -25,9 +26,9 @@ if [ ! -f "$tarfile" ]; then
exit 1
fi

backupdir=${2:-/opt/cloudera/log4shell-backup}
mkdir -p "$backupdir/$(dirname $tarfile)"
targetbackup="$backupdir/$tarfile.backup"
#backupdir=${2:-/opt/cloudera/log4shell-backup}
mkdir -p "$backup_dir/$(dirname $tarfile)"
targetbackup="$backup_dir/$tarfile.backup"
if [ ! -f "$targetbackup" ]; then
echo "Backing up to '$targetbackup'"
cp -f "$tarfile" "$targetbackup"
Expand Down
18 changes: 10 additions & 8 deletions hdp_support_scripts/scan_jndi.sh
100755 → 100644
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@ pattern=JndiLookup.class
pattern_15=ClassArbiter.class
pattern_16="MessagePatternConverter\$LookupMessagePatternConverter.class"

tmpdir=${TMPDIR:-/tmp}
mkdir -p $tmpdir
echo "Using tmp directory '$tmpdir'"
backup_dir=$2

#tmpdir=${TMPDIR:-/tmp}
mkdir -p $backup_dir
echo "Using tmp directory '$backup_dir'"

if ! command -v unzip &> /dev/null; then
echo "unzip not found. unzip is required to run this script."
Expand Down Expand Up @@ -51,13 +53,13 @@ do

# Is this jar in jar (uber-jars)?
if unzip -l $jarfile | grep -v 'Archive:' | grep '\.jar$' >/dev/null; then
rm -r -f $tmpdir/unzip_target
mkdir $tmpdir/unzip_target
rm -r -f $backup_dir/unzip_target
mkdir $backup_dir/unzip_target
set +e
unzip -qq $jarfile -d $tmpdir/unzip_target
unzip -qq $jarfile -d $backup_dir/unzip_target
set -e

for f in $(grep -l $pattern $(find $tmpdir/unzip_target -name "*.jar")); do
for f in $(grep -l $pattern $(find $backup_dir/unzip_target -name "*.jar")); do
if grep -q $pattern_15 $f; then
if grep -q $pattern_16 $f; then
echo "Fixed **2.15** version of Log4j-core found in '$f' within '$jarfile'"
Expand All @@ -68,7 +70,7 @@ do
echo "Vulnerable version of Log4j-core found in '$f' within '$jarfile'"
fi
done
rm -r -f $tmpdir/unzip_target
rm -r -f $backup_dir/unzip_target
fi
done

Expand Down