Permalink
Find file
Fetching contributors…
Cannot retrieve contributors at this time
114 lines (95 sloc) 4.84 KB
  1. Assumes:
  2. * hadoop is installed from the CDH distribution.
  3. * jardiff is installed: apt-get install jardiff
  4. * these instructions use colordiff, install it or just say ‘diff’
    #
  1. The target version of hadoop
    version=0.20.2+228
  2. The system-installed hadoop
    orig_dir=/usr/lib/hadoop-0.20
  3. Where to store the patched version
    dist_dir=/home/hadoop/dist/hadoop-${version}-patched
  4. Where patches live
    patch_dir=/home/hadoop/dist/hadoop-patches
  5. Where to put a build
    build_dir=/usr/local/src/hadoop-src/build/hadoop-${version}
  1. We’ll land the patched files into a directory under /home/hadoop/dist. This
  2. lets each machine update from the NFS
    sudo mkdir -p /home/hadoop/dist ; chgrp -R admin /home/hadoop/dist ; chmod -R g+rwX /home/hadoop/dist

#

  1. Get the source distro, expand it
    #
    cd /usr/local/src
    wget http://archive.cloudera.com/cdh/3/hadoop-${version}.tar.gz
    tar xvzf hadoop-${version}.tar.gz
    ln nsf hadoop${version} hadoop-src
  1. We’ll record a diff of the install dir with itself, to detect any changes later
    ( cd $orig_dir ; for foo in `find -iname ‘*.jar’` ; do echo $foo ; jardiff -f $orig_dir/$foo -t $foo -o html ; done ) > /usr/local/src/hadoop-jardiff-orig.log &
  1. Bring it under git
    echo -e “/build\n*.orig\n*.rej\n*~\n.DS_Store\nIcon?\n.svn\n.bzr” > .gitignore
    git init
    git add .
    git ci -m “initial commit”
    git gc # compact
  1. Build with no changes
    ant bin-package -Dupdate=true -Dversion=$version
  1. Verify nothing important has changed.
  2. I had a few minor pathnames change in doc conf files, no biggie.
    git diff HEAD
  3. a random license file here and there
    colordiff -ruw $orig_dir $build_dir —exclude={webapps,conf,pids,logs,.jar,libhadoop.[as]}
  1. use jardiff to see what has actually changed.
    ( cd $build_dir ; for foo in `find -iname ‘*.jar’` ; do echo $foo ; jardiff -f $orig_dir/$foo -t $foo -o html ; done ) > /usr/local/src/hadoop-jardiff-build.log &
    colordiff -ruw /usr/local/src/hadoop-jardiff-orig.log /usr/local/src/hadoop-jardiff-build.log
<notextile>
--- /usr/local/src/hadoop-jardiff-orig.log	2010-06-06 20:11:32.000000000 +0000
+++ /usr/local/src/hadoop-jardiff-build.log	2010-06-06 20:18:26.000000000 +0000
@@ -670,6 +670,16 @@
 </head>
 <body>
 <h2>Comparing hadoop-0.20.2+228-test.jar to hadoop-0.20.2+228-test.jar</h2>
+<h3>Changed classes</h3>
+<ul class="class-changed">
+<li>
+<h4>org.apache.hadoop.fs.s3native.NativeS3FileSystemContractBaseTest</h4>
+<h5>Added:</h5>
+<ul>
+<li>public void testRetryOnIoException() throws java.lang.Exception</li>
+</ul>
+</li>
+</ul>
 <p>API diff generated by <a href="http://www.osjava.org/jardiff/">JarDiff</a>
 </p>
 </body>
</notextile>

#

  1. Now apply your patches and build
    #
    for $patchfile in $patch_dir/*.diff ; do patch -p0 < $patchfile ; done

  2. ant bin-package -Dupdate=true -Dversion=$version
  1. Verify nothing important has changed.
  2. I had a few minor pathnames change in doc conf files, no biggie.
    git diff HEAD
  3. a random license file here and there
    colordiff -ruw $orig_dir $build_dir —exclude={webapps,conf,pids,logs,.jar,libhadoop.[as]}
  1. use jardiff to see what has actually changed.
    ( cd $build_dir ; for foo in `find -iname ‘*.jar’` ; do echo $foo ; jardiff -f $orig_dir/$foo -t $foo -o html ; done ) > /usr/local/src/hadoop-jardiff-build.log &
    colordiff -ruw /usr/local/src/hadoop-jardiff-orig.log /usr/local/src/hadoop-jardiff-build.log
  2. (this may not be as reliable as I thought. You can also use the file times to see what ant updated:)
    ls -ltr `find $build_dir -type f -newer $build_dir/hadoop-0.20.2+228-examples.jar`
  1. copy to the dist dir
    sudo rsync -rlvit $build_dir/ $dist_dir/ —exclude={conf,logs,pids}
    sudo chown -R root:admin $dist_dir/ ; sudo chmod og-w $dist_dir/

sudo rsync -alvi —dry-run $dist_dir/ $orig_dir/ —exclude={conf,logs,pids}

  1. get a list of every node
    . ~/.hadoop-ec2/aws_private_setup.sh ; hadoop-ec2 list gibbon > /tmp/gibbon-hosts.log ; cat /tmp/gibbon-hosts.log | cut -f4 > /tmp/gibbon-slaves.log
  1. on the master node
    for foo in `cat /tmp/gibbon-slaves.log` ; do echo $foo ; sshkp $foo “for foo in chef-client hadoop-0.20-{namenode,secondarynamenode,jobtracker,tasktracker,datanode} ; do sudo service $foo stop ; done” & done
  1. stop the demons
    for foo in `cat /tmp/gibbon-slaves.log` ; do echo $foo ; sshkp $foo ‘for svc in chef-client hadoop-0.20-{tasktracker,datanode} ; do sudo service $svc stop ; done’ & done
  2. Move the new code into place. I wouldn’t do this on more than a couple dozen machines at once, unless you like NFS failures
    for foo in `cat /tmp/gibbon-slaves.log` ; do echo $foo ; sshkp $foo ‘sudo rsync -alvi /home/hadoop/dist/hadoop-0.20.2+228-patched/ /usr/lib/hadoop-0.20/ —exclude=conf —exclude=logs —exclude=pids’ & done
  3. restart
    for foo in `cat /tmp/gibbon-slaves.log` ; do echo $foo ; sshkp $foo ‘for svc in hadoop-0.20-{tasktracker,datanode} ; do sudo service $svc start ; done’ & done

sudo -u hadoop hadoop fsck / | egrep -v ‘^\.+$’