From 87c29ec5846935367ac4d414b5e12002e69f1974 Mon Sep 17 00:00:00 2001 From: Tom White Date: Thu, 9 Jan 2014 14:39:28 +0000 Subject: [PATCH] Upgrade from Hadoop 1.1.2 to 1.2.1. --- README | 4 +- .../.job_20120528201803683_0001.xml.crc | Bin 280 -> 0 bytes .../jobTracker/job_20120528201803683_0001.xml | 277 ------------------ hadoop-meta/pom.xml | 4 +- 4 files changed, 4 insertions(+), 281 deletions(-) delete mode 100644 ch05/build/test/mapred/local/jobTracker/.job_20120528201803683_0001.xml.crc delete mode 100755 ch05/build/test/mapred/local/jobTracker/job_20120528201803683_0001.xml diff --git a/README b/README index 6f3ad6e5..00912123 100644 --- a/README +++ b/README @@ -9,7 +9,7 @@ for the first edition at http://github.com/tomwhite/hadoop-book/tree/1e, and for the second edition at http://github.com/tomwhite/hadoop-book/tree/2e. This version of the code has been tested with: - * Hadoop 1.1.1/0.22.0/0.23.0/2.0.2-alpha + * Hadoop 1.2.1/0.22.0/0.23.0/2.0.2-alpha * Avro 1.5.4 * Pig 0.9.1 * Hive 0.8.0 @@ -25,7 +25,7 @@ You also need to install Maven. Then you can build the code with: -% mvn package -DskipTests -Dhadoop.version=1.1.1 +% mvn package -DskipTests -Dhadoop.version=1.2.1 (Note that only the Hadoop version numbers specified in hadoop-meta/pom.xml can be used here, not any version of Hadoop.) diff --git a/ch05/build/test/mapred/local/jobTracker/.job_20120528201803683_0001.xml.crc b/ch05/build/test/mapred/local/jobTracker/.job_20120528201803683_0001.xml.crc deleted file mode 100644 index 0fd9a53a20812619e11c4be1107d9622478b6153..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 280 zcmV+z0q6c>a$^7h00IE2#KHi376h|w$Zn?Dx!o1wG$d*AIa!svL2qO#7SW&DmU5+L zHp{b(7?+V80SBd)?GVr_dpZWP=N6A(O)7Cd>KuCh#ag!hoXH)D&XEL|>pj`$pXStw zdJ^Np*>28NXUrT5)A{%DKA92~koTViJV>p^86|J!R5n%XFM`Aw03k6=_ijG63ND(_ z$@$YkmW(jdNi4Q34W|KdoLs<$CW&GI diff --git a/ch05/build/test/mapred/local/jobTracker/job_20120528201803683_0001.xml b/ch05/build/test/mapred/local/jobTracker/job_20120528201803683_0001.xml deleted file mode 100755 index e2b25531..00000000 --- a/ch05/build/test/mapred/local/jobTracker/job_20120528201803683_0001.xml +++ /dev/null @@ -1,277 +0,0 @@ - -fs.s3n.implorg.apache.hadoop.fs.s3native.NativeS3FileSystem -mapreduce.tasktracker.tasks.sleeptimebeforesigkill5000 -dfs.datanode.data.dirfile://${hadoop.tmp.dir}/dfs/data -dfs.namenode.checkpoint.size67108864 -mapreduce.reduce.speculativetrue -map.sort.classorg.apache.hadoop.util.QuickSort -hadoop.tmp.dir/tmp/hadoop-${user.name} -dfs.namenode.decommission.nodes.per.interval5 -mapreduce.map.skip.proc.count.autoincrtrue -ipc.client.idlethreshold4000 -mapreduce.map.maxattempts4 -mapreduce.jobtracker.heartbeats.in.second100 -dfs.datanode.data.dir.perm700 -mapreduce.tasktracker.http.address0.0.0.0:50060 -dfs.namenode.checkpoint.dirfile://${hadoop.tmp.dir}/dfs/namesecondary -kfs.blocksize67108864 -dfs.bytes-per-checksum512 -dfs.namenode.support.allow.formattrue -fs.trash.checkpoint.interval0 -dfs.namenode.https-address0.0.0.0:50470 -dfs.datanode.directoryscan.threads1 -dfs.datanode.address0.0.0.0:50010 -dfs.namenode.logging.levelinfo -dfs.block.access.token.enablefalse -io.skip.checksum.errorsfalse -mapreduce.reduce.input.buffer.percent0.0 -mapreduce.tasktracker.instrumentationorg.apache.hadoop.mapred.TaskTrackerMetricsInst -mapred.reducer.new-apitrue -fs.har.impl.disable.cachetrue -mapreduce.map.log.levelINFO -dfs.namenode.handler.count10 -dfs.blockreport.initialDelay0 -fs.defaultFShdfs://localhost:60743 -mapreduce.job.map.classv3.MaxTemperatureMapper -dfs.namenode.name.dirfile://${hadoop.tmp.dir}/dfs/name -dfs.namenode.replication.interval3 -mapreduce.task.merge.progress.records10000 -hadoop.rpc.socket.factory.class.defaultorg.apache.hadoop.net.StandardSocketFactory -dfs.namenode.http-address0.0.0.0:50070 -fs.hdfs.implorg.apache.hadoop.hdfs.DistributedFileSystem -mapred.queue.default.acl-administer-jobs* -mapreduce.tasktracker.healthchecker.script.timeout600000 -mapreduce.tasktracker.http.threads40 -mapreduce.job.jvm.numtasks1 -kfs.replication3 -dfs.block.access.key.update.interval600 -mapreduce.job.complete.cancel.delegation.tokenstrue -io.mapfile.bloom.size1048576 -hadoop.rpc.protectionauthentication -mapreduce.reduce.shuffle.connect.timeout180000 -dfs.namenode.replication.considerLoadtrue -fs.ftp.implorg.apache.hadoop.fs.ftp.FTPFileSystem -mapreduce.job.speculative.slownodethreshold1.0 -file.stream-buffer-size4096 -mapreduce.job.reduces1 -dfs.namenode.safemode.threshold-pct0.999f -dfs.https.server.keystore.resourcessl-server.xml -dfs.namenode.replication.min1 -mapreduce.job.combine.classv1.MaxTemperatureReducer -dfs.namenode.safemode.min.datanodes0 -dfs.client.https.keystore.resourcessl-client.xml -dfs.block.access.token.lifetime600 -mapreduce.job.speculative.slowtaskthreshold1.0 -fs.ftp.host.port21 -kfs.client-write-packet-size65536 -mapreduce.tasktracker.healthchecker.interval60000 -kfs.bytes-per-checksum512 -mapreduce.reduce.merge.inmem.threshold1000 -hadoop.security.group.mappingorg.apache.hadoop.security.ShellBasedUnixGroupsMapping -mapreduce.client.output.filterALL -mapreduce.job.maps1 -s3.replication3 -dfs.namenode.backup.address0.0.0.0:50100 -net.topology.node.switch.mapping.implorg.apache.hadoop.net.ScriptBasedMapping -fs.s3.buffer.dir${hadoop.tmp.dir}/s3 -fs.file.implorg.apache.hadoop.fs.LocalFileSystem -s3native.bytes-per-checksum512 -mapreduce.jobtracker.persist.jobstatus.hours1 -dfs.datanode.ipc.address0.0.0.0:50020 -dfs.namenode.checkpoint.edits.dir${dfs.namenode.checkpoint.dir} -dfs.datanode.balance.bandwidthPerSec1048576 -s3.client-write-packet-size65536 -dfs.image.compression.codecorg.apache.hadoop.io.compress.DefaultCodec -mapreduce.cluster.reducememory.mb-1 -mapreduce.reduce.shuffle.merge.percent0.66 -io.mapfile.bloom.error.rate0.005 -ftp.bytes-per-checksum512 -dfs.namenode.accesstime.precision3600000 -dfs.datanode.https.address0.0.0.0:50475 -mapreduce.map.sort.spill.percent0.80 -mapreduce.output.fileoutputformat.compression.typeRECORD -mapreduce.job.committer.setup.cleanup.neededtrue -dfs.replication.max512 -mapreduce.client.submit.file.replication10 -hadoop.security.authorizationfalse -s3.bytes-per-checksum512 -dfs.namenode.checkpoint.period3600 -dfs.namenode.delegation.token.renew-interval86400000 -dfs.namenode.secondary.http-address0.0.0.0:50090 -mapreduce.jobtracker.tasktracker.maxblacklists4 -mapred.child.java.opts-Xmx200m -mapreduce.jobtracker.persist.jobstatus.activetrue -mapreduce.tasktracker.local.dir.minspacekill0 -mapreduce.jobtracker.webinterface.trustedfalse -mapreduce.cluster.local.dir${hadoop.tmp.dir}/mapred/local -ipc.server.listen.queue.size128 -dfs.permissions.enabledtrue -mapreduce.job.queuenamedefault -dfs.namenode.backup.http-address0.0.0.0:50105 -dfs.namenode.safemode.extension30000 -s3native.blocksize67108864 -mapreduce.jobtracker.retiredjobs.cache.size1000 -dfs.datanode.max.transfer.threads4096 -mapreduce.job.speculative.speculativecap0.1 -file.replication1 -mapreduce.map.skip.maxrecords0 -mapreduce.jobtracker.http.address127.0.0.1:60756 -ftp.client-write-packet-size65536 -hadoop.work.around.non.threadsafe.getpwuidfalse -mapreduce.job.userlog.retain.hours24 -dfs.datanode.dns.nameserverdefault -hadoop.util.hash.typemurmur -mapreduce.tasktracker.taskmemorymanager.monitoringinterval5000 -dfs.blockreport.intervalMsec21600000 -dfs.client.https.need-authfalse -mapreduce.job.acl-view-job -mapreduce.map.speculativetrue -io.map.index.interval128 -dfs.client.block.write.retries3 -mapreduce.reduce.shuffle.read.timeout180000 -mapreduce.input.fileinputformat.split.minsize0 -dfs.stream-buffer-size4096 -mapreduce.map.output.compressfalse -dfs.https.enablefalse -mapreduce.job.output.value.classorg.apache.hadoop.io.IntWritable -mapreduce.tasktracker.dns.interfacedefault -dfs.datanode.failed.volumes.tolerated0 -hadoop.kerberos.kinit.commandkinit -mapreduce.output.fileoutputformat.compressfalse -ipc.client.tcpnodelayfalse -mapreduce.cluster.temp.dir${hadoop.tmp.dir}/mapred/temp -fs.AbstractFileSystem.hdfs.implorg.apache.hadoop.fs.Hdfs -mapreduce.tasktracker.local.dir.minspacestart0 -io.map.index.skip0 -ipc.server.tcpnodelayfalse -ftp.replication3 -dfs.namenode.delegation.key.update-interval86400000 -mapreduce.job.working.dirhdfs://localhost:60743/user/tom -mapred.mapper.new-apitrue -mapreduce.reduce.markreset.buffer.percent0.0 -dfs.default.chunk.view.size32768 -hadoop.logfile.size10000000 -file.client-write-packet-size65536 -mapreduce.job.reduce.slowstart.completedmaps0.05 -mapreduce.job.dirhdfs://localhost:60743/tmp/hadoop-tom/mapred/staging/tom/.staging/job_20120528201803683_0001 -mapreduce.tasktracker.outofband.heartbeatfalse -mapreduce.task.io.sort.factor10 -mapreduce.tasktracker.map.tasks.maximum2 -mapreduce.reduce.memory.mb-1 -fs.s3n.block.size67108864 -dfs.datanode.du.reserved0 -net.topology.script.number.args100 -mapreduce.client.progressmonitor.pollinterval1000 -hadoop.security.authenticationsimple -mapreduce.jobtracker.system.dir${hadoop.tmp.dir}/mapred/system -tfile.fs.output.buffer.size262144 -mapreduce.tasktracker.indexcache.mb10 -mapreduce.jobtracker.maxtasks.perjob-1 -dfs.web.ugiwebuser,webgroup -hadoop.security.groups.cache.secs300 -mapreduce.jobtracker.jobhistory.lru.cache.size5 -mapreduce.map.memory.mb-1 -dfs.datanode.directoryscan.interval21600 -fs.AbstractFileSystem.file.implorg.apache.hadoop.fs.local.LocalFs -mapreduce.reduce.skip.maxgroups0 -mapreduce.task.profilefalse -kfs.stream-buffer-size4096 -fs.s3.maxRetries4 -dfs.datanode.dns.interfacedefault -mapreduce.task.skip.start.attempts2 -mapreduce.task.tmp.dir./tmp -file.blocksize67108864 -fs.ftp.host0.0.0.0 -file.bytes-per-checksum512 -dfs.image.compressfalse -mapreduce.job.acl-modify-job -fs.hftp.implorg.apache.hadoop.hdfs.HftpFileSystem -mapreduce.job.submithostnameClouderas-MacBook-Pro-3.local -mapreduce.task.profile.maps0-2 -mapreduce.jobtracker.expire.trackers.interval600000 -fs.trash.interval0 -fs.s3.sleepTimeSeconds10 -fs.har.implorg.apache.hadoop.fs.HarFileSystem -mapreduce.cluster.mapmemory.mb-1 -mapreduce.reduce.maxattempts4 -mapreduce.client.completion.pollinterval5000 -mapreduce.tasktracker.report.address127.0.0.1:0 -dfs.namenode.decommission.interval30 -dfs.namenode.max.objects0 -ftp.stream-buffer-size4096 -mapreduce.job.output.key.classorg.apache.hadoop.io.Text -dfs.heartbeat.interval3 -mapreduce.job.reduce.classv1.MaxTemperatureReducer -mapreduce.tasktracker.taskcontrollerorg.apache.hadoop.mapred.DefaultTaskController -io.seqfile.sorter.recordlimit1000000 -s3.blocksize67108864 -mapreduce.job.nameMax temperature -dfs.datanode.http.address0.0.0.0:50075 -mapreduce.input.fileinputformat.inputdirhdfs://localhost:60743/user/tom/input -mapreduce.jobtracker.restart.recoverfalse -dfs.image.transfer.bandwidthPerSec0 -mapreduce.task.profile.reduces0-2 -s3native.stream-buffer-size4096 -dfs.namenode.edits.dir${dfs.namenode.name.dir} -mapreduce.output.fileoutputformat.outputdiroutput -mapreduce.output.fileoutputformat.compression.codecorg.apache.hadoop.io.compress.DefaultCodec -mapreduce.job.end-notification.retry.attempts0 -fs.kfs.implorg.apache.hadoop.fs.kfs.KosmosFileSystem -dfs.replication3 -mapreduce.jobtracker.jobhistory.block.size3145728 -mapreduce.jobtracker.addresslocalhost:60755 -mapreduce.task.timeout600000 -mapreduce.input.fileinputformat.numinputfiles1 -mapreduce.reduce.shuffle.input.buffer.percent0.70 -dfs.client-write-packet-size65536 -mapreduce.job.split.metainfo.maxsize10000000 -hadoop.logfile.count10 -fs.automatic.closetrue -mapreduce.job.submithostaddress192.168.2.17 -s3.stream-buffer-size4096 -io.seqfile.compress.blocksize1000000 -mapreduce.jobtracker.maxmapmemory.mb-1 -fs.s3.block.size67108864 -mapreduce.jobtracker.taskcache.levels2 -mapreduce.jobtracker.staging.root.dir${hadoop.tmp.dir}/mapred/staging -ftp.blocksize67108864 -fs.hsftp.implorg.apache.hadoop.hdfs.HsftpFileSystem -mapreduce.job.maxtaskfailures.per.tracker4 -mapreduce.tasktracker.reduce.tasks.maximum2 -io.seqfile.lazydecompresstrue -mapreduce.jobtracker.handler.count10 -mapreduce.jobtracker.persist.jobstatus.dir/jobtracker/jobsInfo -hadoop.common.configuration.version0.22.0 -dfs.permissions.superusergroupsupergroup -mapreduce.cluster.acls.enabledfalse -mapreduce.reduce.skip.proc.count.autoincrtrue -ipc.client.connection.maxidletime10000 -mapreduce.job.end-notification.retry.interval30000 -mapreduce.jobtracker.instrumentationorg.apache.hadoop.mapred.JobTrackerMetricsInst -mapreduce.tasktracker.dns.nameserverdefault -ipc.client.kill.max10 -ipc.client.connect.max.retries10 -hadoop.hdfs.configuration.version1 -fs.s3.implorg.apache.hadoop.fs.s3.S3FileSystem -dfs.blocksize67108864 -io.seqfile.local.dir${hadoop.tmp.dir}/io/local -mapreduce.jobtracker.maxreducememory.mb-1 -io.native.lib.availabletrue -io.file.buffer.size4096 -io.serializationsorg.apache.hadoop.io.serializer.WritableSerialization,org.apache.hadoop.io.serializer.avro.AvroSpecificSerialization,org.apache.hadoop.io.serializer.avro.AvroReflectSerialization -dfs.datanode.handler.count3 -mapreduce.task.userlog.limit.kb0 -tfile.fs.input.buffer.size262144 -mapreduce.reduce.shuffle.parallelcopies5 -mapreduce.task.io.sort.mb100 -mapreduce.reduce.log.levelINFO -mapreduce.map.output.compress.codecorg.apache.hadoop.io.compress.DefaultCodec -fs.df.interval60000 -dfs.namenode.delegation.token.max-lifetime604800000 -mapreduce.task.files.preserve.failedtasksfalse -s3native.client-write-packet-size65536 -s3native.replication3 -io.compression.codecsorg.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec -mapreduce.jobtracker.taskschedulerorg.apache.hadoop.mapred.JobQueueTaskScheduler -tfile.io.chunk.size1048576 - \ No newline at end of file diff --git a/hadoop-meta/pom.xml b/hadoop-meta/pom.xml index 8ffdf457..36afa64e 100644 --- a/hadoop-meta/pom.xml +++ b/hadoop-meta/pom.xml @@ -32,11 +32,11 @@ A module which allows the Hadoop dependencies to be specified by a Maven profile hadoop.version - 1.1.2 + 1.2.1 - 1.1.2 + 1.2.1