hadoop-ID的示例分析
小编给大家分享一下hadoop-ID的示例分析，相信大部分人都还不怎么了解，因此分享这篇文章给大家参考一下，希望大家阅读完这篇文章后大有收获，下面让我们一起去了解一下吧！
我们开始来分析Hadoop MapReduce的内部的运行机制。用户向Hadoop提交Job（作业），作业在JobTracker对象的控制下执行。Job被分解成为Task（任务），分发到集群中，在TaskTracker的控制下运行。Task包括MapTask和ReduceTask，是MapReduce的Map操作和Reduce操作执行的地方。这中任务分布的方法比较类似于HDFS中NameNode和DataNode的分工，NameNode对应的是JobTracker，DataNode对应的是TaskTracker。JobTracker，TaskTracker和MapReduce的客户端通过RPC通信，具体可以参考HDFS部分的分析。
我们先来分析一些辅助类，首先是和ID有关的类，ID的继承树如下：
* Licensed to the Apache Software Foundation (ASF) under onepackage org.apache.hadoop.mapreduce;import java.io.DataInput;/** * A general identifier, which internally stores the id * as an integer. This is the super class of {@link JobID}, * {@link TaskID} and {@link TaskAttemptID}. * * @see JobID * @see TaskID * @see TaskAttemptID */public abstract class ID implements WritableComparable { protected static final char SEPARATOR = '_'; protected int id; /** constructs an ID object from the given int */ public ID(int id) { this.id = id; } protected ID() { } /** returns the int which represents the identifier */ public int getId() { return id; } @Override public String toString() { return String.valueOf(id); } @Override public int hashCode() { return Integer.valueOf(id).hashCode(); } @Override public boolean equals(Object o) { if (this == o) return true; if(o == null) return false; if (o.getClass() == this.getClass()) { ID that = (ID) o; return this.id == that.id; } else return false; } /** Compare IDs by associated numbers */ public int compareTo(ID that) { return this.id - that.id; } public void readFields(DataInput in) throws IOException { this.id = in.readInt(); } public void write(DataOutput out) throws IOException { out.writeInt(id); } } * Licensed to the Apache Software Foundation (ASF) under onepackage org.apache.hadoop.mapreduce;import java.io.DataInput;/** * JobID represents the immutable and unique identifier for * the job. JobID consists of two parts. First part * represents the jobtracker identifier, so that jobID to jobtracker map * is defined. For cluster setup this string is the jobtracker * start time, for local setting, it is "local". * Second part of the JobID is the job number.
* An example JobID is : *job_200707121733_0003, which represents the third job * running at the jobtracker started at200707121733. ** Applications should never construct or parse JobID strings, but rather * use appropriate constructors or {@link #forName(String)} method. * * @see TaskID * @see TaskAttemptID * @see org.apache.hadoop.mapred.JobTracker#getNewJobId() * @see org.apache.hadoop.mapred.JobTracker#getStartTime() */public class JobID extends org.apache.hadoop.mapred.ID implements Comparable
{ protected static final String JOB = "job"; private final Text jtIdentifier; protected static final NumberFormat idFormat = NumberFormat.getInstance(); static { idFormat.setGroupingUsed(false); idFormat.setMinimumIntegerDigits(4); } /** * Constructs a JobID object * @param jtIdentifier jobTracker identifier * @param id job number */ public JobID(String jtIdentifier, int id) { super(id); this.jtIdentifier = new Text(jtIdentifier); } public JobID() { jtIdentifier = new Text(); } public String getJtIdentifier() { return jtIdentifier.toString(); } @Override public boolean equals(Object o) { if (!super.equals(o)) return false; JobID that = (JobID)o; return this.jtIdentifier.equals(that.jtIdentifier); } /**Compare JobIds by first jtIdentifiers, then by job numbers*/ @Override public int compareTo(ID o) { JobID that = (JobID)o; int jtComp = this.jtIdentifier.compareTo(that.jtIdentifier); if(jtComp == 0) { return this.id - that.id; } else return jtComp; } /** * Add the stuff after the "job" prefix to the given builder. This is useful, * because the sub-ids use this substring at the start of their string. * @param builder the builder to append to * @return the builder that was passed in */ public StringBuilder appendTo(StringBuilder builder) { builder.append(SEPARATOR); builder.append(jtIdentifier); builder.append(SEPARATOR); builder.append(idFormat.format(id)); return builder; } @Override public int hashCode() { return jtIdentifier.hashCode() + id; } @Override public String toString() { return appendTo(new StringBuilder(JOB)).toString(); } @Override public void readFields(DataInput in) throws IOException { super.readFields(in); this.jtIdentifier.readFields(in); } @Override public void write(DataOutput out) throws IOException { super.write(out); jtIdentifier.write(out); } /** Construct a JobId object from given string * @return constructed JobId object or null if the given String is null * @throws IllegalArgumentException if the given string is malformed */ public static JobID forName(String str) throws IllegalArgumentException { if(str == null) return null; try { String[] parts = str.split("_"); if(parts.length == 3) { if(parts[0].equals(JOB)) { return new org.apache.hadoop.mapred.JobID(parts[1], Integer.parseInt(parts[2])); } } }catch (Exception ex) {//fall below } throw new IllegalArgumentException("JobId string : " + str + " is not properly formed"); } }
以上是"hadoop-ID的示例分析"这篇文章的所有内容，感谢各位的阅读！相信大家都有了一定的了解，希望分享的内容对大家有所帮助，如果还想学习更多知识，欢迎关注行业资讯频道！
