Index: C:/Dokumente und Einstellungen/chef/workspace/mapred/conf/nutch-default.xml
===================================================================
--- C:/Dokumente und Einstellungen/chef/workspace/mapred/conf/nutch-default.xml (revision 307440)
+++ C:/Dokumente und Einstellungen/chef/workspace/mapred/conf/nutch-default.xml (working copy)
@@ -360,6 +360,12 @@
+ ndfs.datanode
+ 50010
+ The port that the ndfs datanode server runs at.
+
+
+
ndfs.name.dir
/tmp/nutch/ndfs/name
Determines where on the local filesystem the NDFS name node
@@ -387,6 +393,27 @@
+ mapred.job.tracker.info.port
+ 50030
+ The port that the MapReduce job tracker info webserver runs at.
+
+
+
+
+ mapred.task.tracker.output.port
+ 50040
+ The port that the MapReduce task tracker output server runs at.
+
+
+
+
+ mapred.task.tracker.report.port
+ 50050
+ The port that the MapReduce task tracker report server runs at.
+
+
+
+
mapred.local.dir
/tmp/nutch/mapred/local
The local directory where MapReduce stores intermediate
Index: C:/Dokumente und Einstellungen/chef/workspace/mapred/src/java/org/apache/nutch/ndfs/DataNode.java
===================================================================
--- C:/Dokumente und Einstellungen/chef/workspace/mapred/src/java/org/apache/nutch/ndfs/DataNode.java (revision 291588)
+++ C:/Dokumente und Einstellungen/chef/workspace/mapred/src/java/org/apache/nutch/ndfs/DataNode.java (working copy)
@@ -66,6 +66,7 @@
Vector receivedBlockList = new Vector();
int xmitsInProgress = 0;
Daemon dataXceiveServer = null;
+ private NutchConf fConf;
/**
* Create given a configuration and a dataDir.
@@ -73,18 +74,19 @@
public DataNode(NutchConf conf, String datadir) throws IOException {
this(InetAddress.getLocalHost().getHostName(),
new File(datadir),
- createSocketAddr(conf.get("fs.default.name", "local")));
+ createSocketAddr(conf.get("fs.default.name", "local")), conf);
+
}
/**
* Needs a directory to find its data (and config info)
*/
- public DataNode(String machineName, File datadir, InetSocketAddress nameNodeAddr) throws IOException {
+ public DataNode(String machineName, File datadir, InetSocketAddress nameNodeAddr, NutchConf conf) throws IOException {
this.namenode = (DatanodeProtocol) RPC.getProxy(DatanodeProtocol.class, nameNodeAddr);
this.data = new FSDataset(datadir);
ServerSocket ss = null;
- int tmpPort = 7000;
+ int tmpPort = conf.getInt("ndfs.datanode", 50010);
while (ss == null) {
try {
ss = new ServerSocket(tmpPort);
Index: C:/Dokumente und Einstellungen/chef/workspace/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java
===================================================================
--- C:/Dokumente und Einstellungen/chef/workspace/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java (revision 307441)
+++ C:/Dokumente und Einstellungen/chef/workspace/mapred/src/java/org/apache/nutch/mapred/TaskTracker.java (working copy)
@@ -65,17 +65,20 @@
NutchFileSystem fs = null;
static final String SUBDIR = "taskTracker";
+ private NutchConf fConf;
+
/**
* Start with the local machine name, and the default JobTracker
*/
- public TaskTracker() throws IOException {
- this(JobTracker.getAddress(NutchConf.get()));
- }
+ public TaskTracker(NutchConf conf) throws IOException {
+ this(JobTracker.getAddress(conf), conf);
+ }
/**
* Start with the local machine name, and the addr of the target JobTracker
*/
- public TaskTracker(InetSocketAddress jobTrackAddr) throws IOException {
+ public TaskTracker(InetSocketAddress jobTrackAddr, NutchConf conf) throws IOException {
+ this.fConf = conf;
this.jobTrackAddr = jobTrackAddr;
initialize();
}
@@ -95,17 +98,32 @@
this.tasks = new TreeMap();
this.runningTasks = new TreeMap();
- // generate port numbers
- this.taskReportPort = 32768+r.nextInt(32768);
- this.mapOutputPort = 32768+r.nextInt(32768);
+ // port numbers
+ this.taskReportPort = this.fConf.getInt("mapred.task.tracker.report.port", 50050);
+ this.mapOutputPort = this.fConf.getInt("mapred.task.tracker.output.port", 50040);
// RPC initialization
- this.taskReportServer =
- RPC.getServer(this, taskReportPort, MAX_CURRENT_TASKS, false);
- this.taskReportServer.start();
- this.mapOutputServer =
- RPC.getServer(this, mapOutputPort, MAX_CURRENT_TASKS, false);
- this.mapOutputServer.start();
+ while (true) {
+ try {
+ this.taskReportServer = RPC.getServer(this, this.taskReportPort, MAX_CURRENT_TASKS, false);
+ this.taskReportServer.start();
+ break;
+ } catch (Exception e) {
+ LOG.info("Could not open report server at " + this.taskReportPort + ", trying new port");
+ this.taskReportPort++;
+ }
+
+ }
+ while (true) {
+ try {
+ this.mapOutputServer = RPC.getServer(this, this.mapOutputPort, MAX_CURRENT_TASKS, false);
+ this.mapOutputServer.start();
+ break;
+ } catch (Exception e) {
+ LOG.info("Could not open mapoutput server at " + this.mapOutputPort + ", trying new port");
+ this.mapOutputPort++;
+ }
+ }
// Clear out temporary files that might be lying around
MapOutputFile.cleanupStorage();
@@ -613,7 +631,7 @@
System.exit(-1);
}
- TaskTracker tt = new TaskTracker();
+ TaskTracker tt = new TaskTracker(NutchConf.get());
tt.run();
}
}
Index: C:/Dokumente und Einstellungen/chef/workspace/mapred/src/java/org/apache/nutch/mapred/JobTracker.java
===================================================================
--- C:/Dokumente und Einstellungen/chef/workspace/mapred/src/java/org/apache/nutch/mapred/JobTracker.java (revision 307441)
+++ C:/Dokumente und Einstellungen/chef/workspace/mapred/src/java/org/apache/nutch/mapred/JobTracker.java (working copy)
@@ -226,7 +226,7 @@
LOG.info("Property '" + key + "' is " + val);
}
- this.infoPort = conf.getInt("mapred.job.tracker.info.port", 7845);
+ this.infoPort = conf.getInt("mapred.job.tracker.info.port", 50030);
this.infoServer = new JobTrackerInfoServer(this, infoPort);
this.infoServer.start();