fileQueue) {
+ this.fileQueue = fileQueue;
+ consumerId = numConsumers.incrementAndGet();
+ }
+
+ public IFileProcessorFutureResult call() {
+ currentState = STATE.ACTIVELY_CONSUMING;
+
+ try {
+ FileResource fileResource = getNextFileResource();
+ while (fileResource != null) {
+ logger.debug("file consumer is about to process: " + fileResource.getResourceId());
+ boolean consumed = _processFileResource(fileResource);
+ logger.debug("file consumer has finished processing: " + fileResource.getResourceId());
+
+ if (consumed) {
+ numResourcesConsumed++;
+ }
+ fileResource = getNextFileResource();
+ }
+ } catch (InterruptedException e) {
+ setEndedState(STATE.THREAD_INTERRUPTED);
+ }
+
+ setEndedState(STATE.COMPLETED);
+ return new FileConsumerFutureResult(currentFile, numResourcesConsumed);
+ }
+
+
+ /**
+ * Main piece of code that needs to be implemented. Clients
+ * are responsible for closing streams and handling the exceptions
+ * that they'd like to handle.
+ *
+ * Unchecked throwables can be thrown past this, of course. When an unchecked
+ * throwable is thrown, this logs the error, and then rethrows the exception.
+ * Clients/subclasses should make sure to catch and handle everything they can.
+ *
+ * The design goal is that the whole process should close up and shutdown soon after
+ * an unchecked exception or error is thrown.
+ *
+ * Make sure to call {@link #incrementHandledExceptions()} appropriately in
+ * your implementation of this method.
+ *
+ *
+ * @param fileResource
+ * @return whether or not a file was successfully processed
+ */
+ public abstract boolean processFileResource(FileResource fileResource);
+
+
+ /**
+ * Make sure to call this appropriately!
+ */
+ protected void incrementHandledExceptions() {
+ numHandledExceptions++;
+ }
+
+
+ public boolean isStillActive() {
+ return (! Thread.currentThread().isInterrupted() &&
+ (currentState == STATE.NOT_YET_STARTED ||
+ currentState == STATE.ACTIVELY_CONSUMING));
+ }
+
+ private boolean _processFileResource(FileResource fileResource) {
+ currentFile = new FileStarted(fileResource.getResourceId());
+ boolean consumed = false;
+ try {
+ consumed = processFileResource(fileResource);
+ } catch (RuntimeException e) {
+ setEndedState(STATE.CONSUMER_EXCEPTION);
+ throw e;
+ } catch (Error e) {
+ setEndedState(STATE.CONSUMER_ERROR);
+ throw e;
+ }
+ //if anything is thrown from processFileResource, then the fileStarted
+ //will remain what it was right before the exception was thrown.
+ currentFile = null;
+ return consumed;
+ }
+
+ /**
+ * This politely asks the consumer to shutdown.
+ * Before processing another file, the consumer will check to see
+ * if it has been asked to terminate.
+ *
+ * This offers another method for politely requesting
+ * that a FileResourceConsumer stop processing
+ * besides passing it {@link org.apache.tika.batch.PoisonFileResource}.
+ *
+ */
+ public void pleaseRetire() {
+ setEndedState(STATE.RETIRED);
+ }
+
+ /**
+ * Returns the name and start time of a file that is currently being processed.
+ * If no file is currently being processed, this will return null.
+ *
+ * @return FileStarted or null
+ */
+ public FileStarted getCurrentFile() {
+ return currentFile;
+ }
+
+ public int getNumResourcesConsumed() {
+ return numResourcesConsumed;
+ }
+
+ public int getNumHandledExceptions() {
+ return numHandledExceptions;
+ }
+
+ /**
+ * Checks to see if the currentFile being processed (if there is one)
+ * has gone stale (still being worked on after staleThresholdMillis).
+ *
+ * If the consumer has gone stale, this will return the currentFile and
+ * set the state to GONE_STALE.
+ *
+ * If the consumer was already staled out earlier or
+ * is not processing a file or has been working on a file
+ * for less than #staleThresholdMills, then this will return null.
+ *
+ * @param staleThresholdMillis
+ * @return null or the file started that triggered the stale condition
+ */
+ public FileStarted checkForStaleMillis(long staleThresholdMillis) {
+ //if it isn't actually running, don't bother obtaining lock
+ if (currentState != STATE.ACTIVELY_CONSUMING) {
+ return null;
+ }
+ synchronized(lock) {
+ //check again once the lock has been obtained
+ if (currentState != STATE.ACTIVELY_CONSUMING) {
+ return null;
+ }
+ FileStarted tmp = currentFile;
+ if (tmp == null) {
+ return null;
+ }
+ if (tmp.getElapsedMillis() > staleThresholdMillis) {
+ setEndedState(STATE.GONE_STALE);
+ return tmp;
+ }
+ }
+ return null;
+ }
+
+ private FileResource getNextFileResource() throws InterruptedException {
+ FileResource fileResource = null;
+ long start = new Date().getTime();
+ while (fileResource == null) {
+ //check to see if thread is interrupted before polling
+ if (Thread.currentThread().isInterrupted()) {
+ setEndedState(STATE.THREAD_INTERRUPTED);
+ logger.debug("Consumer thread was interrupted.");
+ break;
+ }
+
+ synchronized(lock) {
+ //need to lock here to prevent race condition with other threads setting state
+ if (currentState != STATE.ACTIVELY_CONSUMING) {
+ logger.debug("Consumer already closed because of: "+ currentState.toString());
+ break;
+ }
+ }
+ fileResource = fileQueue.poll(1L, TimeUnit.SECONDS);
+ if (fileResource != null) {
+ if (fileResource instanceof PoisonFileResource) {
+ setEndedState(STATE.SWALLOWED_POISON);
+ fileResource = null;
+ }
+ break;
+ }
+ logger.debug(consumerId + " is waiting for file and the queue size is: " + fileQueue.size());
+
+ long elapsed = new Date().getTime() - start;
+ if (maxConsecWaitInMillis > 0 && elapsed > maxConsecWaitInMillis) {
+ setEndedState(STATE.EXCEEDED_MAX_CONSEC_WAIT_MILLIS);
+ break;
+ }
+ }
+ return fileResource;
+ }
+
+ protected void close(Closeable closeable) {
+ if (closeable != null) {
+ try {
+ closeable.close();
+ } catch (IOException e){
+ logger.error(e.getMessage());
+ }
+ }
+ closeable = null;
+ }
+
+ protected void flushAndClose(Closeable closeable) {
+ if (closeable == null) {
+ return;
+ }
+ if (closeable instanceof Flushable){
+ try {
+ ((Flushable)closeable).flush();
+ } catch (IOException e) {
+ logger.error(e.getMessage());
+ }
+ }
+ close(closeable);
+ }
+
+ //do not overwrite a finished state. This should
+ //represent the initial cause; all subsequent calls
+ //to set will be ignored!!!
+ private void setEndedState(STATE cause) {
+ synchronized(lock) {
+ if (currentState == STATE.NOT_YET_STARTED ||
+ currentState == STATE.ACTIVELY_CONSUMING) {
+ currentState = cause;
+ }
+ }
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/OutputStreamFactory.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/OutputStreamFactory.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/OutputStreamFactory.java (revision 0)
@@ -0,0 +1,29 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.metadata.Metadata;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+public interface OutputStreamFactory {
+
+ public OutputStream getOutputStream(Metadata metadata) throws IOException;
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/AutoDetectParserFactory.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/AutoDetectParserFactory.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/AutoDetectParserFactory.java (revision 0)
@@ -0,0 +1,34 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.parser.AutoDetectParser;
+import org.apache.tika.parser.Parser;
+
+/**
+ * Simple class for AutoDetectParser
+ */
+public class AutoDetectParserFactory implements ParserFactory {
+
+ @Override
+ public Parser getParser(TikaConfig config) {
+ return new AutoDetectParser(config);
+ }
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/FileStarted.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/FileStarted.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/FileStarted.java (revision 0)
@@ -0,0 +1,116 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Date;
+
+/**
+ * Simple class to record the time when a FileResource's processing started.
+ */
+class FileStarted {
+
+ private final String resourceId;
+ private final long started;
+
+ /**
+ * Initializes a new FileStarted class with {@link #resourceId}
+ * and sets {@link #started} as new Date().getTime().
+ *
+ * @param resourceId
+ */
+ public FileStarted(String resourceId) {
+ this(resourceId, new Date().getTime());
+ }
+
+ public FileStarted(String resourceId, long started) {
+ this.resourceId = resourceId;
+ this.started = started;
+ }
+
+
+ /**
+ * @return id of resource
+ */
+ public String getResourceId() {
+ return resourceId;
+ }
+
+ /**
+ * @return time at which processing on this file started
+ */
+ public long getStarted() {
+ return started;
+ }
+
+ /**
+ * @return elapsed milliseconds this the start of processing of this
+ * file resource
+ */
+ public long getElapsedMillis() {
+ long now = new Date().getTime();
+ return now - started;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result
+ + ((resourceId == null) ? 0 : resourceId.hashCode());
+ result = prime * result + (int) (started ^ (started >>> 32));
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof FileStarted)) {
+ return false;
+ }
+ FileStarted other = (FileStarted) obj;
+ if (resourceId == null) {
+ if (other.resourceId != null) {
+ return false;
+ }
+ } else if (!resourceId.equals(other.resourceId)) {
+ return false;
+ }
+ if (started != other.started) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("FileStarted [resourceId=");
+ builder.append(resourceId);
+ builder.append(", started=");
+ builder.append(started);
+ builder.append("]");
+ return builder.toString();
+ }
+
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/FileResourceCrawler.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/FileResourceCrawler.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/FileResourceCrawler.java (revision 0)
@@ -0,0 +1,224 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.log4j.Logger;
+import org.apache.tika.extractor.DocumentSelector;
+import org.apache.tika.metadata.Metadata;
+
+import java.util.Date;
+import java.util.Iterator;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+public abstract class FileResourceCrawler implements Callable {
+
+ private volatile boolean hasCompletedCrawling = false;
+ private volatile boolean isActive = true;
+ private volatile boolean timedOut = false;
+
+ //how long to pause if can't add to queue
+ private static final long PAUSE_INCREMENT_MILLIS = 1000;
+
+ protected static Logger logger = Logger.getLogger(FileResourceCrawler.class.toString());
+
+ private int maxFilesToAdd = -1;
+ private int maxFilesToConsider = -1;
+
+ private final ArrayBlockingQueue queue;
+ private final int numConsumers;
+
+
+ private long maxConsecWaitInMillis = 300000;//300,000ms = 5 minutes
+ private DocumentSelector documentSelector = null;
+
+ //number of files added to queue
+ private int added = 0;
+ //number of files considered including those that were rejected by documentSelector
+ private int considered = 0;
+
+ /**
+ * @param queue shared queue
+ * @param numConsumers number of consumers (needs to know how many poisons to add when done)
+ */
+ public FileResourceCrawler(ArrayBlockingQueue queue, int numConsumers) {
+ this.queue = queue;
+ this.numConsumers = numConsumers;
+ }
+
+ public abstract void start() throws InterruptedException;
+
+ public FileResourceCrawlerFutureResult call() {
+ try {
+ start();
+ } catch (InterruptedException e) {
+ //this can be triggered by shutdownNow in BatchProcess
+ logger.info("InterruptedException in FileCrawler: " + e.getMessage());
+ } catch (Exception e) {
+ logger.error("Exception in FileCrawler: " + e.getMessage());
+ } finally {
+ isActive = false;
+ }
+
+ try {
+ shutdown();
+ } catch (InterruptedException e) {
+ //swallow
+ }
+
+ return new FileResourceCrawlerFutureResult(considered, added);
+ }
+
+ protected boolean tryToAdd(FileResource fileResource) throws InterruptedException {
+
+ if (maxFilesToAdd > -1 && added >= maxFilesToAdd) {
+ shutdown();
+ }
+
+ boolean isAdded = false;
+ if (select(fileResource.getMetadata())) {
+ long totalConsecutiveWait = 0;
+ while (queue.offer(fileResource, 1L, TimeUnit.SECONDS) == false) {
+
+ logger.info("FileResourceCrawler is pausing. Queue is full: " + queue.size());
+ Thread.sleep(PAUSE_INCREMENT_MILLIS);
+ totalConsecutiveWait += PAUSE_INCREMENT_MILLIS;
+ if (maxConsecWaitInMillis > -1 && totalConsecutiveWait > maxConsecWaitInMillis) {
+ timedOut = true;
+ logger.error("Crawler had to wait longer than max consecutive wait time.");
+ throw new InterruptedException("FileResourceCrawler had to wait longer than max consecutive wait time.");
+ }
+ if (Thread.currentThread().isInterrupted()) {
+ logger.info("FileResourceCrawler shutting down because of interrupted thread.");
+ throw new InterruptedException("FileResourceCrawler interrupted.");
+ }
+ }
+ isAdded = true;
+ added++;
+ } else {
+ logger.debug("crawler did not select: "+fileResource.getResourceId());
+ }
+ considered++;
+ return isAdded;
+ }
+
+ private void shutdown() throws InterruptedException {
+
+ if (hasCompletedCrawling == true) {
+ return;
+ }
+ int i = 0;
+ long start = new Date().getTime();
+ while (queue.offer(new PoisonFileResource(), 1L, TimeUnit.SECONDS) != false) {
+ long elapsed = new Date().getTime() - start;
+ if (maxConsecWaitInMillis > -1 && elapsed > maxConsecWaitInMillis) {
+ logger.error("Crawler timed out while trying to add poison");
+ throw new InterruptedException("FileResourceCrawler timed out while trying to shutdown.");
+ }
+ if (i++ >= numConsumers) {
+ break;
+ }
+
+ }
+ hasCompletedCrawling = true;
+ }
+
+ /**
+ * If the crawler stops for any reason, it is no longer active.
+ *
+ * @return whether crawler is active or not
+ */
+ public boolean isActive() {
+ return isActive;
+ }
+
+ public void setMaxConsecWaitInMillis(long maxConsecWaitInMillis) {
+ this.maxConsecWaitInMillis = maxConsecWaitInMillis;
+ }
+ public void setDocumentSelector(DocumentSelector documentSelector) {
+ this.documentSelector = documentSelector;
+ }
+
+ public int getConsidered() {
+ return considered;
+ }
+
+ protected boolean select(Metadata m) {
+ return documentSelector.select(m);
+ }
+
+ /**
+ * Maximum number of files to add. If {@link #maxFilesToAdd} < 0 (default),
+ * then this crawler will add all documents.
+ *
+ * @param maxFilesToAdd
+ */
+ public void setMaxFilesToAdd(int maxFilesToAdd) {
+ this.maxFilesToAdd = maxFilesToAdd;
+ }
+
+
+ /**
+ * Maximum number of files to consider. A file is considered
+ * whether or not the DocumentSelector selects a document.
+ *
+ * If {@link #maxFilesToConsider} < 0 (default), then this crawler
+ * will add all documents.
+ *
+ * @param maxFilesToConsider
+ */
+ public void setMaxFilesToConsider(int maxFilesToConsider) {
+ this.maxFilesToConsider = maxFilesToConsider;
+ }
+
+ /**
+ * Use sparingly. This synchronizes on the queue!
+ * @return whether this queue contains any non-poison file resources
+ */
+ public boolean isQueueEmpty() {
+ int size= 0;
+ synchronized(queue) {
+ Iterator it = queue.iterator();
+ while (it.hasNext()) {
+ if (it.next() instanceof PoisonFileResource) {
+ } else {
+ size++;
+ }
+ }
+ }
+ return size == 0;
+ }
+
+ /**
+ * Returns whether the crawler timed out while trying to add a resource
+ * to the queue.
+ *
+ * If the crawler timed out while trying to add poison, this is not
+ * set to true.
+ *
+ * @return
+ */
+ public boolean wasTimedOut() {
+ return timedOut;
+ }
+
+ public int getAdded() {
+ return added;
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/ConsumersManager.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/ConsumersManager.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/ConsumersManager.java (revision 0)
@@ -0,0 +1,45 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.List;
+
+/**
+ * Simple interface around a collection of consumers that allows
+ * for initializing and shutting shared resources (e.g. db connection, index, writer, etc.)
+ */
+public interface ConsumersManager {
+ /**
+ * Get the consumers
+ * @return consumers
+ */
+ public List getConsumers();
+
+ /**
+ * This is called by BatchProcess before submitting the threads
+ */
+ public void init();
+
+ /**
+ * This is called by BatchProcess immediately before closing.
+ * Beware! Some of the consumers may have hung or may not
+ * have completed.
+ */
+ public void shutdown();
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/InterrupterFutureResult.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/InterrupterFutureResult.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/InterrupterFutureResult.java (revision 0)
@@ -0,0 +1,22 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class InterrupterFutureResult implements IFileProcessorFutureResult {
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/IStatusReporter.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/IStatusReporter.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/IStatusReporter.java (revision 0)
@@ -0,0 +1,24 @@
+package org.apache.tika.batch;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.concurrent.Callable;
+
+
+public interface IStatusReporter extends Callable {
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/PoisonFileResource.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/PoisonFileResource.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/PoisonFileResource.java (revision 0)
@@ -0,0 +1,54 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.metadata.Metadata;
+
+import java.io.InputStream;
+
+/**
+ * Sentinel class for the crawler to add to the queue to let
+ * the consumers know that they should shutdown.
+ */
+class PoisonFileResource implements FileResource {
+
+ /**
+ * always returns null
+ */
+ @Override
+ public Metadata getMetadata() {
+ return null;
+ }
+
+ /**
+ * always returns null
+ */
+ @Override
+ public InputStream openInputStream() {
+ return null;
+ }
+
+ /**
+ * always returns null
+ */
+ @Override
+ public String getResourceId() {
+ return null;
+ }
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/BatchProcess.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/BatchProcess.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/BatchProcess.java (revision 0)
@@ -0,0 +1,450 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+
+/**
+ * This is the main processor class for a single process.
+ *
+ * It requires a {@link FileResourceCrawler} and {@link FileResourceConsumer}s, and it can also
+ * support a {@link SimpleLogStatusReporter} and an {@link CommandLineInterrupter}.
+ *
+ * This is designed to shutdown if there are too many stale processors (runaway parser)
+ * or an OutOfMemoryError. Consider using {@link BatchProcessDriverCLI}
+ * for more robust handling of these types of errors.
+ */
+public class BatchProcess {
+
+ public enum BATCH_CONSTANTS {
+ BATCH_PROCESS_EXCEEDED_MAX_ALIVE_TIME,
+ BATCH_PROCESS_FATAL_MUST_RESTART
+ }
+
+ private enum CAUSE_FOR_TERMINATION {
+ COMPLETED_NORMALLY,
+ CONSUMER_EXCEPTION_NO_RESTART,
+ CONSUMER_EXCEPTION,
+ CRAWLER_TIMED_OUT,
+ TOO_MANY_STALES,
+ USER_INTERRUPTION,
+ BATCH_PROCESS_ALIVE_TOO_LONG,
+ }
+
+ private static final Log logger = LogFactory.getLog(BatchProcess.class);
+
+ // If a file hasn't been processed in this amount of time,
+ // report it to the console. When the directory crawler has stopped, the thread will
+ // be terminated and the file name will be logged
+ private long staleThresholdMillis = 5 * 60 * 1000; // 5 minutes
+
+ private long staleCheckPulseMillis = 2 * 60 * 1000; //2 minutes
+ //if there was an early termination via the Interrupter
+ //or because of an uncaught runtime throwable, pause
+ //this long before shutting down to allow parsers to finish
+ private long pauseOnEarlyTerminationMillis = 30000; //30 seconds
+
+ //maximum time that this process should stay alive
+ //to avoid potential memory leaks, not a bad idea to shutdown
+ //every hour or so.
+ private int maxAliveTimeSeconds = -1;
+
+ private final FileResourceCrawler fileResourceCrawler;
+
+ private final ConsumersManager consumersManager;
+
+ private final IStatusReporter reporter;
+
+ private final IInterrupter interrupter;
+
+ private final ArrayBlockingQueue stales;
+
+ private boolean alreadyExecuted = false;
+
+ private final int maxStaleConsumers;
+
+ public BatchProcess(FileResourceCrawler fileResourceCrawler,
+ ConsumersManager consumersManager,
+ IStatusReporter reporter,
+ IInterrupter interrupter, int maxStaleConsumers) {
+ this.fileResourceCrawler = fileResourceCrawler;
+ this.consumersManager = consumersManager;
+ this.reporter = reporter;
+ this.interrupter = interrupter;
+ this.maxStaleConsumers = (maxStaleConsumers > -1) ? maxStaleConsumers : 0;
+ stales = new ArrayBlockingQueue(consumersManager.getConsumers().size());
+ }
+
+ public ParallelFileProcessingResult execute()
+ throws InterruptedException {
+ if (alreadyExecuted == true) {
+ throw new IllegalStateException("Can only execute BatchRunner once.");
+ }
+ long start = new Date().getTime();
+ logger.info("BatchProcess starting up");
+
+ consumersManager.init();
+ alreadyExecuted = true;
+ long started = new Date().getTime();
+ int considered = 0;
+ int added = 0;
+ int processed = 0;
+ int numConsumers = consumersManager.getConsumers().size();
+ // fileResourceCrawler, statusReporter, the Interrupter, staleChecker
+ int numNonConsumers = 4;
+
+ ExecutorService ex = Executors.newFixedThreadPool(numConsumers
+ + numNonConsumers);
+
+ CompletionService completionService =
+ new ExecutorCompletionService(
+ ex);
+ StaleChecker staleChecker = new StaleChecker();
+ completionService.submit(interrupter);
+ completionService.submit(fileResourceCrawler);
+ completionService.submit(reporter);
+ completionService.submit(staleChecker);
+
+
+ for (FileResourceConsumer consumer : consumersManager.getConsumers()) {
+ completionService.submit(consumer);
+ }
+
+ int removed = 0;
+ int consumersRemoved = 0;
+ int crawlerRemoved = 0;
+
+ CAUSE_FOR_TERMINATION causeForTermination = null;
+ while (true) {
+ try {
+ Future futureResult =
+ completionService.poll(1, TimeUnit.SECONDS);
+
+ if (futureResult != null) {
+ removed++;
+ IFileProcessorFutureResult result = futureResult.get();
+ if (result instanceof FileConsumerFutureResult) {
+ consumersRemoved++;
+ processed += ((FileConsumerFutureResult) result).getFilesProcessed();
+ } else if (result instanceof FileResourceCrawlerFutureResult) {
+ crawlerRemoved++;
+ added += ((FileResourceCrawlerFutureResult) result).getAdded();
+ considered += ((FileResourceCrawlerFutureResult) result).getConsidered();
+ if (fileResourceCrawler.wasTimedOut()) {
+ causeForTermination = CAUSE_FOR_TERMINATION.CRAWLER_TIMED_OUT;
+ break;
+ }
+ } else if (result instanceof InterrupterFutureResult) {
+ causeForTermination = CAUSE_FOR_TERMINATION.USER_INTERRUPTION;
+ break;
+ } else if (result instanceof StaleFutureResult) {
+ causeForTermination = CAUSE_FOR_TERMINATION.TOO_MANY_STALES;
+ break;
+ } //only thing left should be StatusReporterResult
+ }
+
+ if (consumersRemoved >= numConsumers) {
+ causeForTermination = CAUSE_FOR_TERMINATION.COMPLETED_NORMALLY;
+ break;
+ }
+ if (aliveTooLong(started)) {
+ causeForTermination = CAUSE_FOR_TERMINATION.BATCH_PROCESS_ALIVE_TOO_LONG;
+ break;
+ }
+ } catch (Throwable e) {
+ if (isNonRestart(e)) {
+ causeForTermination = CAUSE_FOR_TERMINATION.CONSUMER_EXCEPTION_NO_RESTART;
+ } else {
+ causeForTermination = CAUSE_FOR_TERMINATION.CONSUMER_EXCEPTION;
+ }
+ e.printStackTrace();
+ logger.fatal("Execution exception " + e.getMessage());
+ break;
+ }
+ }
+
+
+ //Step 1, try a gentle shutdown.
+ //This just prevents un-"called" threads from being started
+ ex.shutdown();
+
+ //Step 2, ask consumers to retire politely.
+ //most should have by now under normal circumstances
+ for (FileResourceConsumer consumer : consumersManager.getConsumers()) {
+ consumer.pleaseRetire();
+ }
+
+ //if there are any active/non-stale consumers, await termination
+ if (countActiveConsumers() > 0) {
+ if (causeForTermination != null) {
+ logger.trace("About to awaitTermination: " + pauseOnEarlyTerminationMillis);
+ ex.awaitTermination(pauseOnEarlyTerminationMillis, TimeUnit.MILLISECONDS);
+ } else {
+ ex.awaitTermination(1, TimeUnit.SECONDS);
+ }
+ }
+
+ //Step 3: Gloves come off. We've tried to ask kindly before.
+ //Now it is time shut down. This could corrupt
+ //nio channels via thread interrupts! Hopefully, everything
+ //has shut down by now.
+ logger.trace("About to shutdownNow()");
+ List neverCalled = ex.shutdownNow();
+ logger.trace("TERMINATED " + ex.isTerminated() + " : "
+ + consumersRemoved + " : " + crawlerRemoved);
+
+ int end = numConsumers + numNonConsumers - removed - neverCalled.size();
+
+ for (int t = 0; t < end; t++) {
+ Future future = null;
+ future = completionService.poll(10, TimeUnit.MILLISECONDS);
+
+ logger.trace("In while future==null loop in final shutdown loop");
+ if (future == null) {
+ break;
+ }
+ try {
+ IFileProcessorFutureResult result = future.get();
+ if (result instanceof FileConsumerFutureResult) {
+ FileConsumerFutureResult consumerResult = (FileConsumerFutureResult) result;
+ processed += consumerResult.getFilesProcessed();
+ FileStarted fileStarted = consumerResult.getFileStarted();
+ if (fileStarted != null
+ && fileStarted.getElapsedMillis() > staleThresholdMillis) {
+ logger.warn(fileStarted.getResourceId()
+ + "\t caused a file processor to hang or crash. You may need to remove "
+ + "this file from your input set and rerun.");
+ }
+ } else if (result instanceof FileResourceCrawlerFutureResult) {
+ FileResourceCrawlerFutureResult crawlerResult = (FileResourceCrawlerFutureResult) result;
+ considered += crawlerResult.getConsidered();
+ added += crawlerResult.getAdded();
+ } //else ...we don't care about anything else stopping at this point
+ } catch (ExecutionException e) {
+ logger.error("Execution exception trying to shutdown after shutdownNow:" + e.getMessage());
+ } catch (InterruptedException e) {
+ logger.error("Interrupted exception trying to shutdown after shutdownNow:" + e.getMessage());
+ }
+ }
+ //do we need to restart?
+ String restartMsg = null;
+ if (causeForTermination == CAUSE_FOR_TERMINATION.USER_INTERRUPTION
+ || causeForTermination == CAUSE_FOR_TERMINATION.CONSUMER_EXCEPTION_NO_RESTART) {
+ //do not restart!!!
+ } else if (causeForTermination == CAUSE_FOR_TERMINATION.CONSUMER_EXCEPTION) {
+ restartMsg = "Uncaught consumer throwable";
+ } else if (causeForTermination == CAUSE_FOR_TERMINATION.TOO_MANY_STALES) {
+ if (areResourcesPotentiallyRemaining()) {
+ restartMsg = "Staled out with resources remaining";
+ }
+ } else if (causeForTermination == CAUSE_FOR_TERMINATION.BATCH_PROCESS_ALIVE_TOO_LONG) {
+ restartMsg = BATCH_CONSTANTS.BATCH_PROCESS_EXCEEDED_MAX_ALIVE_TIME.toString();
+ } else if (causeForTermination == CAUSE_FOR_TERMINATION.CRAWLER_TIMED_OUT) {
+ restartMsg = "Crawler timed out.";
+ } else if (crawlerRemoved == 0) {
+ restartMsg = "Crawler did not complete.";
+ } else if (fileResourceCrawler.wasTimedOut()) {
+ restartMsg = "Crawler was timed out.";
+ } else if (fileResourceCrawler.isActive()) {
+ restartMsg = "Crawler is still active.";
+ } else if (! fileResourceCrawler.isQueueEmpty()) {
+ restartMsg = "Resources still exist for processing";
+ }
+
+
+ int exitStatus = handleRestartMsg(restartMsg);
+
+ //need to re-check, report, remove stale consumers
+ staleChecker.checkForStaleConsumers();
+
+ for (FileStarted fs : stales) {
+ logger.error("A parser was still working on " + fs.getResourceId() +
+ " for " + fs.getElapsedMillis() + " milliseconds after it started." +
+ " This exceeds the maxStaleMillis parameter");
+ }
+ logger.info("ConsumersManager is shutting down");
+ consumersManager.shutdown();
+ logger.info("BatchProcess is shutting down");
+
+ double elapsed = ((double) new Date().getTime() - (double) start) / 1000.0;
+ return new
+ ParallelFileProcessingResult(considered, added, processed,
+ elapsed, exitStatus, causeForTermination.toString());
+
+ }
+
+ private boolean isNonRestart(Throwable e) {
+ if (e instanceof BatchNoRestartException) {
+ return true;
+ }
+ Throwable cause = e.getCause();
+ if (cause != null) {
+ return isNonRestart(cause);
+ }
+ return false;
+ }
+
+ private int handleRestartMsg(String restartMsg) {
+ if (restartMsg != null) {
+ if (restartMsg.equals(BATCH_CONSTANTS.BATCH_PROCESS_EXCEEDED_MAX_ALIVE_TIME.toString())) {
+ logger.info(restartMsg);
+ } else {
+ logger.fatal(restartMsg);
+ }
+ //must send to err to communicate with potential driver
+ System.err.println(
+ BATCH_CONSTANTS.BATCH_PROCESS_FATAL_MUST_RESTART.toString() +
+ " >> " + restartMsg);
+ System.err.flush();
+ return 1;
+ }
+ return 0;
+ }
+
+ //could new FileResources be consumed from the Queue?
+ //Because of race conditions, this can return a true
+ //when the real answer is false.
+ //This should never return false, though, if the answer is true!
+ private boolean areResourcesPotentiallyRemaining() {
+ if (fileResourceCrawler.isActive()) {
+ return true;
+ }
+ if (!fileResourceCrawler.isQueueEmpty()) {
+ return true;
+ }
+ return false;
+ }
+
+ private boolean aliveTooLong(long started) {
+ if (maxAliveTimeSeconds < 0) {
+ return false;
+ }
+ double elapsedSeconds = (double) (new Date().getTime() - started) / (double) 1000;
+ if (elapsedSeconds > (double) maxAliveTimeSeconds) {
+ return true;
+ }
+ return false;
+ }
+
+ //snapshot of non-retired consumers; actual number may be smaller by the time
+ //this returns a value!
+ private int countActiveConsumers() {
+ int active = 0;
+ for (FileResourceConsumer consumer : consumersManager.getConsumers()) {
+ if (consumer.isStillActive()) {
+ active++;
+ }
+ }
+ return active;
+ }
+
+ /**
+ * If there is an early termination via an interrupt or too many stale consumers
+ * or because a consumer or other Runnable threw a Throwable, pause this long
+ * before killing the consumers and other threads.
+ *
+ * @param pauseOnEarlyTerminationMillis
+ */
+ public void setPauseOnEarlyTerminationMillis(long pauseOnEarlyTerminationMillis) {
+ this.pauseOnEarlyTerminationMillis = pauseOnEarlyTerminationMillis;
+ }
+
+ /**
+ * The amount of time allowed before a consumer is determined to be stale.
+ *
+ * @param staleThresholdMillis
+ */
+ public void setStaleThresholdMillis(long staleThresholdMillis) {
+ this.staleThresholdMillis = staleThresholdMillis;
+ }
+
+ public void setStaleCheckPulseMillis(long staleCheckPulseMillis) {
+ this.staleCheckPulseMillis = staleCheckPulseMillis;
+ }
+
+ /**
+ * The maximum amount of time that this process can be alive. To avoid
+ * memory leaks, it is sometimes beneficial to shutdown (and restart) the
+ * process periodically.
+ *
+ * If the value is < 0, the process will run until completion, interruption or exception.
+ *
+ * @param maxAliveTimeSeconds maximum amount of time in seconds to remain alive
+ */
+ public void setMaxAliveTimeSeconds(int maxAliveTimeSeconds) {
+ this.maxAliveTimeSeconds = maxAliveTimeSeconds;
+ }
+
+ private class StaleChecker implements Callable {
+
+ @Override
+ public StaleFutureResult call() throws Exception {
+ while (stales.size() <= maxStaleConsumers) {
+ try {
+ Thread.sleep(staleCheckPulseMillis);
+ } catch (InterruptedException e) {
+ logger.debug("interrupted ex in stalechecker");
+ break;
+ //just stop.
+ }
+ checkForStaleConsumers();
+ if (countActiveConsumers() == 0) {
+ logger.error("No activeConsumers in StaleChecker");
+ break;
+ }
+ }
+ logger.debug("Stale checker quitting: " + stales.size());
+ return new StaleFutureResult(stales.size());
+ }
+
+ private void checkForStaleConsumers() {
+ for (FileResourceConsumer consumer : consumersManager.getConsumers()) {
+ FileStarted fs = consumer.checkForStaleMillis(staleThresholdMillis);
+ if (fs != null) {
+ stales.add(fs);
+ }
+ }
+ }
+ }
+
+ private class StaleFutureResult implements IFileProcessorFutureResult {
+ private final int staleCount;
+
+ private StaleFutureResult(final int staleCount) {
+ this.staleCount = staleCount;
+ }
+
+ protected int getStaleCount() {
+ return staleCount;
+ }
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/SimpleLogStatusReporter.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/SimpleLogStatusReporter.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/SimpleLogStatusReporter.java (revision 0)
@@ -0,0 +1,213 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tika.util.BatchLocalization;
+import org.apache.tika.util.DurationFormatUtils;
+
+import java.text.NumberFormat;
+import java.util.Date;
+import java.util.Locale;
+
+/**
+ * Basic class to use for reporting status from both the crawler and the consumers.
+ * This wakes up roughly every {@link #sleepMillis} and log.info's a status report.
+ */
+
+public class SimpleLogStatusReporter implements IStatusReporter {
+
+ private final Log logger = LogFactory.getLog(SimpleLogStatusReporter.class);
+
+ //require references to these so that the
+ //StatusReporter can query them when it wakes up
+ private final ConsumersManager consumersManager;
+ private final FileResourceCrawler crawler;
+
+ //local time that the StatusReporter started
+ private final long start;
+ //how long to sleep between reporting intervals
+ private long sleepMillis = 1000;
+
+ //how long before considering a parse "stale" (potentially hung forever)
+ private long staleThresholdMillis = 100000;
+
+ /**
+ * Initialize with the crawler and consumers
+ *
+ * @param crawler crawler to ping at intervals
+ * @param consumersManager consumers to ping at intervals
+ */
+ public SimpleLogStatusReporter(FileResourceCrawler crawler, ConsumersManager consumersManager) {
+ this.consumersManager = consumersManager;
+ this.crawler = crawler;
+ start = new Date().getTime();
+ }
+
+ /**
+ * Override for different behavior.
+ *
+ * This reports the string at the info level to this class' logger.
+ *
+ * @param s
+ */
+ protected void report(String s) {
+ logger.info(s);
+ }
+
+ /**
+ * Startup the reporter.
+ */
+ public IFileProcessorFutureResult call() {
+ NumberFormat numberFormat = NumberFormat.getNumberInstance(Locale.US);
+ try {
+ while (true) {
+ Thread.sleep(sleepMillis);
+ int cnt = getRoughCountConsumed();
+ int exceptions = getRoughCountExceptions();
+ long elapsed = new Date().getTime() - start;
+ double elapsedSecs = (double) elapsed / (double) 1000;
+ int avg = (elapsedSecs > 5 || cnt > 100) ? (int) ((double) cnt / elapsedSecs) : -1;
+
+ String elapsedString = DurationFormatUtils.formatMillis(new Date().getTime() - start);
+ String docsPerSec = avg > -1 ? String.format(BatchLocalization.getLocale(),
+ " (%s docs per sec)",
+ numberFormat.format(avg)) : "";
+ String msg =
+ String.format(
+ BatchLocalization.getLocale(),
+ "Processed %s documents in %s%s.",
+ numberFormat.format(cnt), elapsedString, docsPerSec);
+ report(msg);
+ if (exceptions == 1){
+ msg = "There has been one handled exception.";
+ } else {
+ msg =
+ String.format(BatchLocalization.getLocale(),
+ "There have been %s handled exceptions.",
+ numberFormat.format(exceptions));
+ }
+ report(msg);
+
+ reportStale();
+
+ int stillAlive = getStillAlive();
+ if (stillAlive == 1) {
+ msg = "There is one file processor still active.";
+ } else {
+ msg = "There are " + numberFormat.format(stillAlive) + " file processors still active.";
+ }
+ report(msg);
+
+ int crawled = crawler.getConsidered();
+ int added = crawler.getAdded();
+ if (crawled == 1) {
+ msg = "The directory crawler has considered 1 file,";
+ } else {
+ msg = "The directory crawler has considered " +
+ numberFormat.format(crawled) + " files, ";
+ }
+ if (added == 1) {
+ msg += "and it has added 1 file.";
+ } else {
+ msg += "and it has added " +
+ numberFormat.format(crawler.getAdded()) + " files.";
+ }
+ msg += "\n";
+ report(msg);
+
+ if (! crawler.isActive()) {
+ msg = "The directory crawler has completed its crawl.\n";
+ report(msg);
+ }
+ }
+ } catch (InterruptedException e) {
+
+ }
+ return new StatusReporterFutureResult();
+ }
+
+
+ /**
+ * Set the amount of time to sleep between reports.
+ * @param sleepMillis
+ */
+ public void setSleepMillis(long sleepMillis) {
+ this.sleepMillis = sleepMillis;
+ }
+
+ /**
+ * Set the amount of time in milliseconds to use as the threshold for determining
+ * a stale parse.
+ *
+ * @param staleThresholdMillis
+ */
+ public void setStaleThresholdMillis(long staleThresholdMillis) {
+ this.staleThresholdMillis = staleThresholdMillis;
+ }
+
+
+ private void reportStale() {
+ for (FileResourceConsumer consumer : consumersManager.getConsumers()) {
+ FileStarted fs = consumer.getCurrentFile();
+ if (fs == null) {
+ continue;
+ }
+ long elapsed = fs.getElapsedMillis();
+ if (elapsed > staleThresholdMillis) {
+ String elapsedString = Double.toString((double) elapsed / (double) 1000);
+ report("A thread has been working on " + fs.getResourceId() +
+ " for " + elapsedString + " seconds.");
+ }
+ }
+ }
+
+ /*
+ * This returns a rough (unsynchronized) count of resources consumed.
+ */
+ private int getRoughCountConsumed() {
+ int ret = 0;
+ for (FileResourceConsumer consumer : consumersManager.getConsumers()) {
+ ret += consumer.getNumResourcesConsumed();
+ }
+ return ret;
+ }
+
+ private int getStillAlive() {
+ int ret = 0;
+ for (FileResourceConsumer consumer : consumersManager.getConsumers()) {
+ if ( consumer.isStillActive()) {
+ ret++;
+ }
+ }
+ return ret;
+ }
+
+ /**
+ * This returns a rough (unsynchronized) count of caught/handled exceptions.
+ * @return
+ */
+ public int getRoughCountExceptions() {
+ int ret = 0;
+ for (FileResourceConsumer consumer : consumersManager.getConsumers()) {
+ ret += consumer.getNumHandledExceptions();
+ }
+ return ret;
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/IFileProcessorFutureResult.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/IFileProcessorFutureResult.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/IFileProcessorFutureResult.java (revision 0)
@@ -0,0 +1,26 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * stub interface to allow for different result types from different processors
+ *
+ */
+public interface IFileProcessorFutureResult {
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/ParserFactory.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/ParserFactory.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/ParserFactory.java (revision 0)
@@ -0,0 +1,27 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.parser.Parser;
+
+public interface ParserFactory {
+
+ public Parser getParser(TikaConfig config);
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/BatchProcessDriverCLI.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/BatchProcessDriverCLI.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/BatchProcessDriverCLI.java (revision 0)
@@ -0,0 +1,246 @@
+package org.apache.tika.batch;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.cli.Options;
+import org.apache.log4j.Logger;
+import org.apache.tika.io.IOUtils;
+import org.apache.tika.util.BatchLocalization;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
+import java.io.Writer;
+import java.util.Locale;
+
+public class BatchProcessDriverCLI {
+
+ private static Logger logger = Logger.getLogger(BatchProcessDriverCLI.class);
+
+ private int maxProcessRestarts = -1;
+ private long pulseMillis = 1000;
+
+ private boolean mustRestartProcess = false;
+ private boolean mustStopSelf = false;
+ private Process process = null;
+
+ private StreamGobbler errorGobbler = null;
+ private StreamGobbler outGobbler = null;
+ private StreamWriter stdinWriter = null;
+
+ private final String[] commandLine;
+ private int numRestarts = 0;
+
+ private Options options = null;
+
+ public BatchProcessDriverCLI(String[] commandLine){
+ this.commandLine = commandLine;
+ }
+
+ public void execute() throws Exception {
+ start();
+ while (mustStopSelf == false) {
+ int exit = Integer.MIN_VALUE;
+ boolean hasExited = false;
+ try {
+ exit = process.exitValue();
+ hasExited = true;
+ stop();
+ } catch (IllegalThreadStateException e) {
+ //hasn't exited
+ }
+ //Even if the process has exited,
+ //wait just a little bit to make sure that
+ //mustRestart hasn't been set to true
+ try {
+ Thread.sleep(pulseMillis);
+ } catch (InterruptedException e) {
+ //swallow
+ }
+ if (hasExited && exit == 0 && ! mustRestartProcess) {
+ break;
+ }
+ if ((hasExited && exit != 0) || mustRestartProcess) {
+ restart();
+ }
+ }
+ if (mustStopSelf) {
+ shutdownNow();
+ }
+ }
+
+ private void shutdownNow() {
+ for (int i = 0; i < 10; i++) {
+ try {
+ int exit = process.exitValue();
+ stop();
+ return;
+ } catch (IllegalThreadStateException e) {
+ //hasn't exited
+ }
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ //swallow
+ }
+ }
+ logger.error("Process didn't stop after 10 seconds after shutdown. " +
+ "I am forcefully killing it.");
+ stop();
+
+ }
+
+ public int getNumRestarts() {
+ return numRestarts;
+ }
+
+ public boolean getStoppedSelf() {
+ return mustStopSelf;
+ }
+
+ private boolean restart() throws Exception {
+ if (maxProcessRestarts > -1 && numRestarts >= maxProcessRestarts) {
+ logger.warn("Exceeded the maximum number of process restarts. Driver is shutting down now.");
+ stop();
+ return false;
+ }
+ logger.warn("Must restart process ("+numRestarts+" restarts so far).");
+ stop();
+ start();
+ numRestarts++;
+ return true;
+ }
+
+ private void stop() {
+ if (process != null) {
+ process.destroy();
+ }
+ mustRestartProcess = false;
+ errorGobbler.stopGobblingAndDie();
+ outGobbler.stopGobblingAndDie();
+ stdinWriter.stopGobblingAndDie();
+ }
+
+ private void start() throws Exception {
+ ProcessBuilder builder = new ProcessBuilder(commandLine);
+ builder.directory(new File("."));
+ process = builder.start();
+ errorGobbler = new StreamGobbler(process.getErrorStream());
+ new Thread(errorGobbler).start();
+
+ outGobbler = new StreamGobbler(process.getInputStream());
+ new Thread(outGobbler).start();
+
+ stdinWriter = new StreamWriter(System.in, process.getOutputStream());
+ new Thread(stdinWriter).start();
+
+ }
+
+
+ /**
+ * Class that transfers anything sent to stdin to the process driver
+ * to the stdin of the BatchProcess. This allows for a graceful interrupt.
+ */
+ private class StreamWriter implements Runnable {
+ private BufferedReader reader;
+ private Writer writer;
+ private boolean running = true;
+
+ private StreamWriter(InputStream is, OutputStream os) {
+ try {
+ this.reader = new BufferedReader(new InputStreamReader(new BufferedInputStream(is),
+ BatchLocalization.getEncoding()));
+ this.writer = new OutputStreamWriter(os, BatchLocalization.getEncoding());
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException("Unsupported encoding:" +
+ BatchLocalization.getEncoding());
+ }
+ }
+
+ @Override
+ public void run() {
+ String line = null;
+ try {
+ while ((line = reader.readLine()) != null && this.running) {
+ writer.write(String.format(Locale.ENGLISH, "%s%n", line));
+ writer.flush();
+ mustStopSelf = true;
+ break;
+ }
+ } catch (IOException e) {
+ //swallow ioe
+ }
+ }
+ private void stopGobblingAndDie() {
+ running = false;
+ }
+
+
+ }
+
+ private class StreamGobbler implements Runnable {
+ //plagiarized from oap.oodt's StreamGobbler
+ private final BufferedReader reader;
+ private boolean running = true;
+
+ private StreamGobbler(InputStream is){
+
+ try {
+ this.reader = new BufferedReader(new InputStreamReader(new BufferedInputStream(is),
+ BatchLocalization.getEncoding()));
+ } catch (UnsupportedEncodingException e) {
+ throw new RuntimeException("UnsupportedEncodingException: "+
+ BatchLocalization.getEncoding());
+ }
+ }
+
+ @Override
+ public void run() {
+ String line = null;
+ try {
+ while ((line = reader.readLine()) != null && this.running) {
+ if (line.startsWith(BatchProcess.BATCH_CONSTANTS.BATCH_PROCESS_FATAL_MUST_RESTART.toString())) {
+ mustRestartProcess = true;
+ }
+ logger.info("BatchProcess: "+line);
+ }
+ } catch (IOException e) {
+ //swallow ioe
+ }
+ }
+
+ private void stopGobblingAndDie() {
+ running = false;
+ IOUtils.closeQuietly(reader);
+ }
+ }
+
+
+ public static void main(String[] args) throws Exception {
+
+ BatchProcessDriverCLI runner = new BatchProcessDriverCLI(args);
+ runner.execute();
+ System.out.println("FSBatchProcessDriver has gracefully completed");
+ System.exit(0);
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/FileConsumerFutureResult.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/FileConsumerFutureResult.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/FileConsumerFutureResult.java (revision 0)
@@ -0,0 +1,37 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class FileConsumerFutureResult implements IFileProcessorFutureResult {
+
+ private final FileStarted fileStarted;
+ private final int filesProcessed;
+
+ public FileConsumerFutureResult(FileStarted fs, int filesProcessed) {
+ this.fileStarted = fs;
+ this.filesProcessed = filesProcessed;
+ }
+
+ public FileStarted getFileStarted() {
+ return fileStarted;
+ }
+
+ public int getFilesProcessed() {
+ return filesProcessed;
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/FileResource.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/FileResource.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/FileResource.java (revision 0)
@@ -0,0 +1,68 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.Property;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+
+/**
+ * This is a basic interface to handle a logical "file".
+ * This should enable code-agnostic handling of files from different
+ * sources: file system, database, etc.
+ *
+ */
+public interface FileResource {
+
+ //The literal lowercased extension of a file. This may or may not
+ //have any relationship to the actual type of the file.
+ public static final Property FILE_EXTENSION = Property.internalText("tika:file_ext");
+
+ /**
+ * This is only used in logging to identify which file
+ * may have caused problems. While it is probably best
+ * to use unique ids for the sake of debugging, it is not
+ * necessary that the ids be unique. This id
+ * is never used as a hashkey by the batch processors, for example.
+ *
+ * @return an id for a FileResource
+ */
+ public String getResourceId();
+
+ /**
+ * This gets the metadata available before the parsing of the file.
+ * This will typically be "external" metadata: file name,
+ * file size, file location, data stream, etc. That is, things
+ * that are known about the file from outside information, not
+ * file-internal metadata.
+ *
+ * @return Metadata
+ */
+ public Metadata getMetadata();
+
+ /**
+ *
+ * @return an InputStream for the FileResource
+ * @throws java.io.IOException
+ */
+ public InputStream openInputStream() throws IOException;
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/BasicTikaFSConsumer.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/BasicTikaFSConsumer.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/BasicTikaFSConsumer.java (revision 0)
@@ -0,0 +1,143 @@
+package org.apache.tika.batch.fs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tika.batch.BatchNoRestartException;
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.batch.OutputStreamFactory;
+import org.apache.tika.batch.ParserFactory;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.sax.ContentHandlerFactory;
+import org.xml.sax.ContentHandler;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+import java.util.concurrent.ArrayBlockingQueue;
+
+/**
+ * Basic FileResourceConsumer that reads files from an input
+ * directory and writes content to the output directory.
+ *
+ * This catches all exceptions and errors and then logs them.
+ * This will re-throw errors.
+ *
+ */
+public class BasicTikaFSConsumer extends FileResourceConsumer {
+
+ Log log = LogFactory.getLog(BasicTikaFSConsumer.class);
+
+
+ private boolean parseRecursively = true;
+ private final ParserFactory parserFactory;
+ private final ContentHandlerFactory contentHandlerFactory;
+ private final OutputStreamFactory fsOSFactory;
+ private String outputEncoding = "UTF-8";
+
+
+ public BasicTikaFSConsumer(ArrayBlockingQueue queue,
+ ParserFactory parserFactory,
+ ContentHandlerFactory contentHandlerFactory,
+ OutputStreamFactory fsOSFactory) {
+ super(queue);
+ this.parserFactory = parserFactory;
+ this.contentHandlerFactory = contentHandlerFactory;
+ this.fsOSFactory = fsOSFactory;
+ }
+
+ @Override
+ public boolean processFileResource(FileResource fileResource) {
+
+ Parser parser = parserFactory.getParser(TikaConfig.getDefaultConfig());
+ ParseContext context = new ParseContext();
+ if (parseRecursively == true) {
+ context.set(Parser.class, parser);
+ }
+
+ OutputStream os = null;
+ try {
+ os = fsOSFactory.getOutputStream(fileResource.getMetadata());
+ } catch (IOException e) {
+ log.fatal(e.getMessage());
+ flushAndClose(os);
+ throw new BatchNoRestartException("IOException trying to open output stream for "+
+ fileResource.getResourceId() + " :: " + e.getMessage());
+ }
+ //os can be null if fsOSFactory is set to skip processing a file if the target
+ //file already exists
+ if (os == null) {
+ log.debug("Skipping: " + fileResource.getMetadata().get(FSProperties.FS_ABSOLUTE_PATH));
+ return false;
+ }
+ InputStream is = null;
+ try {
+ is = fileResource.openInputStream();
+ } catch (IOException e) {
+ incrementHandledExceptions();
+ log.error(e.getMessage());
+ flushAndClose(os);
+ return false;
+ }
+
+ ContentHandler handler;
+ try {
+ handler = contentHandlerFactory.getNewContentHandler(os, getOutputEncoding());
+ } catch (UnsupportedEncodingException e) {
+ close(is);
+ flushAndClose(os);
+ throw new RuntimeException(e.getMessage());
+ }
+
+ //now actually call parse!
+ Throwable thrown = null;
+ try {
+ parser.parse(is, handler,
+ fileResource.getMetadata(), context);
+ } catch (Throwable t) {
+ log.error(t.getMessage());
+ thrown = t;
+ } finally {
+ close(is);
+ flushAndClose(os);
+ }
+
+ if (thrown != null) {
+ if (thrown instanceof Error) {
+ throw (Error)thrown;
+ } else {
+ incrementHandledExceptions();
+ }
+ return false;
+ }
+ return true;
+ }
+
+ public String getOutputEncoding() {
+ return outputEncoding;
+ }
+
+ public void setOutputEncoding(String outputEncoding) {
+ this.outputEncoding = outputEncoding;
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/FSFileResource.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/FSFileResource.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/FSFileResource.java (revision 0)
@@ -0,0 +1,99 @@
+package org.apache.tika.batch.fs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.util.BatchLocalization;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * FileSystem(FS)Resource wraps a file name.
+ *
+ * This class automatically sets the following keys in Metadata:
+ *
+ * - Metadata.RESOURCE_NAME_KEY (file name)
+ * - Metadata.CONTENT_LENGTH
+ * - FSProperties.FS_ABSOLUTE_PATH
+ * - FileResource.FILE_EXTENSION
+ *
,
+ */
+public class FSFileResource implements FileResource {
+
+ private final File fullPath;
+ private final Metadata metadata;
+
+ public FSFileResource(File srcRoot, File fullPath) {
+ this.fullPath = fullPath;
+ this.metadata = new Metadata();
+ //child path must actually be a child
+ assert(FSUtil.checkThisIsAncestorOfThat(srcRoot, fullPath));
+ String relativePath = fullPath.getAbsolutePath().substring(srcRoot.getAbsolutePath().length()+1);
+
+ //need to set these now so that the filter can determine
+ //whether or not to crawl this file
+ metadata.set(Metadata.RESOURCE_NAME_KEY, fullPath.getName());
+ metadata.set(Metadata.CONTENT_LENGTH, Long.toString(fullPath.length()));
+ metadata.set(FSProperties.FS_ABSOLUTE_PATH, fullPath.getAbsolutePath());
+ metadata.set(FSProperties.FS_REL_PATH, relativePath);
+ metadata.set(FileResource.FILE_EXTENSION, getExtension(fullPath));
+ }
+
+ /**
+ * Simple extension extractor that takes whatever comes after the
+ * last period in the path. It returns a lowercased version of that.
+ *
+ * If there is no period, it returns an empty string.
+ *
+ * @param fullPath
+ * @return
+ */
+ private String getExtension(File fullPath) {
+ String p = fullPath.getName();
+ int i = p.lastIndexOf(".");
+ if (i > -1) {
+ return p.substring(i + 1).toLowerCase(BatchLocalization.getLocale());
+ }
+ return "";
+ }
+
+ /**
+ *
+ * @return file's absolutePath
+ */
+ @Override
+ public String getResourceId() {
+ return fullPath.getAbsolutePath();
+ }
+
+ @Override
+ public Metadata getMetadata() {
+ return metadata;
+ }
+
+ @Override
+ public InputStream openInputStream() throws IOException {
+ //no need to include Metadata because we already set the
+ //same information in the initializer
+ return TikaInputStream.get(fullPath);
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/FSBatchProcessCLI.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/FSBatchProcessCLI.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/FSBatchProcessCLI.java (revision 0)
@@ -0,0 +1,101 @@
+package org.apache.tika.batch.fs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.tika.batch.BatchProcess;
+import org.apache.tika.batch.ParallelFileProcessingResult;
+import org.apache.tika.batch.builders.BatchProcessBuilder;
+import org.apache.tika.io.IOUtils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+public class FSBatchProcessCLI {
+ public static String FINISHED_STRING = "Main thread in TikaFSBatchCLI has finished processing.";
+
+ public Options getOptions() {
+ Options options = new Options();
+ options.addOption("c", "config", true, "xml config file");
+ options.addOption("randomCrawl", false, "crawl files randomly");
+ options.addOption("numConsumers", true, "how many consumer threads to use (default=number of processors-1)");
+ options.addOption("maxFileSizeBytes", true, "if an input file is larger than this, skip it.");
+ options.addOption("maxStaleConsumers", true, "maximum number of stale consumers to allow before shutdown");
+ options.addOption("maxQueueSize", true, "maximum size of the FileResource queue");
+ options.addOption("fileList", true, "list of files to process (files relative to srcDir)");
+ options.addOption("fileListEncoding", true, "encoding for fileList");
+ options.addOption("srcDir", "sourceDirectory", true, "source directory (must be specified!)");
+ options.addOption("startDir", "startDirectory", true, "start directory (default: srcDir). Must be a child of or equal to srcDir");
+ options.addOption("targDir", "targetDirectory", true, "target directory (must be specified!)");
+ options.addOption("recursiveParserWrapper", false, "use the recursive parser wrapper or not (default = false)");
+ options.addOption("handleExisting", false, "if a target file already exists, do you want to: overwrite, rename or skip");
+ options.addOption("basicHandlerType", true, "what type of content handler: xml, text, html, body");
+ options.addOption("targetSuffix", true, "suffix to add to the end of the target file name");
+ options.addOption("staleThresholdMillis", true, "how long to wait before determining that a consumer has gone stale");
+ options.addOption("?", "help", false, "this help message");
+ return options;
+ }
+
+ public static void main(String[] args) throws Exception {
+ Options options = new FSBatchProcessCLI().getOptions();
+
+ CommandLineParser cliParser = new GnuParser();
+ CommandLine line = cliParser.parse(options, args);
+
+ if (line.hasOption("help")) {
+ HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.printHelp("tika filesystem batch", options);
+ System.exit(-1);
+ }
+
+ Map mapArgs = new HashMap();
+ for (Option option : line.getOptions()) {
+ String v = option.getValue();
+ if (v == null || v.equals("")) {
+ v = "true";
+ }
+ mapArgs.put(option.getOpt(), v);
+ }
+ String configFilePath = line.getOptionValue("c");
+ InputStream is = null;
+ if (configFilePath != null) {
+ File configFile = new File(configFilePath);
+ is = new FileInputStream(configFile);
+ }
+ if (is == null) {
+ throw new RuntimeException("Must specify a configuration file: -c");
+ }
+ BatchProcessBuilder b = new BatchProcessBuilder();
+ BatchProcess process = b.build(is, mapArgs);
+ IOUtils.closeQuietly(is);
+ ParallelFileProcessingResult result = process.execute();
+ System.out.println(FINISHED_STRING);
+ System.out.println("\n");
+ System.out.println(result.toString());
+ System.exit(result.getExitStatus());
+ }
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/FSConsumersManager.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/FSConsumersManager.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/FSConsumersManager.java (revision 0)
@@ -0,0 +1,49 @@
+package org.apache.tika.batch.fs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.ConsumersManager;
+import org.apache.tika.batch.FileResourceConsumer;
+
+import java.util.Collections;
+import java.util.List;
+
+public class FSConsumersManager implements ConsumersManager {
+
+ private final List consumers;
+
+ public FSConsumersManager(List consumers) {
+ this.consumers = Collections.unmodifiableList(consumers);
+ }
+
+ @Override
+ public List getConsumers() {
+ return consumers;
+ }
+
+ @Override
+ public void init() {
+ //noop
+ }
+
+ @Override
+ public void shutdown() {
+ //noop
+ }
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/FSListCrawler.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/FSListCrawler.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/FSListCrawler.java (revision 0)
@@ -0,0 +1,80 @@
+package org.apache.tika.batch.fs;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceCrawler;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
+import java.util.concurrent.ArrayBlockingQueue;
+
+/**
+ * Class that "crawls" a list of files.
+ */
+public class FSListCrawler extends FileResourceCrawler {
+
+ private final BufferedReader reader;
+ private final File root;
+
+ public FSListCrawler(ArrayBlockingQueue fileQueue,
+ int numConsumers, File root, File list, String encoding)
+ throws FileNotFoundException, UnsupportedEncodingException {
+ super(fileQueue, numConsumers);
+ reader = new BufferedReader(new InputStreamReader(new FileInputStream(list), encoding));
+ this.root = root;
+
+ }
+
+ public void start() throws InterruptedException {
+ String line = nextLine();
+
+ while (line != null) {
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException("file adder interrupted");
+ }
+ File f = new File(root, line);
+ if (! f.exists()) {
+ super.logger.warn("File doesn't exist:"+f.getAbsolutePath());
+ line = nextLine();
+ continue;
+ }
+ if (f.isDirectory()) {
+ super.logger.warn("File is a directory:"+f.getAbsolutePath());
+ line = nextLine();
+ continue;
+ }
+ tryToAdd(new FSFileResource(root, f));
+ line = nextLine();
+ }
+ }
+
+ private String nextLine() {
+ String line = null;
+ try {
+ line = reader.readLine();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return line;
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/FSProperties.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/FSProperties.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/FSProperties.java (revision 0)
@@ -0,0 +1,17 @@
+package org.apache.tika.batch.fs;
+
+import org.apache.tika.metadata.Property;
+
+public class FSProperties {
+ private final static String TIKA_BATCH_FS_NAMESPACE = "tika_batch_fs";
+
+ /**
+ * File's absolute path
+ */
+ public final static Property FS_ABSOLUTE_PATH = Property.internalText(TIKA_BATCH_FS_NAMESPACE+":absolute_path");
+
+ /**
+ * File's relative path (including file name) from a given source root
+ */
+ public final static Property FS_REL_PATH = Property.internalText(TIKA_BATCH_FS_NAMESPACE+":relative_path");
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/FSUtil.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/FSUtil.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/FSUtil.java (revision 0)
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.batch.fs;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Utility class to handle some common issues when
+ * reading from and writing to a file system (FS).
+ */
+public class FSUtil {
+
+ public static boolean checkThisIsAncestorOfThat(File ancestor, File child) {
+ int ancLen = ancestor.getAbsolutePath().length();
+ int childLen = child.getAbsolutePath().length();
+ if (childLen <= ancLen) {
+ return false;
+ }
+
+ String childBase = child.getAbsolutePath().substring(0, ancLen);
+ return childBase.equals(ancestor.getAbsolutePath());
+
+ }
+
+ public static boolean checkThisIsAncestorOfOrSameAsThat(File ancestor, File child) {
+ if (ancestor.equals(child)) {
+ return true;
+ }
+ return checkThisIsAncestorOfThat(ancestor, child);
+ }
+
+ public enum HANDLE_EXISTING {
+ OVERWRITE,
+ RENAME,
+ SKIP
+ }
+
+ ;
+
+ private final static Pattern FILE_NAME_PATTERN =
+ Pattern.compile("\\A(.*?)(?:\\((\\d+)\\))?\\.([^\\.]+)\\Z");
+
+ /**
+ * Given a target root and an initial relative path,
+ * return the target file according to the HANDLE_EXISTING strategy
+ *
+ * In the most basic use case, given a root directory "input",
+ * a file's relative path "dir1/dir2/fileA.docx", and a target directory
+ * "output", the target file would be "output/dir1/dir2/fileA.docx."
+ *
+ * If HANDLE_EXISTING is set to OVERWRITE, this will not check to see if the target already exists,
+ * and the returned file could overwrite an existing file!!!
+ *
+ * If HANDLE_EXISTING is set to RENAME, this will try to increment a counter at the end of
+ * the file name (fileA(2).docx) until there is a file name that doesn't exist.
+ *
+ * This will return null if handleExisting == HANDLE_EXISTING.SKIP and
+ * the candidate file already exists.
+ *
+ * This will throw an IOException if HANDLE_EXISTING is set to
+ * RENAME, and a candidate cannot target file cannot be found
+ * after trying to increment the file count (e.g. fileA(2).docx) 10000 times
+ * and then after trying 20,000 UUIDs.
+ *
+ * @param targetRoot
+ * @param initialRelativePath
+ * @param handleExisting
+ * @param suffix suffix to add to files, can be null
+ * @return
+ * @throws java.io.IOException
+ */
+ public static File getTargetFile(File targetRoot, String initialRelativePath,
+ HANDLE_EXISTING handleExisting, String suffix) throws IOException {
+ String localSuffix = (suffix == null) ? "" : suffix;
+ File cand = new File(targetRoot, initialRelativePath+"."+localSuffix);
+ if (cand.isFile()) {
+ if (handleExisting.equals(HANDLE_EXISTING.OVERWRITE)) {
+ return cand;
+ } else if (handleExisting.equals(HANDLE_EXISTING.SKIP)) {
+ return null;
+ }
+ }
+
+ //if we're here, the target file exists, and
+ //we must find a new name for it.
+
+ //groups for "testfile(1).txt":
+ //group(1) is "testfile"
+ //group(2) is 1
+ //group(3) is "txt"
+ //Note: group(2) can be null
+ int cnt = 0;
+ String fNameBase = null;
+ String fNameExt = "";
+ //this doesn't include the addition of the localSuffix
+ File candOnly = new File(targetRoot, initialRelativePath);
+ Matcher m = FILE_NAME_PATTERN.matcher(candOnly.getName());
+ if (m.find()) {
+ fNameBase = m.group(1);
+
+ if (m.group(2) != null) {
+ try {
+ cnt = Integer.parseInt(m.group(2));
+ } catch (NumberFormatException e) {
+ //swallow
+ }
+ }
+ if (m.group(3) != null) {
+ fNameExt = m.group(3);
+ }
+ }
+
+ File targetParent = cand.getParentFile();
+ while (fNameBase != null && cand.isFile() && ++cnt < 10000) {
+ String candFileName = fNameBase + "(" + cnt + ")." + fNameExt+"."+localSuffix;
+ cand = new File(targetParent, candFileName);
+ }
+ //reset count to 0 and try 20000 times
+ cnt = 0;
+ while (cand.isFile() && cnt++ < 20000) {
+ UUID uid = UUID.randomUUID();
+ cand = new File(targetParent, uid.toString() + fNameExt+"."+localSuffix);
+ }
+
+ if (cand.isFile()) {
+ throw new IOException("Couldn't find candidate target file after trying " +
+ "very, very hard");
+ }
+ return cand;
+ }
+
+}
\ No newline at end of file
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/FSDocumentSelector.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/FSDocumentSelector.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/FSDocumentSelector.java (revision 0)
@@ -0,0 +1,78 @@
+package org.apache.tika.batch.fs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.extractor.DocumentSelector;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.util.PropsUtil;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Selector that chooses files based on their file name
+ * and their size, as determined by Metadata.RESOURCE_NAME_KEY and Metadata.CONTENT_LENGTH.
+ *
+ * The {@link #excludeFileName} pattern is applied first (if it isn't null).
+ * Then the {@link #includeFileName} pattern is applied (if it isn't null),
+ * and finally, the size limit is applied if it is above 0.
+ */
+public class FSDocumentSelector implements DocumentSelector {
+
+ //can be null!
+ private final Pattern includeFileName;
+
+ //can be null!
+ private final Pattern excludeFileName;
+ private final long maxFileSizeBytes;
+
+ public FSDocumentSelector(Pattern includeFileName, Pattern excludeFileName, long maxFileSizeBytes) {
+ this.includeFileName = includeFileName;
+ this.excludeFileName = excludeFileName;
+ this.maxFileSizeBytes = maxFileSizeBytes;
+ }
+
+ @Override
+ public boolean select(Metadata metadata) {
+ String fName = metadata.get(Metadata.RESOURCE_NAME_KEY);
+ long sz = PropsUtil.getLong(metadata.get(Metadata.CONTENT_LENGTH), -1);
+
+ if (excludeFileName != null && fName != null) {
+ Matcher m = excludeFileName.matcher(fName);
+ if (m.find()) {
+ return false;
+ }
+ }
+
+ if (includeFileName != null && fName != null) {
+ Matcher m = includeFileName.matcher(fName);
+ if (m.find()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ if (maxFileSizeBytes > 0 && sz > 0) {
+ if (sz > maxFileSizeBytes) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/builders/FSCrawlerBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/builders/FSCrawlerBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/builders/FSCrawlerBuilder.java (revision 0)
@@ -0,0 +1,128 @@
+package org.apache.tika.batch.fs.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceCrawler;
+import org.apache.tika.batch.builders.BatchProcessBuilder;
+import org.apache.tika.batch.builders.ICrawlerBuilder;
+import org.apache.tika.batch.fs.FSDirectoryCrawler;
+import org.apache.tika.batch.fs.FSDocumentSelector;
+import org.apache.tika.extractor.DocumentSelector;
+import org.apache.tika.util.PropsUtil;
+import org.apache.tika.util.XMLDOMUtil;
+import org.w3c.dom.Node;
+
+import java.io.File;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.regex.Pattern;
+
+/**
+ * Builds either an FSDirectoryCrawler or an FSListCrawler.
+ */
+public class FSCrawlerBuilder implements ICrawlerBuilder {
+
+ private final static String MAX_CONSEC_WAIT_MILLIS = "maxConsecWaitMillis";
+ private final static String MAX_FILES_TO_ADD_ATTR = "maxFilesToAdd";
+ private final static String MAX_FILES_TO_CONSIDER_ATTR = "maxFilesToConsider";
+
+
+ private final static String CRAWL_ORDER = "crawlOrder";
+ private final static String SRC_DIR_ATTR = "srcDir";
+ private final static String SRC_START_DIR_ATTR = "startDir";
+ private final static String MAX_FILE_SIZE_BYTES_ATTR = "maxFileSizeBytes";
+
+
+ private final static String INCLUDE_FILE_PAT_ATTR = "includeFilePat";
+ private final static String EXCLUDE_FILE_PAT_ATTR = "excludeFilePat";
+
+ @Override
+ public FileResourceCrawler build(Node node, Map runtimeAttributes,
+ ArrayBlockingQueue queue) {
+
+ Map attributes = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+
+ int numConsumers = BatchProcessBuilder.getNumConsumers(runtimeAttributes);
+
+ File srcDir = PropsUtil.getFile(attributes.get(SRC_DIR_ATTR), new File("input"));
+ FileResourceCrawler crawler = null;
+ if (attributes.containsKey("fileList")) {
+ String randomCrawlString = attributes.get(CRAWL_ORDER);
+
+ if (randomCrawlString != null) {
+ //TODO: change to logger warn or throw RuntimeException?
+ System.err.println("randomCrawl attribute is ignored by FSListCrawler");
+ }
+ File fileList = PropsUtil.getFile(attributes.get("fileList"), null);
+ String encoding = PropsUtil.getString(attributes.get("fileListEncoding"), "UTF-8");
+ try {
+ crawler = new org.apache.tika.batch.fs.FSListCrawler(queue, numConsumers, srcDir, fileList, encoding);
+ } catch (java.io.FileNotFoundException e) {
+ throw new RuntimeException("fileList file not found for FSListCrawler: " + fileList.getAbsolutePath());
+ } catch (java.io.UnsupportedEncodingException e) {
+ throw new RuntimeException("fileList encoding not supported: "+encoding);
+ }
+ } else {
+ FSDirectoryCrawler.CRAWL_ORDER crawlOrder = getCrawlOrder(attributes.get(CRAWL_ORDER));
+ File startDir = PropsUtil.getFile(attributes.get(SRC_START_DIR_ATTR), null);
+ if (startDir == null) {
+ crawler = new FSDirectoryCrawler(queue, numConsumers, srcDir, crawlOrder);
+ } else {
+ crawler = new FSDirectoryCrawler(queue, numConsumers, srcDir, startDir, crawlOrder);
+ }
+ }
+
+ crawler.setMaxFilesToConsider(PropsUtil.getInt(attributes.get(MAX_FILES_TO_CONSIDER_ATTR), -1));
+ crawler.setMaxFilesToAdd(PropsUtil.getInt(attributes.get(MAX_FILES_TO_ADD_ATTR), -1));
+
+ DocumentSelector selector = buildSelector(attributes);
+ if (selector != null) {
+ crawler.setDocumentSelector(selector);
+ }
+
+ crawler.setMaxConsecWaitInMillis(PropsUtil.getLong(attributes.get(MAX_CONSEC_WAIT_MILLIS), 300000));//5 minutes
+ return crawler;
+ }
+
+ private FSDirectoryCrawler.CRAWL_ORDER getCrawlOrder(String s) {
+ if (s == null || s.trim().length() == 0) {
+ return FSDirectoryCrawler.CRAWL_ORDER.OS_ORDER;
+ } else if (s.toLowerCase().contains("rand")) {
+ return FSDirectoryCrawler.CRAWL_ORDER.RANDOM;
+ } else if (s.toLowerCase().contains("sort")) {
+ return FSDirectoryCrawler.CRAWL_ORDER.SORTED;
+ } else {
+ return FSDirectoryCrawler.CRAWL_ORDER.OS_ORDER;
+ }
+ }
+
+ private DocumentSelector buildSelector(Map attributes) {
+ String includeString = attributes.get(INCLUDE_FILE_PAT_ATTR);
+ String excludeString = attributes.get(EXCLUDE_FILE_PAT_ATTR);
+ long maxFileSize = PropsUtil.getLong(attributes.get(MAX_FILE_SIZE_BYTES_ATTR), -1);
+ Pattern includePat = (includeString != null && includeString.length() > 0) ? Pattern.compile(includeString) : null;
+ Pattern excludePat = (excludeString != null && excludeString.length() > 0) ? Pattern.compile(excludeString) : null;
+
+ return new FSDocumentSelector(includePat, excludePat, maxFileSize);
+
+ }
+
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/builders/BasicTikaFSConsumersBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/builders/BasicTikaFSConsumersBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/builders/BasicTikaFSConsumersBuilder.java (revision 0)
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tika.batch.fs.builders;
+
+import org.apache.tika.batch.ConsumersManager;
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.batch.OutputStreamFactory;
+import org.apache.tika.batch.ParserFactory;
+import org.apache.tika.batch.builders.AbstractConsumersBuilder;
+import org.apache.tika.batch.builders.BatchProcessBuilder;
+import org.apache.tika.batch.builders.ContentHandlerFactoryBuilder;
+import org.apache.tika.batch.fs.BasicTikaFSConsumer;
+import org.apache.tika.batch.fs.FSConsumersManager;
+import org.apache.tika.batch.fs.FSOutputStreamFactory;
+import org.apache.tika.batch.fs.FSUtil;
+import org.apache.tika.batch.fs.RecursiveParserWrapperFSConsumer;
+import org.apache.tika.sax.ContentHandlerFactory;
+import org.apache.tika.util.ClassLoaderUtil;
+import org.apache.tika.util.PropsUtil;
+import org.apache.tika.util.XMLDOMUtil;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.io.File;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+
+public class BasicTikaFSConsumersBuilder extends AbstractConsumersBuilder {
+
+ @Override
+ public ConsumersManager build(Node node, Map runtimeAttributes,
+ ArrayBlockingQueue queue) {
+
+ //figure out if we're building a recursiveParserWrapper
+ boolean recursiveParserWrapper = false;
+ String recursiveParserWrapperString = runtimeAttributes.get("recursiveParserWrapper");
+ if (recursiveParserWrapperString != null){
+ recursiveParserWrapper = PropsUtil.getBoolean(recursiveParserWrapperString, recursiveParserWrapper);
+ } else {
+ Node recursiveParserWrapperNode = node.getAttributes().getNamedItem("recursiveParserWrapper");
+ if (recursiveParserWrapperNode != null) {
+ recursiveParserWrapper = PropsUtil.getBoolean(recursiveParserWrapperNode.getNodeValue(), recursiveParserWrapper);
+ }
+ }
+
+ List consumers = new LinkedList();
+ int numConsumers = BatchProcessBuilder.getNumConsumers(runtimeAttributes);
+
+ NodeList nodeList = node.getChildNodes();
+ Node contentHandlerFactoryNode = null;
+ Node parserFactoryNode = null;
+ Node outputStreamFactoryNode = null;
+
+ for (int i = 0; i < nodeList.getLength(); i++){
+ Node child = nodeList.item(i);
+ String cn = child.getNodeName();
+ if (cn.equals("parser")){
+ parserFactoryNode = child;
+ } else if (cn.equals("contenthandler")) {
+ contentHandlerFactoryNode = child;
+ } else if (cn.equals("outputstream")) {
+ outputStreamFactoryNode = child;
+ }
+ }
+
+ if (contentHandlerFactoryNode == null || parserFactoryNode == null
+ || outputStreamFactoryNode == null) {
+ throw new RuntimeException("You must specify a ContentHandlerFactory, "+
+ "a ParserFactory and an OutputStreamFactory");
+ }
+ ContentHandlerFactory contentHandlerFactory = getContentHandlerFactory(contentHandlerFactoryNode, runtimeAttributes);
+ ParserFactory parserFactory = getParserFactory(parserFactoryNode, runtimeAttributes);
+ OutputStreamFactory outputStreamFactory = getOutputStreamFactory(outputStreamFactoryNode, runtimeAttributes);
+
+ if (recursiveParserWrapper) {
+ for (int i = 0; i < numConsumers; i++) {
+ FileResourceConsumer c = new RecursiveParserWrapperFSConsumer(queue,
+ parserFactory, contentHandlerFactory, outputStreamFactory);
+ consumers.add(c);
+ }
+ } else {
+ for (int i = 0; i < numConsumers; i++) {
+ FileResourceConsumer c = new BasicTikaFSConsumer(queue,
+ parserFactory, contentHandlerFactory, outputStreamFactory);
+ consumers.add(c);
+ }
+ }
+ return new FSConsumersManager(consumers);
+ }
+
+
+ private ContentHandlerFactory getContentHandlerFactory(Node node, Map runtimeAttributes) {
+
+ Map localAttrs = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+ String className = localAttrs.get("builderClass");
+ if (className == null) {
+ throw new RuntimeException("Must specify builderClass for contentHandler");
+ }
+ ContentHandlerFactoryBuilder builder = ClassLoaderUtil.buildClass(ContentHandlerFactoryBuilder.class, className);
+ return builder.build(node, runtimeAttributes);
+ }
+
+ private ParserFactory getParserFactory(Node node, Map runtimeAttributes) {
+ //TODO: add ability to set TikaConfig file path
+ Map localAttrs = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+ String className = localAttrs.get("class");
+ return ClassLoaderUtil.buildClass(ParserFactory.class, className);
+ }
+
+ private OutputStreamFactory getOutputStreamFactory(Node node, Map runtimeAttributes) {
+ Map attrs = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+
+ File targDir = PropsUtil.getFile(attrs.get("targDir"), null);
+ FSUtil.HANDLE_EXISTING handleExisting = null;
+ String handleExistingString = attrs.get("handleExisting");
+ if (handleExistingString == null) {
+ handleExistingException(); }
+ if (handleExistingString.equals("overwrite")){
+ handleExisting = FSUtil.HANDLE_EXISTING.OVERWRITE;
+ } else if (handleExistingString.equals("rename")) {
+ handleExisting = FSUtil.HANDLE_EXISTING.RENAME;
+ } else if (handleExistingString.equals("skip")) {
+ handleExisting = FSUtil.HANDLE_EXISTING.SKIP;
+ } else {
+ handleExistingException();
+ }
+
+ String suffix = attrs.get("targetSuffix");
+
+ FSOutputStreamFactory factory = new FSOutputStreamFactory(targDir, handleExisting, suffix);
+ return factory;
+ }
+
+ private void handleExistingException() {
+ throw new RuntimeException("Must specify handleExisting in FSOutputStreamFactory; values include: overwrite, rename, skip");
+ }
+
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/RecursiveParserWrapperFSConsumer.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/RecursiveParserWrapperFSConsumer.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/RecursiveParserWrapperFSConsumer.java (revision 0)
@@ -0,0 +1,180 @@
+package org.apache.tika.batch.fs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.tika.batch.BatchNoRestartException;
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.batch.OutputStreamFactory;
+import org.apache.tika.batch.ParserFactory;
+import org.apache.tika.config.TikaConfig;
+import org.apache.tika.exception.TikaException;
+import org.apache.tika.metadata.Metadata;
+import org.apache.tika.metadata.serialization.JsonMetadataList;
+import org.apache.tika.parser.ParseContext;
+import org.apache.tika.parser.Parser;
+import org.apache.tika.parser.RecursiveParserWrapper;
+import org.apache.tika.sax.ContentHandlerFactory;
+import org.xml.sax.helpers.DefaultHandler;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.io.Writer;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+
+/**
+ * Basic FileResourceConsumer that reads files from an input
+ * directory and writes content to the output directory.
+ *
+ * This tries to catch most of the common exceptions and log them as errors.
+ */
+public class RecursiveParserWrapperFSConsumer extends FileResourceConsumer {
+
+ Log log = LogFactory.getLog(RecursiveParserWrapperFSConsumer.class);
+
+
+ private final ParserFactory parserFactory;
+ private final ContentHandlerFactory contentHandlerFactory;
+ private final OutputStreamFactory fsOSFactory;
+ private String outputEncoding = "UTF-8";
+
+
+ public RecursiveParserWrapperFSConsumer(ArrayBlockingQueue queue,
+ ParserFactory parserFactory,
+ ContentHandlerFactory contentHandlerFactory,
+ OutputStreamFactory fsOSFactory) {
+ super(queue);
+ this.parserFactory = parserFactory;
+ this.contentHandlerFactory = contentHandlerFactory;
+ this.fsOSFactory = fsOSFactory;
+ }
+
+ @Override
+ public boolean processFileResource(FileResource fileResource) {
+
+ Parser wrapped = parserFactory.getParser(TikaConfig.getDefaultConfig());
+ RecursiveParserWrapper parser = new RecursiveParserWrapper(wrapped, contentHandlerFactory);
+ ParseContext context = new ParseContext();
+
+// if (parseRecursively == true) {
+ context.set(Parser.class, parser);
+// }
+
+ //try to open outputstream first
+ OutputStream os = null;
+ try {
+ os = fsOSFactory.getOutputStream(fileResource.getMetadata());
+ } catch (IOException e) {
+ log.fatal(e.getMessage());
+ throw new BatchNoRestartException("IOException trying to open output stream for "+
+ fileResource.getResourceId() + " :: " + e.getMessage());
+ }
+ //os can be null if fsOSFactory is set to skip processing a file and the target
+ //file already exists
+ if (os == null) {
+ log.debug("Skipping: " + fileResource.getMetadata().get(FSProperties.FS_ABSOLUTE_PATH));
+ return false;
+ }
+ String absolutePath = fileResource.getMetadata().get(FSProperties.FS_ABSOLUTE_PATH);
+
+ //try to open the inputstream before the parse.
+ //if the parse hangs or throws a nasty exception, at least there will
+ //be a zero byte file there so that the batchrunner can skip that problematic
+ //file during the next run.
+ InputStream is = null;
+ try {
+ is = fileResource.openInputStream();
+ } catch (IOException e) {
+ log.error(e.getMessage());
+ incrementHandledExceptions();
+ flushAndClose(os);
+ return false;
+ }
+
+ Throwable thrown = null;
+ List metadataList = null;
+ try {
+ parser.parse(is, new DefaultHandler(),
+ fileResource.getMetadata(), context);
+ metadataList = parser.getMetadata();
+ } catch (Throwable t) {
+ thrown = t;
+ if (t instanceof Error) {
+ log.fatal(absolutePath + "\t" + t.getMessage());
+ } else {
+ log.error(absolutePath + "\t" + t.getMessage());
+ }
+ if (metadataList == null) {
+ metadataList = new LinkedList();
+ }
+ Metadata m = null;
+ if (metadataList.size() > 0) {
+ m = metadataList.remove(0);
+ } else {
+ m = fileResource.getMetadata();
+ }
+ StringWriter stringWriter = new StringWriter();
+ PrintWriter w = new PrintWriter(stringWriter);
+ t.printStackTrace(w);
+ m.add(RecursiveParserWrapper.PARSE_EXCEPTION, stringWriter.toString());
+ metadataList.add(0, m);
+ } finally {
+ close(is);
+ }
+
+ Writer writer = null;
+
+ try {
+ writer = new OutputStreamWriter(os, getOutputEncoding());
+ JsonMetadataList.toJson(metadataList, writer);
+ } catch (IOException e) {
+ log.error(absolutePath + "\t" + e.getMessage());
+ } catch (TikaException e) {
+ log.error(absolutePath + "\t" + e.getMessage());
+ } finally {
+ flushAndClose(writer);
+ }
+
+ if (thrown != null) {
+ if (thrown instanceof Error) {
+ throw (Error) thrown;
+ } else {
+ incrementHandledExceptions();
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ public String getOutputEncoding() {
+ return outputEncoding;
+ }
+
+ public void setOutputEncoding(String outputEncoding) {
+ this.outputEncoding = outputEncoding;
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/strawman/StrawManTikaAppDriver.java (revision 0)
@@ -0,0 +1,215 @@
+package org.apache.tika.batch.fs.strawman;
+
+
+import org.apache.log4j.Logger;
+import org.apache.tika.io.IOUtils;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Simple single-threaded class that calls tika-app against every file in a directory.
+ *
+ * This is exceedingly robust. One file per process.
+ *
+ * However, you can use this to compare performance against tika-batch fs code.
+ *
+ *
+ */
+public class StrawManTikaAppDriver implements Callable {
+
+ private static AtomicInteger threadCount = new AtomicInteger(0);
+ private final int totalThreads;
+ private final int threadNum;
+ private int rootLen = -1;
+ private File srcDir = null;
+ private File targDir = null;
+ private String[] args = null;
+ private Logger logger = Logger.getLogger(StrawManTikaAppDriver.class);
+
+
+ public StrawManTikaAppDriver(File srcDir, File targDir, int totalThreads, String[] args) {
+ rootLen = srcDir.getAbsolutePath().length()+1;
+ this.srcDir = srcDir;
+ this.targDir = targDir;
+ this.args = args;
+ threadNum = threadCount.getAndIncrement();
+ this.totalThreads = totalThreads;
+ }
+
+
+ private int processDirectory(File srcDir) {
+ int processed = 0;
+ for (File f : srcDir.listFiles()) {
+ List childDirs = new ArrayList();
+ if (f.isDirectory()) {
+ childDirs.add(f);
+ } else {
+ processed += processFile(f);
+ }
+ for (File dir : childDirs) {
+ processed += processDirectory(dir);
+
+ }
+ }
+ return processed;
+ }
+
+ private int processFile(File f) {
+ if (totalThreads > 1) {
+ int hashCode = f.getAbsolutePath().hashCode();
+ if (Math.abs(hashCode % totalThreads) != threadNum) {
+ return 0;
+ }
+ }
+ File targFile = new File(targDir, f.getAbsolutePath().substring(rootLen)+".txt");
+ targFile.getAbsoluteFile().getParentFile().mkdirs();
+ if (! targFile.getParentFile().exists()) {
+ logger.fatal("parent directory for "+ targFile + " was not made!");
+ throw new RuntimeException("couldn't make parent file for " + targFile);
+ }
+ List commandLine = new ArrayList();
+ for (int i = 0; i < args.length; i++) {
+ commandLine.add(args[i]);
+ }
+ commandLine.add("-t");
+ commandLine.add("\""+f.getAbsolutePath()+"\"");
+ ProcessBuilder builder = new ProcessBuilder(commandLine.toArray(new String[commandLine.size()]));
+ logger.info("about to process: "+f.getAbsolutePath());
+ Process proc = null;
+ RedirectGobbler gobbler = null;
+ Thread gobblerThread = null;
+ try {
+ OutputStream os = new FileOutputStream(targFile);
+ proc = builder.start();
+ gobbler = new RedirectGobbler(proc.getInputStream(), os);
+ gobblerThread = new Thread(gobbler);
+ gobblerThread.start();
+ } catch (IOException e) {
+ logger.error(e.getMessage());
+ return 0;
+ }
+
+ boolean finished = false;
+ long totalTime = 180000;//3 minutes
+ long pulse = 100;
+ for (int i = 0; i < totalTime; i += pulse) {
+ try {
+ Thread.currentThread().sleep(pulse);
+ } catch (InterruptedException e) {
+ //swallow
+ }
+ try {
+ int exit = proc.exitValue();
+ finished = true;
+ break;
+ } catch (IllegalThreadStateException e) {
+ //swallow
+ }
+ }
+ if (!finished) {
+ logger.warn("Had to kill process working on: " + f.getAbsolutePath());
+ proc.destroy();
+ }
+ gobbler.close();
+ gobblerThread.interrupt();
+ return 1;
+ }
+
+
+ @Override
+ public Integer call() throws Exception {
+ long start = new Date().getTime();
+
+ int processed = processDirectory(srcDir);
+ double elapsedSecs = ((double)new Date().getTime()-(double)start)/(double)1000;
+ logger.info("Finished processing " + processed + " files in " + elapsedSecs + " seconds.");
+ return new Integer(processed);
+ }
+
+ private class RedirectGobbler implements Runnable {
+ private OutputStream os = null;
+ private InputStream is = null;
+
+ private RedirectGobbler(InputStream is, OutputStream os) {
+ this.is = is;
+ this.os = os;
+ }
+ private void gobble(InputStream is) {
+ }
+ private void close() {
+ if (os != null) {
+ try {
+ os.flush();
+ } catch (IOException e) {
+ logger.error("can't flush");
+ }
+ try {
+ os.close();
+ is.close();
+ } catch (IOException e) {
+ logger.error("can't close");
+ }
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ IOUtils.copy(is, os);
+ } catch (IOException e) {
+ logger.error("IOException while gobbling");
+ }
+ }
+ }
+ public static void main(String[] args) {
+ long start = new Date().getTime();
+ File srcDir = new File(args[0]);
+ File targDir = new File(args[1]);
+ int totalThreads = Integer.parseInt(args[2]);
+
+ List commandLine = new ArrayList();
+ for (int i = 3; i < args.length; i++) {
+ commandLine.add(args[i]);
+ }
+ totalThreads = (totalThreads < 1) ? 1 : totalThreads;
+ ExecutorService ex = Executors.newFixedThreadPool(totalThreads);
+ ExecutorCompletionService completionService =
+ new ExecutorCompletionService(ex);
+
+ for (int i = 0; i < totalThreads; i++) {
+ StrawManTikaAppDriver driver =
+ new StrawManTikaAppDriver(srcDir, targDir, totalThreads, commandLine.toArray(new String[commandLine.size()]));
+ completionService.submit(driver);
+ }
+
+ int totalFilesProcessed = 0;
+ for (int i = 0; i < totalThreads; i++) {
+ try {
+ Future future = completionService.take();
+ if (future != null) {
+ totalFilesProcessed += future.get();
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (ExecutionException e) {
+ e.printStackTrace();
+ }
+ }
+ double elapsedSeconds = (double)(new Date().getTime()-start)/(double)1000;
+ System.out.println("Processed "+totalFilesProcessed + " in " + elapsedSeconds + " seconds");
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/FSDirectoryCrawler.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/FSDirectoryCrawler.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/FSDirectoryCrawler.java (revision 0)
@@ -0,0 +1,141 @@
+package org.apache.tika.batch.fs;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceCrawler;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+
+public class FSDirectoryCrawler extends FileResourceCrawler {
+
+ public enum CRAWL_ORDER
+ {
+ SORTED, //alphabetical order; necessary for cross-platform unit tests
+ RANDOM, //Shuffle
+ OS_ORDER //OS chooses
+ };
+ private final File root;
+ private final File startDirectory;
+ private final Comparator fileComparator = new FileNameComparator();
+ private CRAWL_ORDER crawlOrder;
+
+ public FSDirectoryCrawler(ArrayBlockingQueue fileQueue,
+ int numConsumers, File root, CRAWL_ORDER crawlOrder) {
+ super(fileQueue, numConsumers);
+ this.root = root;
+ this.startDirectory = root;
+ this.crawlOrder = crawlOrder;
+ }
+
+ public FSDirectoryCrawler(ArrayBlockingQueue fileQueue,
+ int numConsumers, File root, File startDirectory,
+ CRAWL_ORDER crawlOrder) {
+ super(fileQueue, numConsumers);
+ this.root = root;
+ this.startDirectory = startDirectory;
+ this.crawlOrder = crawlOrder;
+ assert(FSUtil.checkThisIsAncestorOfOrSameAsThat(root, startDirectory));
+ }
+
+ public void start() throws InterruptedException {
+ addFiles(root);
+ }
+
+ private void addFiles(File directory) throws InterruptedException {
+
+ if (directory == null ||
+ !directory.isDirectory() || !directory.canRead()) {
+ super.logger.warn("FSFileAdder can't read this directory: " + directory.getAbsolutePath());
+ return;
+ }
+
+ List directories = new ArrayList();
+ File[] fileArr = directory.listFiles();
+ if (fileArr == null) {
+ super.logger.info("Empty directory: " + directory.getAbsolutePath());
+ return;
+ }
+
+ List files = new ArrayList(Arrays.asList(fileArr));
+
+ if (crawlOrder == CRAWL_ORDER.RANDOM) {
+ Collections.shuffle(files);
+ } else if (crawlOrder == CRAWL_ORDER.SORTED) {
+ Collections.sort(files, fileComparator);
+ }
+
+ int numFiles = 0;
+ for (File f : files) {
+ if (Thread.currentThread().isInterrupted()) {
+ throw new InterruptedException("file adder interrupted");
+ }
+
+ if (f.isFile()) {
+ numFiles++;
+ if (numFiles == 1) {
+ handleFirstFileInDirectory(f);
+ }
+ }
+ if (f.isDirectory()) {
+ directories.add(f);
+ continue;
+ }
+ boolean added = tryToAdd(new FSFileResource(root, f));
+ logger.debug("trying to add: "+f.getAbsolutePath() + " : " + added);
+
+ }
+
+ for (File f : directories) {
+ addFiles(f);
+ }
+ return;
+ }
+
+ /**
+ * Override this if you have any special handling
+ * for the first actual file that the crawler comes across
+ * in a directory. For example, it might be handy to call
+ * mkdirs() on a target directory if your FileResourceConsumers
+ * are writing to a file.
+ *
+ * @param f
+ */
+ public void handleFirstFileInDirectory(File f) {
+ //no-op
+ }
+
+ //simple lexical order for the file name, we don't really care about localization.
+ //we do want this, though, because file.compareTo behaves differently
+ //on different OS's.
+ private class FileNameComparator implements Comparator {
+
+ @Override
+ public int compare(File f1, File f2) {
+ if (f1 == null || f2 == null) {
+ return 0;
+ }
+ return f1.getName().compareTo(f2.getName());
+ }
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/fs/FSOutputStreamFactory.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/fs/FSOutputStreamFactory.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/fs/FSOutputStreamFactory.java (revision 0)
@@ -0,0 +1,72 @@
+package org.apache.tika.batch.fs;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.OutputStreamFactory;
+import org.apache.tika.metadata.Metadata;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+public class FSOutputStreamFactory implements OutputStreamFactory {
+
+ private final FSUtil.HANDLE_EXISTING handleExisting;
+ private final File targetRoot;
+ private final String suffix;
+
+ public FSOutputStreamFactory(File targetRoot, FSUtil.HANDLE_EXISTING handleExisting, String suffix) {
+ this.handleExisting = handleExisting;
+ this.targetRoot = targetRoot.getAbsoluteFile();
+ this.suffix = suffix;
+ }
+
+ /**
+ * This tries to create a file based on the {@link org.apache.tika.batch.fs.FSUtil.HANDLE_EXISTING}
+ * value that was passed in during initialization.
+ *
+ * If {@link #handleExisting} is set to "SKIP" and the target file already exists,
+ * this will return null.
+ *
+ * If a target file can be found, this will try to mkdirs for that target file.
+ * If mkdirs() fails, this will throw an IOException.
+ *
+ * Finally, this will open an output stream for the appropriate target file.
+ * @param metadata must have a value set for FSMetadataProperties.FS_ABSOLUTE_PATH or
+ * else NullPointerException will be thrown!
+ * @return
+ * @throws IOException, NullPointerException
+ */
+ @Override
+ public OutputStream getOutputStream(Metadata metadata) throws IOException {
+ String initialRelativePath = metadata.get(FSProperties.FS_REL_PATH);
+ File targetFile = FSUtil.getTargetFile(targetRoot, initialRelativePath, handleExisting, suffix);
+ if (targetFile == null) {
+ return null;
+ }
+ if (! targetFile.getParentFile().isDirectory()) {
+ boolean success = targetFile.getParentFile().mkdirs();
+ //with multithreading, it is possible that the parent file was created between
+ //the test and the attempt to .mkdirs(); mkdirs() returns false if the dirs already exist
+ if (! success && ! targetFile.getParentFile().isDirectory()) {
+ throw new IOException("Couldn't create parent directory for:"+targetFile.getAbsolutePath());
+ }
+ }
+ return new FileOutputStream(targetFile);
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/ParallelFileProcessingResult.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/ParallelFileProcessingResult.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/ParallelFileProcessingResult.java (revision 0)
@@ -0,0 +1,99 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class ParallelFileProcessingResult {
+ private final int considered;
+ private final int added;
+ private final int consumed;
+ private final double secondsElapsed;
+ private final int exitStatus;
+ private final String causeForTermination;
+
+ public ParallelFileProcessingResult(int considered, int added, int consumed, double secondsElapsed,
+ int exitStatus,
+ String causeForTermination) {
+ this.considered = considered;
+ this.added = added;
+ this.consumed = consumed;
+ this.secondsElapsed = secondsElapsed;
+ this.exitStatus = exitStatus;
+ this.causeForTermination = causeForTermination;
+ }
+
+ /**
+ * Returns the number of file resources considered.
+ * If a filter causes the crawler to ignore a number of resources,
+ * this number could be higher than that returned by {@link #getConsumed()}.
+ *
+ * @return number of file resources considered
+ */
+ public int getConsidered() {
+ return considered;
+ }
+
+ /**
+ * @return number of resources added to the queue
+ */
+ public int getAdded() {
+ return added;
+ }
+
+ /**
+ * @return number of resources that were tried to be consumed. There
+ * may have been an exception.
+ */
+ public int getConsumed() {
+ return consumed;
+ }
+
+ /**
+ * @return whether the {@link BatchProcess} was interrupted
+ * by an {@link CommandLineInterrupter}.
+ */
+ public String getCauseForTermination() {
+ return causeForTermination;
+ }
+
+ /**
+ * @returns seconds elapsed since the start of the batch processing.
+ */
+ public double secondsElapsed() {
+ return secondsElapsed;
+ }
+
+ /**
+ *
+ * @return intendedExitStatus
+ */
+ public int getExitStatus() {
+ return exitStatus;
+ }
+
+ @Override
+ public String toString() {
+ return "ParallelFileProcessingResult{" +
+ "considered=" + considered +
+ ", added=" + added +
+ ", consumed=" + consumed +
+ ", secondsElapsed=" + secondsElapsed +
+ ", exitStatus=" + exitStatus +
+ ", causeForTermination='" + causeForTermination + '\'' +
+ '}';
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/ICrawlerBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/ICrawlerBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/ICrawlerBuilder.java (revision 0)
@@ -0,0 +1,32 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceCrawler;
+import org.w3c.dom.Node;
+
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+
+public interface ICrawlerBuilder extends ObjectFromDOMAndQueueBuilder{
+
+ public FileResourceCrawler build(Node node, Map attributes,
+ ArrayBlockingQueue queue);
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/ObjectFromDOMBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/ObjectFromDOMBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/ObjectFromDOMBuilder.java (revision 0)
@@ -0,0 +1,15 @@
+package org.apache.tika.batch.builders;
+
+
+import org.w3c.dom.Node;
+
+import java.util.Map;
+
+/**
+ * Interface for things that build objects from a DOM Node and a map of runtime attributes
+ * @param
+ */
+public interface ObjectFromDOMBuilder {
+
+ public T build(Node node, Map runtimeAttributes);
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/IInterupterBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/IInterupterBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/IInterupterBuilder.java (revision 0)
@@ -0,0 +1,27 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.IInterrupter;
+import org.w3c.dom.Node;
+
+import java.util.Map;
+
+public interface IInterupterBuilder {
+ public IInterrupter build(Node n, Map commandlineArguments);
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/CommandLineInterrupterBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/CommandLineInterrupterBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/CommandLineInterrupterBuilder.java (revision 0)
@@ -0,0 +1,32 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.CommandLineInterrupter;
+import org.apache.tika.batch.IInterrupter;
+import org.w3c.dom.Node;
+
+import java.util.Map;
+
+public class CommandLineInterrupterBuilder implements IInterupterBuilder {
+
+ @Override
+ public IInterrupter build(Node n, Map commandlineArguments) {
+ return new CommandLineInterrupter();
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/SimpleLogReporterBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/SimpleLogReporterBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/SimpleLogReporterBuilder.java (revision 0)
@@ -0,0 +1,43 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.ConsumersManager;
+import org.apache.tika.batch.FileResourceCrawler;
+import org.apache.tika.batch.SimpleLogStatusReporter;
+import org.apache.tika.util.PropsUtil;
+import org.apache.tika.util.XMLDOMUtil;
+import org.w3c.dom.Node;
+
+import java.util.Map;
+
+public class SimpleLogReporterBuilder implements StatusReporterBuilder {
+
+ @Override
+ public SimpleLogStatusReporter build(FileResourceCrawler crawler, ConsumersManager consumersManager,
+ Node n, Map commandlineArguments) {
+
+ Map attributes = XMLDOMUtil.mapifyAttrs(n, commandlineArguments);
+ long sleepMillis = PropsUtil.getLong(attributes.get("sleepMillis"), 1000);
+ long staleThresholdMillis = PropsUtil.getLong(attributes.get("staleThresholdMillis"), 500000);
+ SimpleLogStatusReporter reporter = new SimpleLogStatusReporter(crawler, consumersManager);
+ reporter.setSleepMillis(sleepMillis);
+ reporter.setStaleThresholdMillis(staleThresholdMillis);
+ return reporter;
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/BatchProcessBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/BatchProcessBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/BatchProcessBuilder.java (revision 0)
@@ -0,0 +1,290 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.BatchProcess;
+import org.apache.tika.batch.ConsumersManager;
+import org.apache.tika.batch.FileResource;
+import org.apache.tika.batch.FileResourceCrawler;
+import org.apache.tika.batch.IInterrupter;
+import org.apache.tika.batch.IStatusReporter;
+import org.apache.tika.util.ClassLoaderUtil;
+import org.apache.tika.util.XMLDOMUtil;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+
+
+public class BatchProcessBuilder {
+
+ public final static int DEFAULT_MAX_QUEUE_SIZE = 1000;
+ public final static String MAX_QUEUE_SIZE_KEY = "maxQueueSize";
+ public final static String NUM_CONSUMERS_KEY = "numConsumers";
+
+ /**
+ * Builds a FileResourceBatchProcessor from runtime arguments and a
+ * input stream of a configuration file. With the exception of the QueueBuilder,
+ * the builders choose how to adjudicate between
+ * runtime arguments and the elements in the configuration file.
+ *
+ * This does not close the InputStream!
+ * @param is inputStream
+ * @param runtimeAttributes incoming runtime attributes
+ * @return batch process
+ * @throws IOException
+ */
+ public BatchProcess build(InputStream is, Map runtimeAttributes) throws IOException {
+ Document doc = null;
+ DocumentBuilderFactory fact = DocumentBuilderFactory.newInstance();
+ DocumentBuilder docBuilder = null;
+ try {
+ docBuilder = fact.newDocumentBuilder();
+ doc = docBuilder.parse(is);
+ } catch (ParserConfigurationException e) {
+ throw new IOException(e);
+ } catch (SAXException e) {
+ throw new IOException(e);
+ }
+ Node docElement = doc.getDocumentElement();
+ return build(docElement, runtimeAttributes);
+ }
+
+ /**
+ * Builds a FileResourceBatchProcessor from runtime arguments and a
+ * document node of a configuration file. With the exception of the QueueBuilder,
+ * the builders choose how to adjudicate between
+ * runtime arguments and the elements in the configuration file.
+ *
+ * @param docElement document element of the xml config file
+ * @param incomingRuntimeAttributes runtime arguments
+ * @return FileResourceBatchProcessor
+ */
+ public BatchProcess build(Node docElement, Map incomingRuntimeAttributes) {
+
+ //key components
+ long staleThresholdMillis = XMLDOMUtil.getLong("staleThresholdMillis",
+ incomingRuntimeAttributes, docElement);
+ long staleCheckPulseMillis = XMLDOMUtil.getLong("staleCheckPulseMillis",
+ incomingRuntimeAttributes, docElement);
+ long pauseOnEarlyTerminationMillis = XMLDOMUtil.getLong("pauseOnEarlyTerminationMillis",
+ incomingRuntimeAttributes, docElement);
+ int maxStaleConsumers = XMLDOMUtil.getInt("maxStaleConsumers",
+ incomingRuntimeAttributes, docElement);
+ int maxAliveTimeSeconds = XMLDOMUtil.getInt("maxAliveTimeSeconds",
+ incomingRuntimeAttributes, docElement);
+ FileResourceCrawler crawler = null;
+ ConsumersManager consumersManager = null;
+ IStatusReporter reporter = null;
+ IInterrupter interrupter = null;
+
+ /*
+ * TODO: This is a bit smelly. NumConsumers needs to be used by the crawler
+ * and the consumers. This copies the incomingRuntimeAttributes and then
+ * supplies the numConsumers from the commandline (if it exists) or from the config file
+ * At least this creates an unmodifiable defensive copy of incomingRuntimeAttributes...
+ */
+ Map runtimeAttributes = setNumConsumersInRuntimeAttributes(docElement, incomingRuntimeAttributes);
+
+ //build queue
+ ArrayBlockingQueue queue = buildQueue(docElement, runtimeAttributes);
+
+ NodeList children = docElement.getChildNodes();
+ Map keyNodes = new HashMap();
+ for (int i = 0; i < children.getLength(); i++) {
+ Node child = children.item(i);
+ if (child.getNodeType() != Node.ELEMENT_NODE) {
+ continue;
+ }
+ String nodeName = child.getNodeName();
+ keyNodes.put(nodeName, child);
+ }
+ //build consumers
+ consumersManager = buildConsumersManager(keyNodes.get("consumers"), runtimeAttributes, queue);
+
+ //build crawler
+ crawler = buildCrawler(queue, keyNodes.get("crawler"), runtimeAttributes);
+
+ reporter = buildReporter(crawler, consumersManager, keyNodes.get("reporter"), runtimeAttributes);
+
+ interrupter = buildInterrupter(keyNodes.get("interrupter"), runtimeAttributes);
+
+ BatchProcess proc = new BatchProcess(
+ crawler, consumersManager, reporter, interrupter, maxStaleConsumers);
+
+ if (staleThresholdMillis > -1) {
+ proc.setStaleThresholdMillis(staleThresholdMillis);
+ }
+
+ if (pauseOnEarlyTerminationMillis > -1) {
+ proc.setPauseOnEarlyTerminationMillis(pauseOnEarlyTerminationMillis);
+ }
+
+ if (staleCheckPulseMillis > -1) {
+ proc.setStaleCheckPulseMillis(staleCheckPulseMillis);
+ }
+ proc.setMaxAliveTimeSeconds(maxAliveTimeSeconds);
+ return proc;
+ }
+
+ private IInterrupter buildInterrupter(Node node, Map runtimeAttributes) {
+ Map attrs = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+ String className = attrs.get("builderClass");
+ if (className == null) {
+ throw new RuntimeException("Need to specify class name in interrupter element");
+ }
+ IInterupterBuilder builder = ClassLoaderUtil.buildClass(IInterupterBuilder.class, className);
+
+ return builder.build(node, runtimeAttributes);
+
+ }
+
+ private IStatusReporter buildReporter(FileResourceCrawler crawler, ConsumersManager consumersManager,
+ Node node, Map runtimeAttributes) {
+
+ Map attrs = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+ String className = attrs.get("builderClass");
+ if (className == null) {
+ throw new RuntimeException("Need to specify class name in reporter element");
+ }
+ StatusReporterBuilder builder = ClassLoaderUtil.buildClass(StatusReporterBuilder.class, className);
+
+ return builder.build(crawler, consumersManager, node, runtimeAttributes);
+
+ }
+
+ /**
+ * numConsumers is needed by both the crawler and the consumers. This utility method
+ * is to be used to extract the number of consumers from a map of String key value pairs.
+ * @param attrs
+ * @return
+ */
+ public static int getNumConsumers(Map attrs) {
+ String nString = attrs.get(BatchProcessBuilder.NUM_CONSUMERS_KEY);
+ if (nString == null || nString.equals("default")) {
+ return AbstractConsumersBuilder.getDefaultNumConsumers();
+ }
+ int n = -1;
+ try {
+ n = Integer.parseInt(nString);
+ } catch (NumberFormatException e) {
+ //swallow
+ }
+ if (n < 1) {
+ n = AbstractConsumersBuilder.getDefaultNumConsumers();
+ }
+ return n;
+ }
+
+ private Map setNumConsumersInRuntimeAttributes(Node docElement, Map incomingRuntimeAttributes) {
+ Map runtimeAttributes = new HashMap();
+
+ for(Map.Entry e : incomingRuntimeAttributes.entrySet()) {
+ runtimeAttributes.put(e.getKey(), e.getValue());
+ }
+
+ //if this is set at runtime use that value
+ if (runtimeAttributes.containsKey(NUM_CONSUMERS_KEY)){
+ return Collections.unmodifiableMap(runtimeAttributes);
+ }
+ Node ncNode = docElement.getAttributes().getNamedItem("numConsumers");
+ int numConsumers = -1;
+ String numConsumersString = ncNode.getNodeValue();
+ try {
+ numConsumers = Integer.parseInt(numConsumersString);
+ } catch (NumberFormatException e) {
+ //swallow and just use numConsumers
+ }
+ //TODO: should we have a max range check?
+ if (numConsumers < 1) {
+ numConsumers = AbstractConsumersBuilder.getDefaultNumConsumers();
+ }
+ runtimeAttributes.put(NUM_CONSUMERS_KEY, Integer.toString(numConsumers));
+ return Collections.unmodifiableMap(runtimeAttributes);
+ }
+
+ //tries to get maxQueueSize from main element
+ private ArrayBlockingQueue buildQueue(Node docElement,
+ Map runtimeAttributes) {
+ int maxQueueSize = DEFAULT_MAX_QUEUE_SIZE;
+ String szString = runtimeAttributes.get(MAX_QUEUE_SIZE_KEY);
+
+ if (szString == null) {
+ Node szNode = docElement.getAttributes().getNamedItem(MAX_QUEUE_SIZE_KEY);
+ if (szNode != null) {
+ szString = szNode.getNodeValue();
+ }
+ }
+
+ try {
+ maxQueueSize = Integer.parseInt(szString);
+ } catch (NumberFormatException e) {
+ //swallow
+ }
+
+ if (maxQueueSize < 0) {
+ maxQueueSize = DEFAULT_MAX_QUEUE_SIZE;
+ }
+
+ return new ArrayBlockingQueue(maxQueueSize);
+ }
+
+ private ConsumersManager buildConsumersManager(Node node,
+ Map runtimeAttributes, ArrayBlockingQueue queue) {
+
+ Map attrs = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+ String className = attrs.get("builderClass");
+ if (className == null) {
+ throw new RuntimeException("Need to specify class name in consumers element");
+ }
+ AbstractConsumersBuilder builder = ClassLoaderUtil.buildClass(AbstractConsumersBuilder.class, className);
+
+ return builder.build(node, runtimeAttributes, queue);
+ }
+
+
+ private FileResourceCrawler buildCrawler(ArrayBlockingQueue queue,
+ Node node, Map runtimeAttributes) {
+ Map attrs = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+ String className = attrs.get("builderClass");
+ if (className == null) {
+ throw new RuntimeException("Need to specify class name in crawler element");
+ }
+
+ ICrawlerBuilder builder = ClassLoaderUtil.buildClass(ICrawlerBuilder.class, className);
+ FileResourceCrawler crawler = builder.build(node, runtimeAttributes, queue);
+
+
+ return crawler;
+ }
+
+
+
+
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/ContentHandlerFactoryBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/ContentHandlerFactoryBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/ContentHandlerFactoryBuilder.java (revision 0)
@@ -0,0 +1,29 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.sax.ContentHandlerFactory;
+import org.w3c.dom.Node;
+
+import java.util.Map;
+
+public interface ContentHandlerFactoryBuilder extends ObjectFromDOMBuilder {
+
+ public ContentHandlerFactory build(Node node, Map attributes);
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/ReporterBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/ReporterBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/ReporterBuilder.java (revision 0)
@@ -0,0 +1,30 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.SimpleLogStatusReporter;
+import org.w3c.dom.Node;
+
+import java.util.Map;
+
+/**
+ * Interface for reporter builders
+ */
+public interface ReporterBuilder extends ObjectFromDOMBuilder {
+ public SimpleLogStatusReporter build(Node n, Map runtimeAttributes);
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/ObjectFromDOMAndQueueBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/ObjectFromDOMAndQueueBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/ObjectFromDOMAndQueueBuilder.java (revision 0)
@@ -0,0 +1,20 @@
+package org.apache.tika.batch.builders;
+
+
+import org.apache.tika.batch.FileResource;
+import org.w3c.dom.Node;
+
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+
+/**
+ * Same as {@link org.apache.tika.batch.builders.ObjectFromDOMAndQueueBuilder},
+ * but this is for objects that require access to the common queue.
+ * @param
+ */
+public interface ObjectFromDOMAndQueueBuilder {
+
+ public T build(Node node, Map runtimeAttributes,
+ ArrayBlockingQueue resourceQueue);
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/AbstractConsumersBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/AbstractConsumersBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/AbstractConsumersBuilder.java (revision 0)
@@ -0,0 +1,38 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.ConsumersManager;
+import org.apache.tika.batch.FileResource;
+import org.w3c.dom.Node;
+
+import java.util.Map;
+import java.util.concurrent.ArrayBlockingQueue;
+
+public abstract class AbstractConsumersBuilder {
+
+ public static int getDefaultNumConsumers(){
+ int n = Runtime.getRuntime().availableProcessors()-1;
+ return (n < 1) ? 1 : n;
+ }
+
+ public abstract ConsumersManager build(Node node, Map runtimeAttributes,
+ ArrayBlockingQueue queue);
+
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/DefaultContentHandlerFactoryBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/DefaultContentHandlerFactoryBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/DefaultContentHandlerFactoryBuilder.java (revision 0)
@@ -0,0 +1,76 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.sax.BasicContentHandlerFactory;
+import org.apache.tika.sax.ContentHandlerFactory;
+import org.apache.tika.util.BatchLocalization;
+import org.apache.tika.util.XMLDOMUtil;
+import org.w3c.dom.Node;
+
+import java.util.Map;
+
+/**
+ * Builds BasicContentHandler with type defined by attribute "basicHandlerType"
+ * with possible values: xml, html, text, body, ignore.
+ * Default is text.
+ *
+ * Sets the writeLimit to the value of "writeLimit.
+ * Default is -1;
+ */
+public class DefaultContentHandlerFactoryBuilder implements ContentHandlerFactoryBuilder {
+
+ @Override
+ public ContentHandlerFactory build(Node node, Map runtimeAttributes) {
+ Map attributes = XMLDOMUtil.mapifyAttrs(node, runtimeAttributes);
+ BasicContentHandlerFactory.HANDLER_TYPE type = null;
+ String handlerTypeString = attributes.get("basicHandlerType");
+ if (handlerTypeString == null) {
+ handlerTypeString = "text";
+ }
+ handlerTypeString = handlerTypeString.toLowerCase(BatchLocalization.getLocale());
+ if (handlerTypeString.equals("xml")) {
+ type = BasicContentHandlerFactory.HANDLER_TYPE.XML;
+ } else if (handlerTypeString.equals("text")) {
+ type = BasicContentHandlerFactory.HANDLER_TYPE.TEXT;
+ } else if (handlerTypeString.equals("txt")) {
+ type = BasicContentHandlerFactory.HANDLER_TYPE.TEXT;
+ } else if (handlerTypeString.equals("html")) {
+ type = BasicContentHandlerFactory.HANDLER_TYPE.HTML;
+ } else if (handlerTypeString.equals("body")) {
+ type = BasicContentHandlerFactory.HANDLER_TYPE.BODY;
+ } else if (handlerTypeString.equals("ignore")) {
+ type = BasicContentHandlerFactory.HANDLER_TYPE.IGNORE;
+ } else {
+ type = BasicContentHandlerFactory.HANDLER_TYPE.TEXT;
+ }
+ int writeLimit = -1;
+ String writeLimitString = attributes.get("writeLimit");
+ if (writeLimitString != null) {
+ try {
+ writeLimit = Integer.parseInt(attributes.get("writeLimit"));
+ } catch (NumberFormatException e) {
+ //swallow and default to -1
+ //TODO: should we throw a RuntimeException?
+ }
+ }
+ return new BasicContentHandlerFactory(type, writeLimit);
+ }
+
+
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/builders/StatusReporterBuilder.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/builders/StatusReporterBuilder.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/builders/StatusReporterBuilder.java (revision 0)
@@ -0,0 +1,33 @@
+package org.apache.tika.batch.builders;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.tika.batch.ConsumersManager;
+import org.apache.tika.batch.FileResourceConsumer;
+import org.apache.tika.batch.FileResourceCrawler;
+import org.apache.tika.batch.SimpleLogStatusReporter;
+import org.w3c.dom.Node;
+
+import java.util.List;
+import java.util.Map;
+
+public interface StatusReporterBuilder {
+
+ public SimpleLogStatusReporter build(FileResourceCrawler crawler, ConsumersManager consumers,
+ Node n, Map commandlineArguments);
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/CommandLineInterrupter.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/CommandLineInterrupter.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/CommandLineInterrupter.java (revision 0)
@@ -0,0 +1,51 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+
+/**
+ * Class that waits for input on System.in. If the user enters a keystroke on
+ * System.in, this will send a signal to the FileResourceRunner to shutdown gracefully.
+ *
+ */
+public class CommandLineInterrupter implements IInterrupter {
+
+ public IFileProcessorFutureResult call(){
+ try{
+ BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, "UTF-8"));
+ while (true){
+ if (reader.ready()){
+ String line = reader.readLine();
+ break;
+ } else {
+ Thread.sleep(1000);
+ }
+ }
+ } catch (InterruptedException e){
+ //canceller was interrupted
+ } catch (IOException e){
+ //TODO: log?
+ //e.printStackTrace();
+ }
+ return new InterrupterFutureResult();
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/batch/IInterrupter.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/batch/IInterrupter.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/batch/IInterrupter.java (revision 0)
@@ -0,0 +1,23 @@
+package org.apache.tika.batch;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.concurrent.Callable;
+
+public interface IInterrupter extends Callable {
+}
Index: tika-batch/src/main/java/org/apache/tika/util/DurationFormatUtils.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/util/DurationFormatUtils.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/util/DurationFormatUtils.java (revision 0)
@@ -0,0 +1,62 @@
+package org.apache.tika.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Functionality and naming conventions (roughly) copied from org.apache.commons.lang3
+ * so that we didn't have to add another dependency.
+ *
+ */
+public class DurationFormatUtils {
+
+ public static String formatMillis(long duration) {
+ duration = Math.abs(duration);
+ StringBuilder sb = new StringBuilder();
+ int secs = (int) (duration / 1000) % 60 ;
+ int mins = (int) ((duration / (1000*60)) % 60);
+ int hrs = (int) ((duration / (1000*60*60)) % 24);
+ int days = (int) ((duration / (1000*60*60*24)) % 7);
+
+ //sb.append(millis + " milliseconds");
+ addUnitString(sb, days, "day");
+ addUnitString(sb, hrs, "hour");
+ addUnitString(sb, mins, "minute");
+ addUnitString(sb, secs, "second");
+ if (duration < 1000){
+ addUnitString(sb, duration, "millisecond");
+ }
+
+ return sb.toString();
+ }
+
+ private static void addUnitString(StringBuilder sb, long unit, String unitString){
+
+ if (sb.length() > 0){
+ sb.append(", ");
+ }
+ if (unit == 1){
+ sb.append("1 ");
+ sb.append(unitString);
+ } else if (unit > 1){
+ sb.append(unit);
+ sb.append(" ");
+ sb.append(unitString);
+ sb.append("s");
+ }
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/util/PropsUtil.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/util/PropsUtil.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/util/PropsUtil.java (revision 0)
@@ -0,0 +1,87 @@
+package org.apache.tika.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+
+/**
+ * Utility class to handle properties. If the value is null,
+ * then the defaultMissing value will be returned.
+ */
+public class PropsUtil {
+
+ public static boolean getBoolean(String v, boolean defaultMissing) {
+ if (v == null || v.length() == 0) {
+ return defaultMissing;
+ }
+ if (v.toLowerCase(BatchLocalization.getLocale()).equals("true")) {
+ return true;
+ }
+ if (v.toLowerCase(BatchLocalization.getLocale()).equals("false")) {
+ return false;
+ }
+ return defaultMissing;
+ }
+
+ public static int getInt(String v, int defaultMissing) {
+ if (v == null || v.length() == 0) {
+ return defaultMissing;
+ }
+ try {
+ return Integer.parseInt(v);
+ } catch (NumberFormatException e) {
+
+ }
+ return defaultMissing;
+ }
+
+ public static long getLong(String v, long defaultMissing) {
+ if (v == null || v.length() == 0) {
+ return defaultMissing;
+ }
+ try {
+ return Long.parseLong(v);
+ } catch (NumberFormatException e) {
+
+ }
+ return defaultMissing;
+ }
+
+
+ public static File getFile(String v, File defaultMissing) {
+ if (v == null || v.length() == 0) {
+ return defaultMissing;
+ }
+ //trim initial and final " if they exist
+ if (v.startsWith("\"")) {
+ v = v.substring(1);
+ }
+ if (v.endsWith("\"")) {
+ v = v.substring(0, v.length()-1);
+ }
+
+ return new File(v);
+ }
+
+ public static String getString(String v, String defaultMissing) {
+ if (v == null) {
+ return defaultMissing;
+ }
+ return v;
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/util/ClassLoaderUtil.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/util/ClassLoaderUtil.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/util/ClassLoaderUtil.java (revision 0)
@@ -0,0 +1,41 @@
+package org.apache.tika.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class ClassLoaderUtil {
+
+ @SuppressWarnings("unchecked")
+ public static T buildClass(Class iface, String className) {
+
+ ClassLoader loader = ClassLoader.getSystemClassLoader();
+ Class clazz;
+ try {
+ clazz = loader.loadClass(className);
+ if (iface.isAssignableFrom(clazz)) {
+ return (T) clazz.newInstance();
+ }
+ throw new IllegalArgumentException(iface.toString() + " is not assignable from " + className);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/util/BatchLocalization.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/util/BatchLocalization.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/util/BatchLocalization.java (revision 0)
@@ -0,0 +1,35 @@
+package org.apache.tika.util;
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Locale;
+
+/**
+ * Simple class to handle localization issues.
+ * We should probably push these into TikaConfig and propagate
+ * changes throughout code base.
+ */
+public class BatchLocalization {
+
+ public static Locale getLocale() {
+ return Locale.ENGLISH;
+ }
+
+ public static String getEncoding() {
+ return "UTF-8";
+ }
+}
Index: tika-batch/src/main/java/org/apache/tika/util/XMLDOMUtil.java
===================================================================
--- tika-batch/src/main/java/org/apache/tika/util/XMLDOMUtil.java (revision 0)
+++ tika-batch/src/main/java/org/apache/tika/util/XMLDOMUtil.java (revision 0)
@@ -0,0 +1,109 @@
+package org.apache.tika.util;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class XMLDOMUtil {
+
+ /**
+ * This grabs the attributes from a dom node and overwrites those values with those
+ * specified by the overwrite map.
+ *
+ * @param node
+ * @param overwrite
+ * @return
+ */
+ public static Map mapifyAttrs(Node node, Map overwrite) {
+ Map map = new HashMap();
+ NamedNodeMap nnMap = node.getAttributes();
+ for (int i = 0; i < nnMap.getLength(); i++) {
+ Node attr = nnMap.item(i);
+ map.put(attr.getNodeName(), attr.getNodeValue());
+ }
+ if (overwrite != null) {
+ for (Map.Entry e : overwrite.entrySet()) {
+ map.put(e.getKey(), e.getValue());
+ }
+ }
+ return map;
+ }
+
+
+ /**
+ * Get an int value. Try the runtime attributes first and then back off to
+ * the document element. Throw a RuntimeException if the attribute is not
+ * found or if the value is not parseable as an int.
+ *
+ * @param attrName attribute name to find
+ * @param runtimeAttributes runtime attributes
+ * @param docElement correct element that should have specified attribute
+ * @return specified int value
+ */
+ public static int getInt(String attrName, Map runtimeAttributes, Node docElement) {
+ String stringValue = getStringValue(attrName, runtimeAttributes, docElement);
+ if (stringValue != null) {
+ try {
+ return Integer.parseInt(stringValue);
+ } catch (NumberFormatException e) {
+ //swallow
+ }
+ }
+ throw new RuntimeException("Need to specify a parseable int value in -- "
+ +attrName+" -- in commandline or in config file!");
+ }
+
+
+ /**
+ * Get a long value. Try the runtime attributes first and then back off to
+ * the document element. Throw a RuntimeException if the attribute is not
+ * found or if the value is not parseable as a long.
+ *
+ * @param attrName attribute name to find
+ * @param runtimeAttributes runtime attributes
+ * @param docElement correct element that should have specified attribute
+ * @return specified long value
+ */
+ public static long getLong(String attrName, Map runtimeAttributes, Node docElement) {
+ String stringValue = getStringValue(attrName, runtimeAttributes, docElement);
+ if (stringValue != null) {
+ try {
+ return Long.parseLong(stringValue);
+ } catch (NumberFormatException e) {
+ //swallow
+ }
+ }
+ throw new RuntimeException("Need to specify a \"long\" value in -- "
+ +attrName+" -- in commandline or in config file!");
+ }
+
+ private static String getStringValue(String attrName, Map runtimeAttributes, Node docElement) {
+ String stringValue = runtimeAttributes.get(attrName);
+ if (stringValue == null) {
+ Node staleNode = docElement.getAttributes().getNamedItem(attrName);
+ if (staleNode != null) {
+ stringValue = staleNode.getNodeValue();
+ }
+ }
+ return stringValue;
+ }
+}
Index: tika-batch/src/main/resources/tika-batch-config.xml
===================================================================
--- tika-batch/src/main/resources/tika-batch-config.xml (revision 0)
+++ tika-batch/src/main/resources/tika-batch-config.xml (revision 0)
@@ -0,0 +1,41 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
Index: tika-batch/pom.xml
===================================================================
--- tika-batch/pom.xml (revision 0)
+++ tika-batch/pom.xml (revision 0)
@@ -0,0 +1,159 @@
+
+
+
+ 4.0.0
+
+ 1.2
+
+
+
+ org.apache.tika
+ tika-parent
+ 1.7-SNAPSHOT
+ ../tika-parent/pom.xml
+
+
+ tika-batch
+ bundle
+ Apache Tika batch
+ http://tika.apache.org/
+
+
+
+ ${project.groupId}
+ tika-core
+ ${project.version}
+
+
+ ${project.groupId}
+ tika-parsers
+ ${project.version}
+
+
+ ${project.groupId}
+ tika-serialization
+ ${project.version}
+
+
+ ${project.groupId}
+ tika-xmp
+ ${project.version}
+
+
+ org.slf4j
+ slf4j-log4j12
+ 1.5.6
+
+
+ com.google.code.gson
+ gson
+ 1.7.1
+
+
+ commons-cli
+ commons-cli
+ ${cli.version}
+
+
+ org.apache.tika
+ tika-parsers
+ ${project.version}
+ test-jar
+ test
+
+
+ junit
+ junit
+ test
+
+
+ commons-io
+ commons-io
+ test
+ 2.1
+
+
+
+
+
+
+
+ org.apache.felix
+ maven-bundle-plugin
+ true
+
+
+ ${project.url}
+
+ org.apache.tika.config.TikaActivator
+
+ lazy
+
+
+
+
+ org.apache.rat
+ apache-rat-plugin
+
+
+ src/test/resources/org/apache/tika/**
+
+
+
+
+ org.apache.maven.plugins
+ maven-jar-plugin
+
+
+
+ test-jar
+
+
+
+
+
+ maven-failsafe-plugin
+ 2.10
+
+
+
+ ${project.build.directory}/${project.build.finalName}.jar
+
+
+
+
+
+
+ integration-test
+ verify
+
+
+
+
+
+
+
+
+
+
+ The Apache Software Foundation
+ http://www.apache.org
+
+
+ http://svn.apache.org/viewvc/tika/trunk/tika-batch
+ scm:svn:http://svn.apache.org/repos/asf/tika/trunk/tika-batch
+ scm:svn:https://svn.apache.org/repos/asf/tika/trunk/tika-batch
+
+
+ JIRA
+ https://issues.apache.org/jira/browse/TIKA
+
+
+ Jenkins
+ https://builds.apache.org/job/Tika-trunk/
+
+
+
+
\ No newline at end of file