Index: src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java =================================================================== --- src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java (revision 1221194) +++ src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java (working copy) @@ -93,7 +93,7 @@ int i = fname.lastIndexOf('.'); if (i != -1) { // Trying to resolve the Mime-Type - String contentType = MIME.getMimeType(fname).getName(); + String contentType = MIME.getMimeType(fname); try { Metadata metadata = new Metadata(); metadata.set(Response.CONTENT_LENGTH, Long.toString(entry.getSize())); Index: src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java =================================================================== --- src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java (revision 1221194) +++ src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java (working copy) @@ -33,7 +33,7 @@ import org.apache.nutch.net.protocols.Response; // Tika imports -import org.apache.tika.mime.MimeType; +import org.apache.tika.Tika; // Hadoop imports import org.apache.hadoop.conf.Configuration; @@ -74,6 +74,7 @@ private Configuration conf; private MimeUtil MIME; + private Tika tika; /** Returns the response code. */ public int getCode() { @@ -103,6 +104,7 @@ this.conf = conf; MIME = new MimeUtil(conf); + tika = new Tika(); if (!"file".equals(url.getProtocol())) throw new FileException("Not a file url:" + url); @@ -216,10 +218,10 @@ headers.set(Response.LAST_MODIFIED, HttpDateFormat.toString(f.lastModified())); - MimeType mimeType = MIME.getMimeType(f); - String mimeTypeString = mimeType != null ? mimeType.getName() : ""; - headers.set(Response.CONTENT_TYPE, mimeTypeString); + String mimeType = MIME.getMimeType(f); + headers.set(Response.CONTENT_TYPE, mimeType != null ? mimeType : ""); + // response code this.code = 200; // http OK } Index: src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java =================================================================== --- src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java (revision 1221194) +++ src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java (working copy) @@ -185,7 +185,7 @@ * @return */ private NutchDocument addType(NutchDocument doc, ParseData data, String url) { - MimeType mimeType = null; + String mimeType = null; String contentType = data.getMeta(Response.CONTENT_TYPE); if (contentType == null) { // Note by Jerome Charron on 20050415: @@ -209,13 +209,13 @@ return doc; } - contentType = mimeType.getName(); + contentType = mimeType; doc.add("type", contentType); // Check if we need to split the content type in sub parts if (conf.getBoolean("moreIndexingFilter.indexMimeTypeParts", true)) { - String[] parts = getParts(contentType.toString()); + String[] parts = getParts(contentType); for(String part: parts) { doc.add("type", part); Index: src/java/org/apache/nutch/util/MimeUtil.java =================================================================== --- src/java/org/apache/nutch/util/MimeUtil.java (revision 1221194) +++ src/java/org/apache/nutch/util/MimeUtil.java (working copy) @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; // Tika imports +import org.apache.tika.Tika; import org.apache.tika.mime.MimeType; import org.apache.tika.mime.MimeTypeException; import org.apache.tika.mime.MimeTypes; @@ -49,6 +50,9 @@ /* our Tika mime type registry */ private MimeTypes mimeTypes; + /* the tika detectors */ + private Tika tika; + /* whether or not magic should be employed or not */ private boolean mimeMagic; @@ -56,6 +60,7 @@ private static final Logger LOG = LoggerFactory.getLogger(MimeUtil.class.getName()); public MimeUtil(Configuration conf) { + tika = new Tika(); ObjectCache objectCache = ObjectCache.get(conf); MimeTypes mimeTypez = (MimeTypes) objectCache.getObject(MimeTypes.class .getName()); @@ -133,6 +138,7 @@ * @return The correctly, automatically guessed {@link MimeType} name. */ public String autoResolveContentType(String typeName, String url, byte[] data) { + String strMimeType = null; MimeType type = null; String cleanedMimeType = null; @@ -166,54 +172,58 @@ // if it is, and it's not the default mime type, then go with the mime type // returned by the magic if (this.mimeMagic) { - MimeType magicType = this.mimeTypes.getMimeType(data); - if (magicType != null && !magicType.getName().equals(MimeTypes.OCTET_STREAM) - && !magicType.getName().equals(MimeTypes.PLAIN_TEXT) - && type != null && !type.getName().equals(magicType.getName())) { + strMimeType = tika.detect(data); + + // Deprecated in Tika 1.0 See https://issues.apache.org/jira/browse/NUTCH-1230 + //MimeType magicType = this.mimeTypes.getMimeType(data); + if (strMimeType != null && !strMimeType.equals(MimeTypes.OCTET_STREAM) + && !strMimeType.equals(MimeTypes.PLAIN_TEXT) + && type != null && !type.getName().equals(strMimeType)) { + // If magic enabled and the current mime type differs from that of the // one returned from the magic, take the magic mimeType - type = magicType; + return strMimeType; } // if type is STILL null after all the resolution strategies, go for the // default type - if (type == null) { + if (strMimeType == null) { try { - type = this.mimeTypes.forName(MimeTypes.OCTET_STREAM); + strMimeType = MimeTypes.OCTET_STREAM; } catch (Exception ignore) { } } } - return type.getName(); + return strMimeType; } /** * Facade interface to Tika's underlying {@link MimeTypes#getMimeType(String)} * method. - * + * * @param url * A string representation of the document {@link URL} to sense the * {@link MimeType} for. * @return An appropriate {@link MimeType}, identified from the given * Document url in string form. */ - public MimeType getMimeType(String url) { - return this.mimeTypes.getMimeType(url); + public String getMimeType(String url) { + return tika.detect(url); } /** * A facade interface to Tika's underlying {@link MimeTypes#forName(String)} * method. - * + * * @param name * The name of a valid {@link MimeType} in the Tika mime registry. * @return The object representation of the {@link MimeType}, if it exists, * or null otherwise. */ - public MimeType forName(String name) { + public String forName(String name) { try { - return this.mimeTypes.forName(name); + return this.mimeTypes.forName(name).toString(); } catch (MimeTypeException e) { LOG.error("Exception getting mime type by name: [" + name + "]: Message: " + e.getMessage()); @@ -224,14 +234,21 @@ /** * Facade interface to Tika's underlying {@link MimeTypes#getMimeType(File)} * method. - * + * * @param f * The {@link File} to sense the {@link MimeType} for. * @return The {@link MimeType} of the given {@link File}, or null if it * cannot be determined. */ - public MimeType getMimeType(File f) { - return this.mimeTypes.getMimeType(f); + public String getMimeType(File f) { + try { + return tika.detect(f); + } catch (Exception e) { + LOG.error("Exception getting mime type for file: [" + f.getPath() + + "]: Message: " + e.getMessage()); + return null; + } } -} + +} \ No newline at end of file