Index: tika-core/src/main/resources/org/apache/tika/language/tika.language.properties =================================================================== --- tika-core/src/main/resources/org/apache/tika/language/tika.language.properties (revision 0) +++ tika-core/src/main/resources/org/apache/tika/language/tika.language.properties (revision 0) @@ -0,0 +1,46 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is a tika LanguageIdentifier properties file. +# Its name is org/apache/tika/language/tika.language.properties +# You can override it by placing a copy on the classpath in a file called +# org/apache/tika/language/tika.language.override.properties + +# List of languages for which there are .ngp profiles +# If there exists an ISO 639-1 2-letter code it should be used +# If not, you can choose an ISO 639-2 3-letter code +# See http://www.loc.gov/standards/iso639-2/php/code_list.php +languages=da,de,et,el,en,es,fi,fr,hu,is,it,nl,no,pl,pt,ru,sv,th + +# List of language names in english +name.da=Danish +name.de=German +name.et=Estonian +name.el=Greek +name.en=English +name.es=Spanish +name.fi=Finnish +name.fr=French +name.hu=Hungarian +name.is=Icelandic +name.it=Italian +name.nl=Dutch +name.no=Norwegian +name.pl=Polish +name.pt=Portuguese +name.ru=Russian +name.sv=Swedish +name.th=Thai \ No newline at end of file Index: tika-core/src/main/java/org/apache/tika/language/ProfilingWriter.java =================================================================== --- tika-core/src/main/java/org/apache/tika/language/ProfilingWriter.java (revision 987950) +++ tika-core/src/main/java/org/apache/tika/language/ProfilingWriter.java (working copy) @@ -54,12 +54,20 @@ /** * Returns the language that best matches the current state of the - * language profile. + * language profile. Since TIKA-490, this method may return null if there was some + * problem initializing the LanguageIdentifier (e.g., there was some IOException + * loading the language profiles from the classpath). * - * @return language that best matches the current profile + * @return language that best matches the current profile, or null if + * there is an error initializing the language identifier. */ public LanguageIdentifier getLanguage() { - return new LanguageIdentifier(profile); + try{ + return new LanguageIdentifier(profile); + } + catch(Exception e){ + return null; + } } @Override Index: tika-core/src/main/java/org/apache/tika/language/LanguageIdentifier.java =================================================================== --- tika-core/src/main/java/org/apache/tika/language/LanguageIdentifier.java (revision 987950) +++ tika-core/src/main/java/org/apache/tika/language/LanguageIdentifier.java (working copy) @@ -17,10 +17,13 @@ package org.apache.tika.language; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.HashMap; import java.util.Map; +import java.util.Properties; +import java.util.Set; /** * Identifier of the language that best matches a given content profile. @@ -30,26 +33,42 @@ * @since Apache Tika 0.5 * @see * Europarl: A Parallel Corpus for Statistical Machine Translation - * @see + * @see * ISO 639 Language Codes */ public class LanguageIdentifier { - + /** * The available language profiles. */ private static final Map PROFILES = new HashMap(); + private static final String PROFILE_SUFFIX = ".ngp"; + private static final String PROFILE_ENCODING = "UTF-8"; - private static void addProfile(String language) { + private static Properties props = new Properties(); + private static String errors = ""; + + private static final String PROPERTIES_OVERRIDE_FILE = "tika.language.override.properties"; + private static final String PROPERTIES_FILE = "tika.language.properties"; + private static final String LANGUAGES_KEY = "languages"; + + /* + * Always attempt initializing language profiles when class is loaded first time + */ + static { + initProfiles(false); + } + + private static void addProfile(String language) throws Exception { try { LanguageProfile profile = new LanguageProfile(); InputStream stream = - LanguageIdentifier.class.getResourceAsStream(language + ".ngp"); + LanguageIdentifier.class.getResourceAsStream(language + PROFILE_SUFFIX); try { BufferedReader reader = - new BufferedReader(new InputStreamReader(stream, "UTF-8")); + new BufferedReader(new InputStreamReader(stream, PROFILE_ENCODING)); String line = reader.readLine(); while (line != null) { if (line.length() > 0 && !line.startsWith("#")) { @@ -66,35 +85,18 @@ PROFILES.put(language, profile); } catch (Throwable t) { - // Failed to load this language profile. Log the problem? + throw new Exception("Failed trying to load language profile for language \""+language+"\". Error: "+t.getMessage()); } } - static { - addProfile("da"); // Danish - addProfile("de"); // German - addProfile("et"); // Estonian - addProfile("el"); // Greek - addProfile("en"); // English - addProfile("es"); // Spanish - addProfile("fi"); // Finnish - addProfile("fr"); // French - addProfile("hu"); // Hungarian - addProfile("is"); // Icelandic - addProfile("it"); // Italian - addProfile("nl"); // Dutch - addProfile("no"); // Norwegian - addProfile("pl"); // Polish - addProfile("pt"); // Portuguese - addProfile("ru"); // Russian - addProfile("sv"); // Swedish - addProfile("th"); // Thai - } - private final String language; private final double distance; - + + /** + * Constructs a language identifier based on a LanguageProfile + * @param profile + */ public LanguageIdentifier(LanguageProfile profile) { String minLanguage = "unknown"; double minDistance = 1.0; @@ -110,18 +112,106 @@ this.distance = minDistance; } + /** + * Constructs a language identifier based on a String of text content + * @param content + */ public LanguageIdentifier(String content) { this(new LanguageProfile(content)); } + /** + * Gets the identified language + * @return an ISO 639 code representing the detected language + */ public String getLanguage() { return language; } + /** + * Tries to judge whether the identification is certain enough + * to be trusted. + * WARNING: Will never return true for small amount of input texts. + * @return + */ public boolean isReasonablyCertain() { return distance < 0.022; } + /** + * Builds the language profiles. + * The list of languages are fetched from a property file named "tika.language.properties" + * If a file called "tika.language.override.properties" is found, this is used instead + * The property file contains a key "languages" with values being comma-separated language codes + * @param force Forces reload of property file and language profiles + */ + public static void initProfiles(boolean force) { + if(force) + PROFILES.clear(); + + if(PROFILES.isEmpty()) { + errors = ""; + InputStream stream; + stream = LanguageIdentifier.class.getResourceAsStream(PROPERTIES_OVERRIDE_FILE); + if(stream == null) + stream = LanguageIdentifier.class.getResourceAsStream(PROPERTIES_FILE); + + if(stream != null){ + try { + props = new Properties(); + props.load(stream); + } catch (IOException e) { + errors += "IOException while trying to load property file. Message: " + e.getMessage() + "\n"; + } + } + + String[] languages = props.getProperty(LANGUAGES_KEY).split(","); + for(String language : languages) { + language = language.trim(); + String name = props.getProperty("name."+language, "Unknown"); + try { + addProfile(language); + } catch (Exception e) { + errors += "Language " + language + " (" + name + ") not initialized. Message: " + e.getMessage() + "\n"; + } + } + } + } + + /** + * Tests whether there were errors initializing language config + * @return true if there are errors. Use getErrors() to retrieve. + */ + public static boolean hasErrors() { + return errors != ""; + } + + /** + * Returns a string of error messages related to initializing langauge profiles + * @return + */ + public static String getErrors() { + return errors; + } + + /** + * Returns what languages are supported for language identification + * @return A set of Strings being the ISO 639 language codes + */ + public static Set getSupportedLanguages() { + return PROFILES.keySet(); + } + + /** + * Returns the distance measure between input text and detected language + * This can be used to judge whether the classification is certain or not. + * The method isReasonablyCertain() uses 0.022 as a threshold + * @return + */ + public double getDistance() { + return distance; + } + @Override public String toString() { return language + " (" + distance + ")"; Index: tika-core/src/test/java/org/apache/tika/language/LanguageIdentifierTest.java =================================================================== --- tika-core/src/test/java/org/apache/tika/language/LanguageIdentifierTest.java (revision 987950) +++ tika-core/src/test/java/org/apache/tika/language/LanguageIdentifierTest.java (working copy) @@ -43,8 +43,8 @@ for (String language : languages) { ProfilingWriter writer = new ProfilingWriter(); writeTo(language, writer); - LanguageIdentifier identifier = - new LanguageIdentifier(writer.getProfile()); + LanguageIdentifier identifier = null; + identifier = new LanguageIdentifier(writer.getProfile()); assertTrue(identifier.toString(), identifier.isReasonablyCertain()); assertEquals(language, identifier.getLanguage()); } @@ -57,8 +57,8 @@ ProfilingWriter writer = new ProfilingWriter(); writeTo(language, writer); writeTo(other, writer); - LanguageIdentifier identifier = - new LanguageIdentifier(writer.getProfile()); + LanguageIdentifier identifier = null; + identifier = new LanguageIdentifier(writer.getProfile()); assertFalse(identifier.isReasonablyCertain()); } }