[Mulgara-svn] r1367 - trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene
ronald at mulgara.org
ronald at mulgara.org
Mon Oct 27 12:38:56 UTC 2008
Author: ronald
Date: 2008-10-27 05:38:55 -0700 (Mon, 27 Oct 2008)
New Revision: 1367
Added:
trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneIndexerCache.java
Modified:
trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/FullTextStringIndex.java
trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/FullTextStringIndexUnitTest.java
trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneResolver.java
trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneResolverFactory.java
trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/ReadOnlyLuceneResolver.java
Log:
Added caching of the lucene indexers. In one case this took the query time
from 330ms down to 130ms.
As part of this, the directory operations (creating and deleting directory)
have been moved into the LuceneIndexerCache. Also, the ResolverFactory.delete
has been implemented.
Modified: trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/FullTextStringIndex.java
===================================================================
--- trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/FullTextStringIndex.java 2008-10-27 12:38:48 UTC (rev 1366)
+++ trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/FullTextStringIndex.java 2008-10-27 12:38:55 UTC (rev 1367)
@@ -28,7 +28,6 @@
package org.mulgara.resolver.lucene;
// Java 2 standard packages
-import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
@@ -57,14 +56,7 @@
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockObtainFailedException;
-import org.mulgara.util.TempDir;
-
-
/**
* The utility class which provides an interface of adding, finding and removing
* statements and documents for Lucene.
@@ -91,6 +83,10 @@
/** Logger. This is named after the class. */
private final static Logger logger = Logger.getLogger(FullTextStringIndex.class);
+ //
+ // Constants
+ //
+
/** The field name for the actual literal */
public final static String ID_KEY = "id";
@@ -106,88 +102,61 @@
/** The field name for the reverse literal */
public final static String REVERSE_LITERAL_KEY = "reverseliteral";
- /** Disable reverse literal lookup */
- private static boolean enableReverseTextIndex;
-
//
- // Constants
+ // Fields
//
- /** The Directory for Lucene. */
- private Directory luceneIndexDirectory;
+ /** The lucene indexer cache */
+ private LuceneIndexerCache indexerCache;
- /** The name of the directory */
- private String indexDirectoryName;
-
/** The index writer */
private IndexWriter indexer;
/** The index searcher */
private IndexSearcher indexSearcher;
+ /** Whether any modifications have been made to the index. */
+ private boolean madeMods = false;
+
+ /** Whether to close the indexers when closing this index */
+ private boolean closeIndexers = false;
+
+ /** Enable reverse literal lookup */
+ private boolean enableReverseTextIndex;
+
/** Analyzer used for writing and reading */
private Analyzer analyzer = getAnalyzer();
- private String name;
-
/**
- * Constructor for the FullTextStringIndex object. Uses the system
- * property "mulgara.textindex.reverse.enabled" to set the desired value or
- * will default to "false" if not set.
+ * Create a new FullTextStringIndex object. Uses the system property
+ * "mulgara.textindex.reverse.enabled" to set the desired value for
+ * enableReverseTextIndex, or will default to "false" if not set.
*
- * @param directory the directory to put the index files.
- * @param newName
- * @throws FullTextStringIndexException Failure to initialize write index
+ * @param indexerCache the indexer-cache to use to get the indexers
+ * @param forWrites whether writes will occur or not
+ * @throws FullTextStringIndexException on failure to obtain an index reader or writer
*/
- public FullTextStringIndex(String directory, String newName, boolean forWrites)
+ public FullTextStringIndex(LuceneIndexerCache indexerCache, boolean forWrites)
throws FullTextStringIndexException {
- name = newName;
- enableReverseTextIndex = System.getProperty(
- "mulgara.textindex.reverse.enabled", "false").equalsIgnoreCase("true");
- init(directory, forWrites);
+ this(indexerCache, forWrites, Boolean.getBoolean("mulgara.textindex.reverse.enabled"));
}
/**
- * Constructor for the FullTextStringIndex object
+ * Create a new FullTextStringIndex object.
*
- * @param directory the directory to put the index files.
- * @param newName
- * @param newEnableReverseTextIndex true if you can begin Lucene queries with
- * wildcards.
- * @throws FullTextStringIndexException Failure to initialize write index
+ * @param indexerCache the indexer-cache to use to get the indexers
+ * @param newEnableReverseTextIndex true if you can begin Lucene queries with wildcards.
+ * @throws FullTextStringIndexException on failure to obtain an index reader or writer
*/
- public FullTextStringIndex(String directory, String newName,
- boolean newEnableReverseTextIndex, boolean forWrites) throws FullTextStringIndexException {
- name = newName;
- enableReverseTextIndex = newEnableReverseTextIndex;
- init(directory, forWrites);
+ public FullTextStringIndex(LuceneIndexerCache indexerCache, boolean forWrites,
+ boolean enableReverseTextIndex)
+ throws FullTextStringIndexException {
+ this.indexerCache = indexerCache;
+ this.enableReverseTextIndex = enableReverseTextIndex;
+ initialize(forWrites);
}
/**
- * Initialize the store.
- *
- * @param directory the directory to put the index files.
- * @param forWrites Whether to open an index writer
- * @throws FullTextStringIndexException Failure to initialize write index
- */
- private void init(String directory, boolean forWrites) throws FullTextStringIndexException {
- try {
- Lock lock = FSDirectory.getDirectory(TempDir.getTempDir().getPath()).makeLock(name + ".lock");
-
- synchronized (lock) {
- lock.obtain();
-
- // create/open the indexes for reading and writing.
- initialize(directory, forWrites);
-
- lock.release();
- }
- } catch (IOException ioe) {
- throw new FullTextStringIndexException(ioe);
- }
- }
-
- /**
* Get an instance of the analyzer used on text to produce the index.
*
* @return The analyzer used.
@@ -325,9 +294,11 @@
indexDocument.add(new Field(SUBJECT_KEY, subject, Field.Store.YES, Field.Index.NOT_ANALYZED));
try {
- indexer.addDocument(indexDocument);
+ indexer.addDocument(indexDocument, analyzer);
added = true;
+ madeMods = true;
} catch (IOException ex) {
+ closeIndexers = true;
logger.error("Unable to add fulltext string subject <" + subject + "> predicate <" +
predicate + "> literal <'" + literal + "'> to fulltext string index", ex);
throw new FullTextStringIndexException(
@@ -401,9 +372,11 @@
indexDocument.add(new Field(SUBJECT_KEY, subject, Field.Store.YES, Field.Index.NOT_ANALYZED));
try {
- indexer.addDocument(indexDocument);
+ indexer.addDocument(indexDocument, analyzer);
added = true;
+ madeMods = true;
} catch (IOException ex) {
+ closeIndexers = true;
logger.error("Unable to add fulltext string subject <" + subject + "> predicate <" +
predicate + "> resource <" + resource + "> to fulltext string index", ex);
throw new FullTextStringIndexException(
@@ -439,9 +412,11 @@
}
try {
- indexer.addDocument(indexDocument);
+ indexer.addDocument(indexDocument, analyzer);
added = true;
+ madeMods = true;
} catch (IOException ex) {
+ closeIndexers = true;
logger.error("Unable to add " + indexDocument + " to fulltext string index", ex);
throw new FullTextStringIndexException("Unable to add " + indexDocument + " to fulltext string index", ex);
}
@@ -450,52 +425,6 @@
}
/**
- * Remove all index files from the current initialised directory WARNING : All
- * files are removed in the specified directory. See {@link #removeAll} for an
- * alternate solution.
- *
- * @return return true if successful at removing all index files
- * @throws FullTextStringIndexException Exception occurs when attempting to
- * close the indexes
- */
- public boolean removeAllIndexes() throws FullTextStringIndexException {
- // debug logging
- if (logger.isDebugEnabled()) {
- logger.debug("Removing all indexes from " + luceneIndexDirectory.toString());
- }
-
- boolean deleted = false;
-
- //Delete the directory if it exists
- if (luceneIndexDirectory != null) {
- //Close the reading and writing indexes
- close();
-
- try {
- Lock lock = FSDirectory.getDirectory(TempDir.getTempDir().getPath()).makeLock(name + ".lock");
-
- synchronized (lock) {
- lock.obtain();
-
- //Remove all files from the directory
- for (String file : luceneIndexDirectory.list()) {
- luceneIndexDirectory.deleteFile(file);
- }
-
- //Remove the directory
- deleted = new File(indexDirectoryName).delete();
-
- lock.release();
- }
- } catch (IOException ioe) {
- throw new FullTextStringIndexException(ioe);
- }
- }
-
- return deleted;
- }
-
- /**
* Remove the extact string from the fulltext string pool
*
* @param subject subject must be supplied
@@ -524,6 +453,7 @@
Term term = new Term(ID_KEY, key);
indexer.deleteDocuments(term);
removed = true; // TODO: could use docCount(), but that seems overly expensive
+ madeMods = true;
if (logger.isDebugEnabled()) {
if (removed) {
@@ -533,6 +463,7 @@
}
}
} catch (IOException ex) {
+ closeIndexers = true;
logger.error("Unable to delete the string '" + key + "'", ex);
throw new FullTextStringIndexException("Unable to delete the string '" + key + "'", ex);
}
@@ -541,51 +472,45 @@
}
/**
- * Remove all entries in the string pool. Unlike {@link removeAllIndexes}, this may be
- * called while readers are active. However, this method may be very slow. Also note
- * that this will <strong>not</strong> remove entries that have been added as part of
- * the current transaction!
+ * Remove all entries in the string pool. Unlike {@link LuceneIndexerCache#removeAllIndexes},
+ * this may be * called while readers are active. However, this method may be very slow. Also
+ * note that this will <strong>not</strong> remove entries that have been added as part of the
+ * current transaction!
*
* @throws FullTextStringIndexException Exception occurs when attempting to remove the documents
*/
public void removeAll() throws FullTextStringIndexException {
// debug logging
if (logger.isDebugEnabled()) {
- logger.debug("Removing all documents from " + luceneIndexDirectory.toString());
+ logger.debug("Removing all documents from " + indexerCache.getDirectory());
}
try {
indexer.deleteDocuments(new MatchAllDocsQuery());
+ madeMods = true;
} catch (IOException ex) {
+ closeIndexers = true;
logger.error("Unable to delete all documents", ex);
throw new FullTextStringIndexException("Unable to delete all documents", ex);
}
}
/**
- * Close the indexes on disk.
- *
- * @throws FullTextStringIndexException if there is an error whilst saving the
- * index.
+ * Close this index and return the indexers to the cache.
*/
- public void close() throws FullTextStringIndexException {
+ public void close() {
if (logger.isDebugEnabled()) {
logger.debug("Closing fulltext indexes");
}
- try {
- if (indexer != null) {
- indexer.close();
- indexer = null;
- }
+ if (indexer != null) {
+ indexerCache.returnWriter(indexer, closeIndexers);
+ indexer = null;
+ }
- if (indexSearcher != null) {
- indexSearcher.close();
- indexSearcher = null;
- }
- } catch (IOException ex) {
- logger.error("Unable to close fulltext string pool indexes", ex);
- throw new FullTextStringIndexException("Unable to close fulltext string pool indexes", ex);
+ if (indexSearcher != null) {
+ indexerCache.returnReader(indexSearcher.getIndexReader(), closeIndexers);
+ indexSearcher = null;
}
}
@@ -598,12 +523,13 @@
if (indexer == null) return;
if (logger.isInfoEnabled()) {
- logger.info("Optimizing fulltext index at " + indexDirectoryName + " please wait...");
+ logger.info("Optimizing fulltext index at " + indexerCache.getDirectory() + " please wait...");
}
try {
indexer.optimize();
} catch (IOException ex) {
+ closeIndexers = true;
logger.error("Unable to optimize existing fulltext string pool index", ex);
throw new FullTextStringIndexException("Unable to optimize existing fulltext string pool index", ex);
}
@@ -679,6 +605,7 @@
logger.debug("Got hits: " + hits.length());
}
} catch (IOException ex) {
+ closeIndexers = true;
logger.error("Unable to read results for query '" + bQuery.toString(LITERAL_KEY) + "'", ex);
throw new FullTextStringIndexException("Unable to read results for query '" + bQuery.toString(LITERAL_KEY) + "'", ex);
} catch (ParseException ex) {
@@ -717,6 +644,7 @@
//Perform query
indexSearcher.search(query, hits = new Hits(indexSearcher.getIndexReader()));
} catch (IOException ex) {
+ closeIndexers = true;
logger.error("Unable to read results for query '" + query.toString(LITERAL_KEY) + "'", ex);
throw new FullTextStringIndexException("Unable to read results for query '" + query.toString(LITERAL_KEY) + "'", ex);
}
@@ -725,93 +653,19 @@
}
/**
- * Verify and initialize the indexes for reading. Will automatically create
- * indexes if they do not exist.
+ * Acquire the indexers.
*
- * @param directory Directory of the index to be initialized
- * @param forWrites Whether to open an index writer
- * @throws FullTextStringIndexException IOException occurs while trying to
+ * @param forWrites whether to acquire an index writer
+ * @throws FullTextStringIndexException if an exception occurs while trying to
* locate or create the indexes
*/
- private void initialize(String directory, boolean forWrites) throws FullTextStringIndexException {
- // debug logging
- if (logger.isDebugEnabled()) {
- logger.debug("Initialization of FullTextStringIndex to directory to " + directory);
- }
-
- File indexDirectory;
-
- try {
- indexDirectoryName = directory;
- indexDirectory = new File(directory);
- } catch (Exception ex) {
- logger.error("Exception when initializing fulltext string index directory", ex);
- throw new FullTextStringIndexException("Exception when initializing fulltext string index directory", ex);
- }
-
- // Ensure index is flushed to disk before reading config.
- close();
-
- // does the directory exist?
- if (!indexDirectory.exists()) {
- // no, make it
- indexDirectory.mkdirs();
- }
-
- // ensure the index directory is a directory
- if (!indexDirectory.isDirectory()) {
- indexDirectory = null;
- logger.fatal("The fulltext string index directory '" + directory + "' is not a directory!");
- throw new FullTextStringIndexException("The fulltext string index index directory '" +
- directory + "' is not a directory!");
- }
-
- // ensure the directory is writeable
- if (forWrites && !indexDirectory.canWrite()) {
- indexDirectory = null;
- logger.fatal("The fulltext string index directory '" + directory + "' is not writeable!");
- throw new FullTextStringIndexException("The fulltext string index directory '" + directory +
- "' is not writeable!");
- }
-
- // Create lucene directory.
- try {
- luceneIndexDirectory = FSDirectory.getDirectory(directory);
- } catch (IOException ioe) {
- throw new FullTextStringIndexException(ioe);
- }
-
- assert luceneIndexDirectory != null;
-
- // Open the index for writing
+ private void initialize(boolean forWrites) throws FullTextStringIndexException {
if (forWrites) {
- try {
- openWriteIndex();
- } catch (LockObtainFailedException lofe) {
- logger.warn("Failed to obtain fulltext index directory lock; forcibly unlocking and trying again", lofe);
-
- /* If it fails once try and unlock the directory and try again. This shouldn't happen
- * unless mulgara was shut down abruptly since mulgara has a single writer lock.
- */
- try {
- IndexWriter.unlock(luceneIndexDirectory);
- } catch (IOException ioe) {
- throw new FullTextStringIndexException("Failed to unlock directory: " + luceneIndexDirectory, ioe);
- }
-
- // Try again - let it fail this time.
- try {
- openWriteIndex();
- } catch (LockObtainFailedException lofe2) {
- throw new FullTextStringIndexException(lofe2);
- }
- }
+ openWriteIndex();
}
- // Open the index for reading
openReadIndex();
- // debug logging
if (logger.isDebugEnabled()) {
logger.debug("Fulltext string index initialized");
}
@@ -821,61 +675,27 @@
* Open the index on disk for writing.
*
* @throws FullTextStringIndexException if there is an error whilst opening the index.
- * @throws LockObtainFailedException if the index is locked by another writer
*/
- private void openWriteIndex() throws FullTextStringIndexException, LockObtainFailedException {
- // debug logging
- if (logger.isDebugEnabled()) {
- logger.debug("Opening index for IndexWriter: " + luceneIndexDirectory);
- }
-
- if (indexer != null) {
- logger.error("Tried to create a new fulltext string index writer when one already exists");
- throw new FullTextStringIndexException("Tried to create a new fulltext string index writer when one already exists");
- }
-
+ private void openWriteIndex() throws FullTextStringIndexException {
try {
- indexer = new IndexWriter(luceneIndexDirectory, analyzer, IndexWriter.MaxFieldLength.LIMITED);
- } catch (LockObtainFailedException lofe) {
- throw lofe;
+ indexer = indexerCache.getWriter();
} catch (IOException ioe) {
+ closeIndexers = true;
throw new FullTextStringIndexException("Unable to open fulltext string pool index", ioe);
}
}
/**
- * (Re)open the index on disk for reading.
+ * Open the index on disk for reading.
*
* @throws FullTextStringIndexException if there is an error whilst opening the index.
*/
- void openReadIndex() throws FullTextStringIndexException {
- // debug logging
- if (logger.isDebugEnabled()) {
- logger.debug("Opening index for IndexSearcher");
- }
-
- IndexReader reader = null;
+ private void openReadIndex() throws FullTextStringIndexException {
try {
- if (indexSearcher != null) {
- IndexReader oldReader = indexSearcher.getIndexReader();
- reader = oldReader.reopen();
- if (reader != oldReader) {
- oldReader.close();
- }
- indexSearcher = null; // Note: don't need to close the searcher
- } else {
- reader = IndexReader.open(luceneIndexDirectory, true);
- }
-
- indexSearcher = new IndexSearcher(reader);
- } catch (IOException ex) {
- logger.error("Unable to open existing fulltext index for searching", ex);
- try {
- if (reader != null && indexSearcher == null) reader.close();
- } catch (IOException ioe2) {
- logger.error("Error closing existing fulltext index reader", ioe2);
- }
- throw new FullTextStringIndexException("Unable to open existing fulltext index for reading", ex);
+ indexSearcher = new IndexSearcher(indexerCache.getReader());
+ } catch (IOException ioe) {
+ closeIndexers = true;
+ throw new FullTextStringIndexException("Unable to open fulltext index for reading", ioe);
}
}
@@ -900,7 +720,10 @@
logger.debug("Comitting fulltext indexes");
}
- if (indexer != null) indexer.commit();
+ if (indexer != null) {
+ indexer.commit();
+ if (madeMods) indexerCache.indexModified(indexer);
+ }
}
/**
Modified: trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/FullTextStringIndexUnitTest.java
===================================================================
--- trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/FullTextStringIndexUnitTest.java 2008-10-27 12:38:48 UTC (rev 1366)
+++ trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/FullTextStringIndexUnitTest.java 2008-10-27 12:38:55 UTC (rev 1367)
@@ -206,7 +206,8 @@
* @throws Exception Test fails
*/
public void testFullTextStringPool() throws Exception {
- FullTextStringIndex index = new FullTextStringIndex(indexDirectory, "fulltextsp", true, true);
+ LuceneIndexerCache cache = new LuceneIndexerCache(indexDirectory);
+ FullTextStringIndex index = null;
try {
// Ensure that reverse search is enabled.
@@ -214,10 +215,12 @@
String has = "http://mulgara.org/mulgara/document#has";
//Clean any existing indexes.
- index.removeAllIndexes();
+ cache.removeAllIndexes();
+ cache.close();
+ cache = new LuceneIndexerCache(indexDirectory);
- //re-create the index
- index = new FullTextStringIndex(indexDirectory, "fulltextsp", true, true);
+ //create the index
+ index = new FullTextStringIndex(cache, true, true);
// Add strings to the index
for (String literal : theStrings) {
@@ -225,7 +228,8 @@
}
index.commit();
- index.openReadIndex();
+ index.close();
+ index = new FullTextStringIndex(cache, true, true);
// Find the strings from the index with both subject & predicate
for (String literal : theStrings) {
@@ -259,7 +263,8 @@
index.remove(document, has, "one two three");
index.commit();
- index.openReadIndex();
+ index.close();
+ index = new FullTextStringIndex(cache, true, true);
assertEquals("Presumed deleted but found 'one two'", 0,
index.find(document, has, "one two").length());
@@ -278,7 +283,8 @@
index.add("subject", "predicate", "this/is/a/slash/test"));
index.commit();
- index.openReadIndex();
+ index.close();
+ index = new FullTextStringIndex(cache, true, true);
long returned = index.find(document, has, "?ommittee").length();
assertEquals("Reverse lookup was expecting 4 documents returned", 4, returned);
@@ -301,7 +307,8 @@
// test removing all documents
index.removeAll();
index.commit();
- index.openReadIndex();
+ index.close();
+ index = new FullTextStringIndex(cache, true, true);
returned = index.find(document, has, "European").length();
assertEquals("Got unexpected documents after removeAll:", 0, returned);
@@ -311,7 +318,8 @@
} finally {
if (index != null) {
index.close();
- assertTrue("Unable to remove all index files", index.removeAllIndexes());
+ cache.close();
+ assertTrue("Unable to remove all index files", cache.removeAllIndexes());
}
}
}
@@ -324,15 +332,17 @@
*/
public void testFullTextStringPoolwithFiles() throws Exception {
// create a new index direcotry
- FullTextStringIndex index = new FullTextStringIndex(indexDirectory, "fulltextsp", true, true);
+ LuceneIndexerCache cache = new LuceneIndexerCache(indexDirectory);
+ FullTextStringIndex index = null;
try {
// make sure the index directory is empty
- index.close();
- assertTrue("Unable to remove all index files", index.removeAllIndexes());
+ assertTrue("Unable to remove all index files", cache.removeAllIndexes());
+ cache.close();
+ cache = new LuceneIndexerCache(indexDirectory);
// create a new index
- index = new FullTextStringIndex(indexDirectory, "fulltextsp", true, true);
+ index = new FullTextStringIndex(cache, true, true);
logger.debug("Obtaining text text documents from " + textDirectory);
@@ -371,7 +381,8 @@
// commit the new docs
index.commit();
- index.openReadIndex();
+ index.close();
+ index = new FullTextStringIndex(cache, true, true);
// Perform a search for 'supernatural' in the
// document content predicate
@@ -401,7 +412,8 @@
// commit the removal
index.commit();
- index.openReadIndex();
+ index.close();
+ index = new FullTextStringIndex(cache, true, true);
// Perform a search for 'supernatural' in the
// document content predicate
@@ -415,7 +427,8 @@
// close the fulltextstringpool
if (index != null) {
index.close();
- assertTrue("Unable to remove all index files", index.removeAllIndexes());
+ cache.close();
+ assertTrue("Unable to remove all index files", cache.removeAllIndexes());
}
}
}
Added: trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneIndexerCache.java
===================================================================
--- trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneIndexerCache.java (rev 0)
+++ trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneIndexerCache.java 2008-10-27 12:38:55 UTC (rev 1367)
@@ -0,0 +1,331 @@
+/*
+ * Copyright 2008 The Topaz Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * Contributions:
+ */
+
+package org.mulgara.resolver.lucene;
+
+import org.apache.log4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+
+/**
+ * A cache of lucene index-readers and index-writers. Opening a lucene index-reader or writer is
+ * fairly expensive, so caching them can provide substantial performance gains. No cache-expiry
+ * has been implemented, however; the assumption is that there will a limited number of lucene
+ * models.
+ *
+ * <p>This also manages the setting up and removal of the index directory.
+ *
+ * @created 2008-09-28
+ * @author Ronald Tschalär
+ * @licence Apache License v2.0
+ */
+public class LuceneIndexerCache {
+ private static final Logger logger = Logger.getLogger(LuceneIndexerCache.class);
+
+ /* our caches */
+ private final Stack<ReaderInfo> freeReaders = new Stack<ReaderInfo>();
+ private final Stack<WriterInfo> freeWriters = new Stack<WriterInfo>();
+ private final Map<IndexReader,ReaderInfo> allocdReaders = new HashMap<IndexReader,ReaderInfo>();
+ private final Map<IndexWriter,WriterInfo> allocdWriters = new HashMap<IndexWriter,WriterInfo>();
+
+ /** The Directory for Lucene. */
+ private final Directory luceneIndexDirectory;
+
+ /** Whether this cache has been closed */
+ private boolean closed = false;
+
+ /**
+ * Create a new cache.
+ *
+ * @param directory the directory to use for the indexes; it is created if it does not exist.
+ * @throws IOException if the directory is not readable, writable, or some other error occurs
+ * trying to access it
+ */
+ public LuceneIndexerCache(String directory) throws IOException {
+ luceneIndexDirectory = FSDirectory.getDirectory(createOrValidateDirectory(directory));
+ clearLocks();
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Fulltext string indexer cache initialized; directory =" + directory);
+ }
+ }
+
+ private static File createOrValidateDirectory(String directory) throws IOException {
+ File indexDirectory = new File(directory);
+
+ // make the directory if it doesn't exist
+ if (!indexDirectory.exists()) {
+ indexDirectory.mkdirs();
+ }
+
+ // ensure the index directory is a directory
+ if (!indexDirectory.isDirectory()) {
+ logger.fatal("The fulltext string index directory '" + directory + "' is not a directory!");
+ throw new IOException("The fulltext string index directory '" + directory +
+ "' is not a directory!");
+ }
+
+ // ensure the directory is writeable
+ if (!indexDirectory.canWrite()) {
+ logger.fatal("The fulltext string index directory '" + directory + "' is not writeable!");
+ throw new IOException("The fulltext string index directory '" + directory +
+ "' is not writeable!");
+ }
+
+ return indexDirectory;
+ }
+
+ private void clearLocks() throws IOException {
+ if (IndexWriter.isLocked(luceneIndexDirectory)) {
+ /* This shouldn't happen unless mulgara was shut down abruptly since mulgara has a single
+ * writer lock.
+ */
+ logger.warn("Fulltext index directory '" + luceneIndexDirectory + "' is locked; forcibly unlocking");
+ IndexWriter.unlock(luceneIndexDirectory);
+ }
+
+ if (IndexWriter.isLocked(luceneIndexDirectory)) {
+ throw new IOException("Fulltext index directory '" + luceneIndexDirectory + "' is locked; " +
+ "forced unlock failed; giving up");
+ }
+ }
+
+ /**
+ * Get an index-reader. It must be returned via {@link #returnReader}.
+ *
+ * @return the index-reader
+ */
+ public synchronized IndexReader getReader() throws IOException {
+ if (closed) throw new IllegalStateException("IndexerCache has been closed: " + luceneIndexDirectory);
+
+ ReaderInfo ri = freeReaders.pop();
+ if (ri == null) {
+ ri = new ReaderInfo(luceneIndexDirectory);
+
+ if (logger.isDebugEnabled()) logger.debug("Creating new index-reader: " + ri.reader);
+ } else if (ri.needsRefresh) {
+ IndexReader reader = ri.reader.reopen();
+ if (reader != ri.reader) {
+ ri.reader.close();
+ ri = new ReaderInfo(reader);
+
+ if (logger.isDebugEnabled()) logger.debug("Refreshed index-reader: " + ri.reader);
+ } else {
+ ri.needsRefresh = false;
+ }
+ } else {
+ if (logger.isDebugEnabled()) logger.debug("Reusing index-reader: " + ri.reader);
+ }
+
+ allocdReaders.put(ri.reader, ri);
+ return ri.reader;
+ }
+
+ /**
+ * Get an index-writer. It must be returned via {@link #returnWriter}.
+ *
+ * @return the index-writer
+ */
+ public synchronized IndexWriter getWriter() throws IOException {
+ if (closed) throw new IllegalStateException("IndexerCache has been closed: " + luceneIndexDirectory);
+
+ WriterInfo wi = freeWriters.pop();
+ if (wi == null) {
+ wi = new WriterInfo(luceneIndexDirectory);
+
+ if (logger.isDebugEnabled()) logger.debug("Created new index-writer: " + wi.writer);
+ } else if (wi.needsRefresh) {
+ wi.writer.close();
+ wi = new WriterInfo(luceneIndexDirectory);
+
+ if (logger.isDebugEnabled()) logger.debug("Refreshed index-writer: " + wi.writer);
+ } else {
+ if (logger.isDebugEnabled()) logger.debug("Reusing index-writer: " + wi.writer);
+ }
+
+ allocdWriters.put(wi.writer, wi);
+ return wi.writer;
+ }
+
+ /**
+ * Return an index-reader to the cache. It must have been previously retrieved through {@link
+ * #getReader}.
+ *
+ * @param reader the reader to return
+ * @param close if true the reader is closed and not returned to the pool
+ */
+ public synchronized void returnReader(IndexReader reader, boolean close) {
+ ReaderInfo ri = allocdReaders.remove(reader);
+
+ if (close || closed) {
+ try {
+ reader.close();
+ if (logger.isDebugEnabled()) logger.debug("Closed index-reader: " + reader);
+ } catch (IOException ioe) {
+ logger.warn("Error closing index-reader: " + reader);
+ }
+ } else {
+ freeReaders.push(ri);
+ if (logger.isDebugEnabled()) logger.debug("Returned index-reader: " + reader);
+ }
+ }
+
+ /**
+ * Return an index-writer to the cache. It must have been previously retrieved through {@link
+ * #getWriter}.
+ *
+ * @param writer the writer to return
+ * @param close if true the writer is closed and not returned to the pool
+ */
+ public synchronized void returnWriter(IndexWriter writer, boolean close) {
+ WriterInfo wi = allocdWriters.remove(writer);
+
+ if (close || closed) {
+ try {
+ writer.close();
+ if (logger.isDebugEnabled()) logger.debug("Closed index-writer: " + writer);
+ } catch (IOException ioe) {
+ logger.warn("Error closing index-writer: " + writer);
+ }
+ } else {
+ freeWriters.push(wi);
+ if (logger.isDebugEnabled()) logger.debug("Returned index-writer: " + writer);
+ }
+ }
+
+ /**
+ * Notify the cache that the index has been modified. All newly returned indexers will be
+ * appropriately refreshed.
+ *
+ * @param writer the writer that made the modification
+ */
+ public synchronized void indexModified(IndexWriter writer) {
+ for (RefreshableObject ro : freeReaders) ro.needsRefresh = true;
+ for (RefreshableObject ro : freeWriters) ro.needsRefresh = true;
+ for (RefreshableObject ro : allocdReaders.values()) ro.needsRefresh = true;
+ for (WriterInfo wi : allocdWriters.values()) {
+ if (wi.writer != writer) wi.needsRefresh = true;
+ }
+
+ if (logger.isDebugEnabled()) logger.debug("All indexers marked for refresh");
+ }
+
+ /**
+ * @return the directory being used for the indexes
+ */
+ public String getDirectory() {
+ return luceneIndexDirectory.toString();
+ }
+
+ /**
+ * Remove all index files from the current initialised directory. WARNING : All
+ * files are removed in the specified directory. This is probably only useful for
+ * testing. See {@link FullTextStringIndex#removeAll} for an alternate solution.
+ *
+ * @return return true if successful at removing all index files
+ * @throws IOException if an exception occurs while attempting to delete the files
+ */
+ public synchronized boolean removeAllIndexes() throws IOException {
+ if (allocdWriters.size() > 0 || allocdReaders.size() > 0) {
+ logger.warn("Attempting to remove all indexes while readers or writers are still active");
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("Removing all indexes from " + luceneIndexDirectory);
+ }
+
+ for (String file : luceneIndexDirectory.list()) {
+ luceneIndexDirectory.deleteFile(file);
+ }
+
+ return ((FSDirectory)luceneIndexDirectory).getFile().delete();
+ }
+
+ /**
+ * Close this cache. All pooled index readers/writers are closed; readers/writers that are still
+ * in use will be closed upon being returned.
+ */
+ public synchronized void close() {
+ if (allocdWriters.size() > 0 || allocdReaders.size() > 0) {
+ logger.warn("Attempting to close indexer-cache while readers or writers are still active");
+ }
+
+ closed = true;
+
+ for (ReaderInfo ri : freeReaders) {
+ try {
+ ri.reader.close();
+ } catch (IOException ioe) {
+ logger.error("Error closing index-reader: " + ri.reader, ioe);
+ }
+ }
+
+ for (WriterInfo wi : freeWriters) {
+ try {
+ wi.writer.close();
+ } catch (IOException ioe) {
+ logger.error("Error closing index-writer: " + wi.writer, ioe);
+ }
+ }
+
+ if (logger.isDebugEnabled()) logger.debug("IndexerCacher closed: " + luceneIndexDirectory);
+ }
+
+ private static abstract class RefreshableObject {
+ public boolean needsRefresh = false;
+ }
+
+ private static class ReaderInfo extends RefreshableObject {
+ public final IndexReader reader;
+
+ public ReaderInfo(Directory directory) throws IOException {
+ reader = IndexReader.open(directory, true);
+ }
+
+ public ReaderInfo(IndexReader reader) {
+ this.reader = reader;
+ }
+ }
+
+ private static class WriterInfo extends RefreshableObject {
+ public final IndexWriter writer;
+
+ public WriterInfo(Directory directory) throws IOException {
+ writer = new IndexWriter(directory, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
+ }
+ }
+
+ private static class Stack<T> extends ArrayList<T> {
+ public void push(T obj) {
+ add(obj);
+ }
+
+ public T pop() {
+ return size() > 0 ? remove(size() - 1) : null;
+ }
+ }
+}
Property changes on: trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneIndexerCache.java
___________________________________________________________________
Name: svn:keywords
+ Id HeadURL Revision
Name: svn:eol-style
+ native
Modified: trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneResolver.java
===================================================================
--- trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneResolver.java 2008-10-27 12:38:48 UTC (rev 1366)
+++ trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneResolver.java 2008-10-27 12:38:55 UTC (rev 1367)
@@ -30,7 +30,6 @@
package org.mulgara.resolver.lucene;
// Java 2 standard packages
-import java.io.File;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.Reader;
@@ -107,10 +106,10 @@
*/
protected final URI modelTypeURI;
- protected final String directory;
-
protected final ResolverSession resolverSession;
+ protected final LuceneResolverFactory resolverFactory;
+
protected final boolean forWrites;
protected final XAResource xares;
@@ -126,21 +125,15 @@
*
* @param modelTypeURI the URI of the lucene model type
* @param resolverSession the session this resolver is associated with
- * @param directory the directory to use for the lucene indexes
* @param resolverFactory the resolver-factory that created us
* @param forWrites whether we may be getting writes
- * @throws IllegalArgumentException if <var>directory</var> is <code>null</code>
*/
- LuceneResolver(URI modelTypeURI, ResolverSession resolverSession, String directory,
- ResolverFactory resolverFactory, boolean forWrites) {
- if (directory == null) {
- throw new IllegalArgumentException("Null directory in LuceneResolver");
- }
-
+ LuceneResolver(URI modelTypeURI, ResolverSession resolverSession,
+ LuceneResolverFactory resolverFactory, boolean forWrites) {
// Initialize fields
this.modelTypeURI = modelTypeURI;
- this.directory = directory;
this.resolverSession = resolverSession;
+ this.resolverFactory = resolverFactory;
this.forWrites = forWrites;
this.xares = new LuceneXAResource(10, resolverFactory, indexes.values());
}
@@ -325,6 +318,8 @@
throw new ResolverException("Error fetching statements", et);
} catch (GlobalizeException eg) {
throw new ResolverException("Error localizing statements", eg);
+ } catch (IOException ioe) {
+ throw new ResolverException("Failed to open string index", ioe);
} catch (FullTextStringIndexException ef) {
throw new ResolverException("Error in string index", ef);
}
@@ -340,6 +335,8 @@
try {
getFullTextStringIndex(model).removeAll();
+ } catch (IOException ioe) {
+ throw new ResolverException("Failed to open string index", ioe);
} catch (FullTextStringIndexException ef) {
throw new ResolverException("Query failed against string index", ef);
}
@@ -381,15 +378,18 @@
return new TuplesWrapperResolution(tuples, constraint);
} catch (TuplesException te) {
throw new QueryException("Failed to sort tuples and close", te);
+ } catch (IOException ioe) {
+ throw new QueryException("Failed to open string index", ioe);
} catch (FullTextStringIndexException ef) {
throw new QueryException("Query failed against string index", ef);
}
}
- private FullTextStringIndex getFullTextStringIndex(long model) throws FullTextStringIndexException {
+ private FullTextStringIndex getFullTextStringIndex(long model)
+ throws FullTextStringIndexException, IOException {
FullTextStringIndex index = indexes.get(model);
if (index == null) {
- index = new FullTextStringIndex(new File(directory, Long.toString(model)).toString(), "gn" + model, forWrites);
+ index = new FullTextStringIndex(resolverFactory.getIndexerCache(Long.toString(model)), forWrites);
indexes.put(model, index);
}
return index;
Modified: trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneResolverFactory.java
===================================================================
--- trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneResolverFactory.java 2008-10-27 12:38:48 UTC (rev 1366)
+++ trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/LuceneResolverFactory.java 2008-10-27 12:38:55 UTC (rev 1367)
@@ -28,7 +28,11 @@
package org.mulgara.resolver.lucene;
// Java 2 standard packages
+import java.io.File;
+import java.io.IOException;
import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
// Third party packages
import org.apache.log4j.Logger;
@@ -71,6 +75,7 @@
public static final URI searchURI = URI.create(Mulgara.NAMESPACE + "search");
public static final URI scoreURI = URI.create(Mulgara.NAMESPACE + "score");
+ private final Map<String,LuceneIndexerCache> indexerCaches = new HashMap<String,LuceneIndexerCache>();
private String directory;
//
@@ -109,24 +114,23 @@
// Methods implementing ResolverFactory
//
- /**
- * {@inheritDoc ResolverFactory}
- *
- * This is actually a non-operation, because the only persistent resources
- * are outside the database.
- */
public void close() {
- // null implementation
+ for (LuceneIndexerCache cache : indexerCaches.values()) {
+ cache.close();
+ }
+ indexerCaches.clear();
}
- /**
- * {@inheritDoc ResolverFactory}
- *
- * This is actually a non-operation, because the only persistent resources
- * are outside the database.
- */
public void delete() {
- // null implementation
+ for (LuceneIndexerCache cache : indexerCaches.values()) {
+ cache.close();
+ try {
+ cache.removeAllIndexes();
+ } catch (IOException ioe) {
+ logger.warn("Error deleting lucene index " + cache.getDirectory());
+ }
+ }
+ indexerCaches.clear();
}
/**
@@ -170,7 +174,25 @@
throws ResolverFactoryException {
if (logger.isDebugEnabled()) logger.debug("Creating Lucene resolver");
return canWrite
- ? new LuceneResolver(modelTypeURI, resolverSession, directory, this, true)
- : new ReadOnlyLuceneResolver(modelTypeURI, resolverSession, directory, this);
+ ? new LuceneResolver(modelTypeURI, resolverSession, this, true)
+ : new ReadOnlyLuceneResolver(modelTypeURI, resolverSession, this);
}
+
+ /**
+ * Get an indexer-cache for the given model.
+ *
+ * @param model the model to the indexer-cache for
+ * @return the indexer-cache
+ * @throws IOException if an error occurred creating an indexer-cache
+ */
+ LuceneIndexerCache getIndexerCache(String model) throws IOException {
+ synchronized (indexerCaches) {
+ LuceneIndexerCache cache = indexerCaches.get(model);
+ if (cache == null) {
+ indexerCaches.put(model, cache = new LuceneIndexerCache(new File(directory, model).toString()));
+ }
+
+ return cache;
+ }
+ }
}
Modified: trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/ReadOnlyLuceneResolver.java
===================================================================
--- trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/ReadOnlyLuceneResolver.java 2008-10-27 12:38:48 UTC (rev 1366)
+++ trunk/src/jar/resolver-lucene/java/org/mulgara/resolver/lucene/ReadOnlyLuceneResolver.java 2008-10-27 12:38:55 UTC (rev 1367)
@@ -36,7 +36,6 @@
// Locally written packages
import org.mulgara.resolver.spi.LocalizeException;
import org.mulgara.resolver.spi.ResolverException;
-import org.mulgara.resolver.spi.ResolverFactory;
import org.mulgara.resolver.spi.ResolverFactoryException;
import org.mulgara.resolver.spi.ResolverSession;
import org.mulgara.resolver.spi.Statements;
@@ -75,14 +74,13 @@
*
* @param modelTypeURI the URI of the lucene model type
* @param resolverSession the session this resolver is associated with
- * @param directory the directory to use for the lucene indexes
* @param resolverFactory the resolver-factory that created us
* @throws IllegalArgumentException if <var>directory</var> is <code>null</code>
*/
- ReadOnlyLuceneResolver(URI modelTypeURI, ResolverSession resolverSession, String directory,
- ResolverFactory resolverFactory)
+ ReadOnlyLuceneResolver(URI modelTypeURI, ResolverSession resolverSession,
+ LuceneResolverFactory resolverFactory)
throws ResolverFactoryException {
- super(modelTypeURI, resolverSession, directory, resolverFactory, false);
+ super(modelTypeURI, resolverSession, resolverFactory, false);
}
//
More information about the Mulgara-svn
mailing list