[Mulgara-dev] Lucene deletes

David Smith DMS at viewpointusa.com
Tue Jun 17 19:59:44 UTC 2008


 
(I am working on mulgara 1.2.1  rev 739)

I was able to track down the problem ( I think ) with the Lucene
deletes.
The problem is that the FullTextStringIndex instance typically holds
both a IndexWriter (indexer variable) and a IndexReader (indexDelete
variable) and this causes problems when the remove attempts to delete
the documents from the index. I was seeing the "Lock obtain timed out"
error on delete.

My fix (see below) was to close the indexer variable during the remove
(or more precisely... In the resolver's modeifyModel method when
deleting ). This seems to allow the deleteDocuments() to complete
sucessfully. I also needed to protect the optimize method from using the
indexer in the case that the indexer was closed().

I would appreciate any feedback on these changes.

I did, briefly, attempt lucene 2.2.0 -- but there were numerous
errors/deprecated classses and I didn't go any further...
A big advantage to 2.2.0 would be that the deleteDocuments() method is
now available on the IndexWriter class and we wouldn't need the double
variables (IndexWriter and IndexReader) anymore.

==

I also have been experimenting with having "drop" delete the lucene
files. That accounts for the changes in removeModel()

==
Diffs:

Index: FullTextStringIndex.java
===================================================================
--- FullTextStringIndex.java	(revision 739)
+++ FullTextStringIndex.java	(working copy)
@@ -599,16 +599,34 @@
 
             for (int i = 0; i < files.length; i++) {
 
+							// debug logging
+							if
(logger.isDebugEnabled()) {
+
logger.debug("Removing file: " + files[i] );
+							}
+
               luceneIndexDirectory.deleteFile(files[i]);
             }
 
             //Remove the directory
+						// debug logging
+						if
(logger.isDebugEnabled()) {
+
logger.debug("Removing directory: " + indexDirectoryName );
+						}
             deleted = new File(indexDirectoryName).delete();
 
+						if
(logger.isDebugEnabled()) {
+
logger.debug("Cleaning up locks");
+						}
+
             //set the status to not modified
             indexLock.setStatus(indexLock.NOT_MODIFIED);
 
             lock.release();
+
+						if
(logger.isDebugEnabled()) {
+
logger.debug("Remove all complete");
+						}
+
           }
         }
         catch (IOException ioe) {
@@ -732,6 +750,46 @@
     }
   }
 
+
+  public void prepForDeletes() throws FullTextStringIndexException
+  {
+    if (logger.isDebugEnabled()) {
+
+      logger.debug("Prep for deletes");
+    }
+
+		close();
+
+    if (logger.isDebugEnabled()) {
+
+      logger.debug("Prep for deletes, closed()");
+    }
+
+		openReadIndex();
+
+    if (logger.isDebugEnabled()) {
+
+      logger.debug("Prep for deletes, complete");
+    }
+	}
+
+  public void prepForReads() throws FullTextStringIndexException
+  {
+    if (logger.isDebugEnabled()) {
+
+      logger.debug("Prep for reads");
+    }
+
+		close();
+		openWriteIndex();
+
+    if (logger.isDebugEnabled()) {
+
+      logger.debug("Prep for reads, complete");
+    }
+
+	}
+
   /**
    * Optimize the index and then flush it to disk.
    *
@@ -740,34 +798,45 @@
    */
   public void optimize() throws FullTextStringIndexException {
 
-    // debug logging
-    if (logger.isDebugEnabled()) {
 
-      logger.debug("Optimizing fulltext string pool indexes to disk");
-    }
+		if( null == indexer ) {
+			// debug logging
+			if (logger.isDebugEnabled()) {
 
-    try {
+				logger.debug("Can't optimize, indexer is
null.");
+			}
+		}
+		else {
 
-      //lock any deletes, adds and optimize from the index
-      synchronized (indexLock) {
+			// debug logging
+			if (logger.isDebugEnabled()) {
 
-        logger.info("Optimizing fulltext index at " +
this.indexDirectoryName +
-            " please wait...");
+				logger.debug("Optimizing fulltext string
pool indexes to disk");
+			}
 
-        //Optimize the indexes
-        indexer.optimize();
+			try {
 
-        //set the index status to modified
-        indexLock.setStatus(indexLock.MODIFIED);
-      }
-    }
-    catch (IOException ex) {
+				//lock any deletes, adds and optimize
from the index
+				synchronized (indexLock) {
 
-      logger.error("Unable to optimize existing fulltext string pool
index", ex);
-      throw new FullTextStringIndexException(
-          "Unable to optimize existing fulltext " + "string pool
index",
-          ex);
-    }
+					logger.info("Optimizing fulltext
index at " + this.indexDirectoryName +
+							" please
wait...");
+
+					//Optimize the indexes
+					indexer.optimize();
+
+					//set the index status to
modified
+
indexLock.setStatus(indexLock.MODIFIED);
+				}
+			}
+			catch (IOException ex) {
+
+				logger.error("Unable to optimize
existing fulltext string pool index", ex);
+				throw new FullTextStringIndexException(
+						"Unable to optimize
existing fulltext " + "string pool index",
+						ex);
+			}
+		}
   }
 
   /**
@@ -1047,6 +1116,9 @@
     }
   }
 
+
+
+
   /**
    * Open the index on disk for writing.
    *
Index: LuceneResolver.java
===================================================================
--- LuceneResolver.java	(revision 739)
+++ LuceneResolver.java	(working copy)
@@ -149,8 +149,17 @@
     try {
       FullTextStringIndex stringIndex = openFullTextStringIndex(model);
 
+      if( !occurs ) {
+				stringIndex.prepForDeletes();
+			}
+
       statements.beforeFirst();
       while (statements.next()) {
+
+				if (logger.isDebugEnabled()) {
+
logger.debug("statments.next()");
+				}
+
         Node subjectNode =
resolverSession.globalize(statements.getSubject());
 
         // Do not insert the triple if it contains a blank node in
subject.
@@ -278,6 +287,11 @@
                 logger.warn("Unable to remove {" + subject + ", " +
predicate +
                     ", " + literal + "} from full text string index");
               }
+
+              if (logger.isDebugEnabled()) {
+                logger.debug("Deleted " + subject + " " + predicate + "
" + literal);
+              }
+
             }
           } catch (FullTextStringIndexException e) {
             throw new ResolverException("Unable to add '" + literal +
@@ -290,9 +304,15 @@
         }
       }
 
+			if( !occurs )
+			{
+				stringIndex.prepForReads();
+			}
+
       // Flush the index
       stringIndex.optimize();
       stringIndex.close();
+
     } catch (TuplesException et) {
       throw new ResolverException("Error fetching statements", et);
     } catch (GlobalizeException eg) {
@@ -307,8 +327,15 @@
    */
   public void removeModel(long model) throws ResolverException {
     if (logger.isDebugEnabled()) {
-      logger.debug("Remove memory model " + model);
+      logger.debug("Remove lucene model " + model);
     }
+
+		try {
+			FullTextStringIndex stringIndex =
openFullTextStringIndex(model);
+			stringIndex.removeAll();
+		} catch ( FullTextStringIndexException ef ) {
+			throw new ResolverException("Error dropping
model", ef );
+		}
   }
 
   /**



More information about the Mulgara-dev mailing list