[Mulgara-svn] r2004 - in trunk/src/jar: store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11 util/java/org/mulgara/util util/java/org/mulgara/util/io util-xa/java/org/mulgara/store/xa
alexhall at mulgara.org
alexhall at mulgara.org
Tue Jul 19 00:01:35 UTC 2011
Author: alexhall
Date: 2011-07-19 00:01:35 +0000 (Tue, 19 Jul 2011)
New Revision: 2004
Added:
trunk/src/jar/util/java/org/mulgara/util/io/MappingUtil.java
Modified:
trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java
trunk/src/jar/util-xa/java/org/mulgara/store/xa/AbstractBlockFile.java
trunk/src/jar/util-xa/java/org/mulgara/store/xa/MappedBlockFile.java
trunk/src/jar/util/java/org/mulgara/util/IntFile.java
trunk/src/jar/util/java/org/mulgara/util/MappedIntFile.java
trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java
Log:
Fix rollback errors that were happening on Windows due to file mapping problems on that operating system.
Modified: trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java
===================================================================
--- trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java 2011-07-18 21:50:12 UTC (rev 2003)
+++ trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java 2011-07-19 00:01:35 UTC (rev 2004)
@@ -64,6 +64,7 @@
import org.mulgara.util.LongMapper;
import org.mulgara.util.functional.Pair;
import org.mulgara.util.io.LBufferedFile;
+import org.mulgara.util.io.MappingUtil;
import static org.mulgara.store.stringpool.xa11.DataStruct.*;
@@ -542,7 +543,7 @@
gNodeToDataAppender.position(offset);
// tell the read-only access to prepare for truncation
gNodeToDataFile.truncate(offset);
- forceTruncate(gNodeToDataAppender, offset);
+ MappingUtil.truncate(gNodeToDataAppender, offset);
} catch (IOException ioe) {
throw new SimpleXAResourceException("I/O error while performing rollback (new committed phase)", ioe);
}
@@ -595,7 +596,8 @@
nextGNodeValue = NodePool.MIN_NODE;
committedNextGNode = NodePool.MIN_NODE;
// this forces a seek to 0
- gNodeToDataAppender.truncate(0);
+ gNodeToDataFile.truncate(0);
+ MappingUtil.truncate(gNodeToDataAppender, 0);
currentPhase = new TreePhase();
}
@@ -801,31 +803,6 @@
/**
- * Force a file truncation. If memory maps prevent this, attempt to clean them up, and retry the truncation.
- * @param fc A channel to the file to be truncated.
- * @param size The new size of the file.
- * @throws IOException If the file is open beyond the truncation point, and this cannot be overridden.
- */
- private void forceTruncate(FileChannel fc, long size) throws IOException {
- long s = fc.size();
- if (s <= size) return;
- int retries = 10;
- for (;;) {
- try {
- fc.truncate(size);
- break;
- } catch (IOException e) {
- // on failure, attempt to clean up all resources keeping the file open
- if (retries-- == 0) throw e;
- System.gc();
- try { Thread.sleep(100); } catch (InterruptedException ie) { }
- System.runFinalization();
- }
- }
- }
-
-
- /**
* Closes all the files involved with a data pool
* @param deleteFiles Remove files after closing them.
* @throws IOException There was an error accessing the filesystem.
@@ -921,7 +898,8 @@
*/
private void updateAppender(long fileSize) throws IOException {
// truncate if the file is longer than the appending position
- if (gNodeToDataAppender.size() > fileSize) gNodeToDataAppender.truncate(fileSize);
+ gNodeToDataFile.truncate(fileSize);
+ MappingUtil.truncate(gNodeToDataAppender, fileSize);
gNodeToDataAppender.position(fileSize);
}
Modified: trunk/src/jar/util/java/org/mulgara/util/IntFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/IntFile.java 2011-07-18 21:50:12 UTC (rev 2003)
+++ trunk/src/jar/util/java/org/mulgara/util/IntFile.java 2011-07-19 00:01:35 UTC (rev 2004)
@@ -28,12 +28,14 @@
package org.mulgara.util;
// Java 2 standard packages
-import java.io.*;
-import java.nio.*;
-import java.nio.channels.*;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
-// Third party packages
import org.apache.log4j.Logger;
+import org.mulgara.util.io.MappingUtil;
/**
* A file consisting of longs, ints or bytes that can be accessed concurrently
@@ -421,27 +423,15 @@
throw new IllegalStateException("FileChannel is null");
}
- int retries = 10;
- for (;;) {
- try {
- fc.truncate(fileSize);
- break;
- } catch (IOException ex) {
- if (retries-- == 0) {
- logger.warn(
- "NOTE: Could not truncate file: \"" + file +
- "\" to size: " + fileSize +
- " - deferring until next time the file is opened."
- );
- logger.info("Could not truncate file", ex);
- break;
- }
-
- // This is needed for Windows.
- System.gc();
- try { Thread.sleep(100); } catch (InterruptedException ie) { }
- System.runFinalization();
- }
+ try {
+ MappingUtil.truncate(fc, fileSize);
+ } catch (IOException ex) {
+ logger.warn(
+ "NOTE: Could not truncate file: \"" + file +
+ "\" to size: " + fileSize +
+ " - deferring until next time the file is opened."
+ );
+ logger.info("Could not truncate file", ex);
}
}
Modified: trunk/src/jar/util/java/org/mulgara/util/MappedIntFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/MappedIntFile.java 2011-07-18 21:50:12 UTC (rev 2003)
+++ trunk/src/jar/util/java/org/mulgara/util/MappedIntFile.java 2011-07-19 00:01:35 UTC (rev 2004)
@@ -30,18 +30,14 @@
// Java 2 standard packages
import java.io.File;
import java.io.IOException;
-import java.lang.reflect.Method;
import java.nio.IntBuffer;
import java.nio.LongBuffer;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
import org.apache.log4j.Logger;
+import org.mulgara.util.io.MappingUtil;
-import sun.misc.Cleaner;
-
/**
* @created 2003-01-09
*
@@ -316,11 +312,7 @@
*/
public synchronized void unmap() {
// Discard the file mappings.
- if (mappedByteBuffers != null) {
- for (MappedByteBuffer buffer : mappedByteBuffers) {
- clean(buffer);
- }
- }
+ MappingUtil.release(mappedByteBuffers);
mappedByteBuffers = null;
intBuffers = null;
longBuffers = null;
@@ -402,22 +394,4 @@
nrMappedRegions = nrRegions;
}
- private static void clean(final Object buffer) {
- if (buffer != null) {
- AccessController.doPrivileged(new PrivilegedAction<Object>() {
- public Object run() {
- try {
- Method getCleanerMethod = buffer.getClass().getMethod("cleaner", new Class[0]);
- getCleanerMethod.setAccessible(true);
- Cleaner cleaner = (Cleaner)getCleanerMethod.invoke(buffer, new Object[0]);
- cleaner.clean();
- } catch (Exception e) {
- logger.warn("Error cleaning buffer", e);
- }
- return null;
- }
- });
- }
- }
-
}
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java 2011-07-18 21:50:12 UTC (rev 2003)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java 2011-07-19 00:01:35 UTC (rev 2004)
@@ -120,9 +120,19 @@
*/
public void truncate(long size) throws IOException {
if (logger.isDebugEnabled()) logger.debug("Truncating file to " + size + ". Current size is " + fc.size());
-
+
if (size == fc.size()) return;
if (size > fc.size()) throw new IOException("Unable to truncate a mapped file larger");
+
+ // Windows doesn't want to truncate a file with any portion mapped, so unmap the whole thing.
+ if (MappingUtil.isWindows()) {
+ if (logger.isDebugEnabled()) logger.debug("Unmapping entire file for Windows.");
+ MappingUtil.release(buffers);
+ buffers = new MappedByteBuffer[0];
+ for (Runnable listener: listeners) listener.run();
+ return;
+ }
+
// if truncating to an address above the mapping, then return
int fullBuffers = buffers.length - 1;
if (fullBuffers >= 0 && size >= fullBuffers * PAGE_SIZE + buffers[fullBuffers].limit()) return;
Added: trunk/src/jar/util/java/org/mulgara/util/io/MappingUtil.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/MappingUtil.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/MappingUtil.java 2011-07-19 00:01:35 UTC (rev 2004)
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2011 Revelytix, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.mulgara.util.io;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+import org.apache.log4j.Logger;
+
+import sun.misc.Cleaner;
+
+/**
+ * Common utilities for working with memory-mapped files. The main purpose of these
+ * utilities is to provide a central place for workarounds for known issues with
+ * memory mapping under Windows.
+ *
+ * @author Alex Hall
+ */
+public class MappingUtil {
+
+ private static final Logger logger = Logger.getLogger(MappingUtil.class);
+
+ /** System property to determine operating system. */
+ private static final String OS_PROP = "os.name";
+
+ /** Only look up the operating system once and then cache it. */
+ private static final boolean IS_WINDOWS;
+ static {
+ String os = System.getProperty(OS_PROP);
+ IS_WINDOWS = (os != null && os.toLowerCase().indexOf("win") >= 0);
+ }
+
+ /** The maximum number of attempts for truncating a mapped file. */
+ private static final int MAX_RETRIES = 10;
+
+ /** Determines whether we are running under Windows. */
+ public static boolean isWindows() {
+ return IS_WINDOWS;
+ }
+
+ /**
+ * Truncate the given file to the given size. This method will re-attempt
+ * the truncation several times, to account for the fact that the truncation fails
+ * in Windows if there exist mapped buffers which have not yet been cleaned up
+ * by the garbage collector.
+ *
+ * All calls in Mulgara to FileChannel.truncate() should pass through this method.
+ *
+ * This method does nothing if the given size is greater than or equal to the current
+ * size of the file.
+ *
+ * @param fc The file channel to truncate.
+ * @param size The new size after truncation.
+ * @throws IOException if the truncation still failed after attempting to clean up system resources.
+ */
+ public static void truncate(FileChannel fc, long size) throws IOException {
+ long s = fc.size();
+ if (s <= size) return;
+ int retries = MAX_RETRIES;
+ for (;;) {
+ try {
+ fc.truncate(size);
+ break;
+ } catch (IOException e) {
+ // on failure, attempt to clean up all resources keeping the file open
+ if (retries-- == 0) {
+ logger.error("Unable to truncate mapped file of size " + s + " to size " + size, e);
+ throw e;
+ }
+ System.gc();
+ try { Thread.sleep(100); } catch (InterruptedException ie) { }
+ System.runFinalization();
+ }
+ }
+ }
+
+ /**
+ * Releases the mapped byte buffer, attempting to pre-emptively clean up the associated
+ * mapping under Windows.
+ * @param buffer A mapped byte buffer.
+ */
+ public static void release(MappedByteBuffer buffer) {
+ if (isWindows()) clean(buffer);
+ }
+
+ /**
+ * Releases an array of mapped byte buffers, setting the references to null to allow
+ * them to be garbage collected.
+ * @param buffers An array of buffers.
+ */
+ public static void release(MappedByteBuffer[] buffers) {
+ if (buffers == null) return;
+ for (int i = 0; i < buffers.length; i++) {
+ release(buffers[i]);
+ buffers[i] = null;
+ }
+ }
+
+ /**
+ * Attempts to pre-emptively invoke the cleaner method on the given object instead
+ * of waiting for the garbage collector to do it. We do this under Windows because
+ * errors can result if a mapped byte buffer is cleaned after its associated file
+ * is closed, truncated, deleted, etc.
+ * @param buffer The buffer to release.
+ */
+ private static void clean(final Object buffer) {
+ if (buffer != null) {
+ AccessController.doPrivileged(new PrivilegedAction<Object>() {
+ public Object run() {
+ try {
+ Method getCleanerMethod = buffer.getClass().getMethod("cleaner", new Class[0]);
+ getCleanerMethod.setAccessible(true);
+ Cleaner cleaner = (Cleaner)getCleanerMethod.invoke(buffer, new Object[0]);
+ cleaner.clean();
+ } catch (Exception e) {
+ logger.warn("Error cleaning buffer", e);
+ }
+ return null;
+ }
+ });
+ }
+ }
+
+}
Modified: trunk/src/jar/util-xa/java/org/mulgara/store/xa/AbstractBlockFile.java
===================================================================
--- trunk/src/jar/util-xa/java/org/mulgara/store/xa/AbstractBlockFile.java 2011-07-18 21:50:12 UTC (rev 2003)
+++ trunk/src/jar/util-xa/java/org/mulgara/store/xa/AbstractBlockFile.java 2011-07-19 00:01:35 UTC (rev 2004)
@@ -36,6 +36,7 @@
// Third party packages
import org.apache.log4j.Logger;
+import org.mulgara.util.io.MappingUtil;
/**
* An abstract class that represents a file which consists of a number of
@@ -142,33 +143,12 @@
long fileSize = nrBlocks * blockSize;
if (fileSize < fc.size()) {
- int retries = 10;
- for (;;) {
- try {
- fc.truncate(fileSize);
- break;
- } catch (IOException ex) {
- if (retries-- == 0) {
- logger.error(
- "Could not truncate file: \"" + file +
- "\" to size: " + fileSize, ex
- );
-
- throw ex;
- }
-
- // This is needed for Windows.
- System.gc();
- try { Thread.sleep(100); } catch (InterruptedException ie) { }
- System.runFinalization();
- }
- }
if (logger.isInfoEnabled()) {
- logger.info(
- "File size was not a multiple of blockSize: \"" + file +
- "\"."
- );
+ logger.info(
+ "File size was not a multiple of blockSize: \"" + file + "\"."
+ );
}
+ MappingUtil.truncate(fc, fileSize);
}
openFiles.add(file);
@@ -268,37 +248,14 @@
public void clear() throws IOException {
this.nrBlocks = 0;
- int retries = 10;
- for (;;) {
- try {
- fc.truncate(0L);
- fc.force(true);
-
- break;
- } catch (ClosedChannelException ex) {
- // The Channel may have been inadvertently closed by another thread
- // being interrupted. Attempt to reopen the channel.
- if (!ensureOpen()) {
- throw ex;
- }
-
- // Loop back and retry the truncate()/force().
- } catch (IOException ex) {
- if (retries-- == 0) {
- if (logger.isInfoEnabled()) {
- logger.info(
- "Could not truncate file: \"" + file +
- "\" to zero length"
- );
- }
-
- throw ex;
- }
-
- // This is needed for Windows.
- System.gc();
- try { Thread.sleep(100); } catch (InterruptedException ie) { }
- System.runFinalization();
+ try {
+ MappingUtil.truncate(fc, 0L);
+ fc.force(true);
+ } catch (ClosedChannelException ex) {
+ // The Channel may have been inadvertently closed by another thread
+ // being interrupted. Attempt to reopen the channel.
+ if (!ensureOpen()) {
+ throw ex;
}
}
}
@@ -310,19 +267,13 @@
* @throws IOException if an I/O error occurs.
*/
public void force() throws IOException {
- for (;;) {
- try {
- fc.force(true);
-
- break;
- } catch (ClosedChannelException ex) {
- // The Channel may have been inadvertently closed by another thread
- // being interrupted. Attempt to reopen the channel.
- if (!ensureOpen()) {
- throw ex;
- }
-
- // Loop back and retry the force().
+ try {
+ fc.force(true);
+ } catch (ClosedChannelException ex) {
+ // The Channel may have been inadvertently closed by another thread
+ // being interrupted. Attempt to reopen the channel.
+ if (!ensureOpen()) {
+ throw ex;
}
}
}
@@ -376,40 +327,23 @@
try {
unmap();
- // NOTE: truncate() can still fail under windows if there are any other
- // references to the MappedByteBuffer (or other Buffers derived from it).
- int retries = 10;
- for (;;) {
- try {
- fc.truncate(nrBlocks * blockSize);
- break;
- } catch (ClosedChannelException ex) {
- // The Channel may have been inadvertently closed by another thread
- // being interrupted. Attempt to reopen the channel.
- if (!ensureOpen()) {
- throw ex;
- }
-
- // Loop back and retry the truncate().
- } catch (IOException ex) {
- if (retries-- == 0) {
- if (logger.isInfoEnabled()) {
- logger.info(
- "NOTE: Could not truncate file: \"" + file +
- "\" to size: " + (nrBlocks * blockSize) +
- " - deferring until next time the file is opened."
- );
- }
-
- // The size should be corrected the next time the file is opened.
- break;
- }
-
- // This is needed for Windows.
- System.gc();
- try { Thread.sleep(100); } catch (InterruptedException ie) { }
- System.runFinalization();
+ try {
+ MappingUtil.truncate(fc, nrBlocks * blockSize);
+ } catch (ClosedChannelException ex) {
+ // The Channel may have been inadvertently closed by another thread
+ // being interrupted. Attempt to reopen the channel.
+ if (!ensureOpen()) {
+ throw ex;
}
+ } catch (IOException ex) {
+ // The size should be corrected the next time the file is opened.
+ if (logger.isInfoEnabled()) {
+ logger.info(
+ "NOTE: Could not truncate file: \"" + file +
+ "\" to size: " + (nrBlocks * blockSize) +
+ " - deferring until next time the file is opened."
+ );
+ }
}
} finally {
try {
Modified: trunk/src/jar/util-xa/java/org/mulgara/store/xa/MappedBlockFile.java
===================================================================
--- trunk/src/jar/util-xa/java/org/mulgara/store/xa/MappedBlockFile.java 2011-07-18 21:50:12 UTC (rev 2003)
+++ trunk/src/jar/util-xa/java/org/mulgara/store/xa/MappedBlockFile.java 2011-07-19 00:01:35 UTC (rev 2004)
@@ -28,18 +28,17 @@
package org.mulgara.store.xa;
// Java 2 standard packages
-import java.io.*;
-import java.lang.reflect.Method;
-import java.nio.*;
-import java.nio.channels.*;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
+import java.nio.LongBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
-// Third party packages
-import org.apache.log4j.Logger;
+import org.mulgara.util.io.MappingUtil;
-import sun.misc.Cleaner;
-
/**
* An implementation of BlockFile which uses memory mapped file IO.
* Rather than mapping the entire file in one go, it gets mapped into
@@ -70,9 +69,6 @@
*/
public final static long REGION_SIZE = 8 * 1024 * 1024;
- /** The logger. */
- private final static Logger logger = Logger.getLogger(MappedBlockFile.class);
-
/**
* The system page size. The offset into a file that a mapped region starts
* at will be a multiple of this value. If this value is not equal to or
@@ -340,11 +336,7 @@
*/
public synchronized void unmap() {
// Discard the file mappings.
- if (mappedByteBuffers != null) {
- for (MappedByteBuffer buffer : mappedByteBuffers) {
- clean(buffer);
- }
- }
+ MappingUtil.release(mappedByteBuffers);
mappedByteBuffers = null;
srcByteBuffers = null;
@@ -474,22 +466,4 @@
nrMappedRegions = nrRegions;
}
- private static void clean(final Object buffer) {
- if (buffer != null) {
- AccessController.doPrivileged(new PrivilegedAction<Object>() {
- public Object run() {
- try {
- Method getCleanerMethod = buffer.getClass().getMethod("cleaner", new Class[0]);
- getCleanerMethod.setAccessible(true);
- Cleaner cleaner = (Cleaner)getCleanerMethod.invoke(buffer, new Object[0]);
- cleaner.clean();
- } catch (Exception e) {
- logger.warn("Error cleaning buffer", e);
- }
- return null;
- }
- });
- }
- }
-
}
More information about the Mulgara-svn
mailing list