[Mulgara-svn] r1995 - trunk/src/jar/util/java/org/mulgara/util/io
pag at mulgara.org
pag at mulgara.org
Fri Jul 15 03:21:49 UTC 2011
Author: pag
Date: 2011-07-15 03:21:49 +0000 (Fri, 15 Jul 2011)
New Revision: 1995
Modified:
trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFile.java
trunk/src/jar/util/java/org/mulgara/util/io/LIOBufferedFile.java
trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java
trunk/src/jar/util/java/org/mulgara/util/io/LReadOnlyIOBufferedFile.java
Log:
Introduced the ability to truncate files that have been memory mapped.
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFile.java 2011-07-15 03:20:20 UTC (rev 1994)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFile.java 2011-07-15 03:21:49 UTC (rev 1995)
@@ -121,6 +121,13 @@
public abstract void seek(long offset) throws IOException;
/**
+ * Truncates the file to a given size, or prepares the file for truncation
+ * if it is read-only.
+ * @param offset The offset into the file to trancate the size to.
+ */
+ public abstract void truncate(long offset) throws IOException;
+
+ /**
* Closes the file resource.
*/
public void close() throws IOException {
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LIOBufferedFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LIOBufferedFile.java 2011-07-15 03:20:20 UTC (rev 1994)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LIOBufferedFile.java 2011-07-15 03:21:49 UTC (rev 1995)
@@ -21,12 +21,17 @@
import java.nio.ByteBuffer;
import java.util.WeakHashMap;
+import org.apache.log4j.Logger;
+
/**
* Object for reading and writing to files using ByteBuffers and standard IO.
* @author pag
*/
public class LIOBufferedFile extends LBufferedFile {
+ /** Logger. */
+ private static final Logger logger = Logger.getLogger(LIOBufferedFile.class);
+
private WeakHashMap<SystemBuffer,Long> offsets = new WeakHashMap<SystemBuffer,Long>();
public LIOBufferedFile(RandomAccessFile file) {
@@ -70,6 +75,12 @@
/* no-op */
}
+ @Override
+ public void truncate(long offset) throws IOException {
+ if (logger.isDebugEnabled()) logger.debug("Truncating IO file to " + offset);
+ file.seek(offset - 1);
+ }
+
/**
* A wrapper class for allowing ByteBuffers to be mapped by == instead of their internal method.
*/
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java 2011-07-15 03:20:20 UTC (rev 1994)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java 2011-07-15 03:21:49 UTC (rev 1995)
@@ -8,12 +8,17 @@
import java.util.ArrayList;
import java.util.List;
+import org.apache.log4j.Logger;
+
/**
* A memory mapped read-only version of LBufferedFile
* @author pag
*/
public class LMappedBufferedFile extends LBufferedFile {
+ /** Logger. */
+ private static final Logger logger = Logger.getLogger(LMappedBufferedFile.class);
+
/** The name of the property for setting the page size */
static final String PAGE_SIZE_PROP = "org.mulgara.io.pagesize";
@@ -50,6 +55,7 @@
super(file);
fc = file.getChannel();
mapFile();
+ if (logger.isDebugEnabled()) logger.debug("Mapping files with pages of: " + PAGE_SIZE);
}
@Override
@@ -66,7 +72,20 @@
ByteBuffer bb = buffers[page].asReadOnlyBuffer();
bb.position(page_offset);
bb.limit(page_offset + length);
- return bb.slice();
+ if (page == buffers.length - 1) {
+ // In the final page, so make a copy
+ // This is needed in case of rollback, because the final page
+ // has the potential of being needed by a read-only transaction
+ // even when we need to remove that page to truncate the file.
+ // We have not synched on this page, but the GC will retry removing it
+ // with short delays, so it should get picked up despite our brief use.
+ ByteBuffer data = ByteBuffer.allocate(length);
+ bb.get(data.array(), 0, length);
+ return data;
+ } else {
+ // normal return of the mapped region slice
+ return bb.slice();
+ }
} else {
ByteBuffer data = ByteBuffer.allocate(length);
byte[] dataArray = data.array();
@@ -95,6 +114,79 @@
// no-op
}
+ /**
+ * Doesn't actually truncate the file, but unmaps the file so that it can be truncated.
+ * @param offset The offset into the file to trancate the size to.
+ */
+ public void truncate(long size) throws IOException {
+ if (logger.isDebugEnabled()) logger.debug("Truncating file to " + size + ". Current size is " + fc.size());
+
+ if (size == fc.size()) return;
+ if (size > fc.size()) throw new IOException("Unable to truncate a mapped file larger");
+
+ // get all the pages, including a possible partial page
+ int pages = (int)((size + PAGE_SIZE - 1) / PAGE_SIZE);
+ // get all the full pages. Either pages or pages-1
+ int fullPages = (int)(size / PAGE_SIZE);
+
+ if (logger.isDebugEnabled()) logger.debug("Existing file holds " + buffers.length + " pages. Truncated file needs " + pages + " pages (" + fullPages + " full pages)");
+ // if the data is fully mapped then there is nothing to do
+ if (fullPages == buffers.length) return;
+
+ // check that this really is a truncation
+ assert pages >= buffers.length;
+
+ // This will be the set of buffers to use in the end
+ MappedByteBuffer[] newBuffers;
+
+ if (pages == buffers.length) {
+ // can't be on a page boundary, else that would mean there was no truncation
+ // since it would either be truncated to the same size, or larger.
+ assert fullPages < pages;
+ // need to remap the final page
+ buffers[pages - 1] = null;
+ newBuffers = buffers;
+ if (logger.isDebugEnabled()) logger.debug("Remapping final page only");
+ } else {
+ assert pages < buffers.length;
+ // need to drop all of the last pages
+ for (int b = fullPages; b < buffers.length; b++) buffers[b] = null;
+ // truncate the buffers array
+ newBuffers = new MappedByteBuffer[pages];
+ System.arraycopy(buffers, 0, newBuffers, 0, fullPages);
+ if (logger.isDebugEnabled()) logger.debug("dropped " + (buffers.length - fullPages) + " pages, saved " + fullPages);
+ }
+
+ // if there's a partial page at the end, then map it
+ if (fullPages < pages) {
+ assert fullPages == pages - 1;
+ newBuffers[fullPages] = fc.map(FileChannel.MapMode.READ_ONLY, fullPages * PAGE_SIZE, size % PAGE_SIZE);
+ if (logger.isDebugEnabled()) logger.debug("Remapped final partial page");
+ }
+
+ // switch over from the previous buffers to the new ones
+ // anyone reading from anything but the partial buffer at the end is
+ // accessing the wrong transaction.
+ MappedByteBuffer[] tmpBuffers = buffers;
+ buffers = newBuffers;
+
+ // We're about to lose the last reference to these buffers when tmpBuffers
+ // goes away, but by putting the following code here anyone looking can see
+ // which buffers we're trying to eliminate.
+ // Note that this will remove any partial buffer at the end, even if most
+ // of it has been remapped into a new partial buffer.
+
+ // Setting these buffers to null is a belt and suspenders approach.
+ // This code is informative, rather than necessary
+ if (tmpBuffers != buffers) {
+ for (int b = fullPages; b < tmpBuffers.length; b++) tmpBuffers[b] = null;
+ }
+ if (logger.isDebugEnabled()) logger.debug("Removed " + (tmpBuffers.length - fullPages) + " pages");
+
+ // tell the listeners that we've remapped
+ for (Runnable listener: listeners) listener.run();
+ }
+
@Override
public void registerRemapListener(Runnable l) {
listeners.add(l);
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LReadOnlyIOBufferedFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LReadOnlyIOBufferedFile.java 2011-07-15 03:20:20 UTC (rev 1994)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LReadOnlyIOBufferedFile.java 2011-07-15 03:21:49 UTC (rev 1995)
@@ -56,6 +56,11 @@
}
@Override
+ public void truncate(long offset) throws IOException {
+ /* This API does not modify the file. */
+ }
+
+ @Override
public void registerRemapListener(Runnable l) {
/* no-op */
}
More information about the Mulgara-svn
mailing list