[Mulgara-svn] r1968 - trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11
pag at mulgara.org
pag at mulgara.org
Tue Jul 6 21:50:25 UTC 2010
Author: pag
Date: 2010-07-06 21:50:25 +0000 (Tue, 06 Jul 2010)
New Revision: 1968
Modified:
trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/DataAVLComparator.java
trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/DataStruct.java
trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java
Log:
Switched to using Long Buffered Files for read-only disk access. This defaults to mmap access, but can fall back to IO. Also added a WeakHashMap for caching anything that is not mapped
Modified: trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/DataAVLComparator.java
===================================================================
--- trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/DataAVLComparator.java 2010-07-06 21:48:03 UTC (rev 1967)
+++ trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/DataAVLComparator.java 2010-07-06 21:50:25 UTC (rev 1968)
@@ -17,13 +17,13 @@
package org.mulgara.store.stringpool.xa11;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import org.mulgara.store.stringpool.SPComparator;
import org.mulgara.store.stringpool.SPObject;
import org.mulgara.store.xa.AVLComparator;
import org.mulgara.store.xa.AVLNode;
+import org.mulgara.util.io.LBufferedFile;
/**
* Comparator for objects in the data pool.
@@ -39,9 +39,9 @@
private final int typeId;
private final int subtypeId;
private final ByteBuffer data;
- private final RandomAccessFile readOnlyFlatFile;
+ private final LBufferedFile readOnlyFlatFile;
- DataAVLComparator(SPComparator spComparator, SPObject.TypeCategory typeCategory, int typeId, int subtypeId, ByteBuffer data, RandomAccessFile flatFile) {
+ DataAVLComparator(SPComparator spComparator, SPObject.TypeCategory typeCategory, int typeId, int subtypeId, ByteBuffer data, LBufferedFile flatFile) {
this.spComparator = spComparator;
this.typeCategoryId = typeCategory.ID;
this.typeId = typeId;
@@ -50,7 +50,7 @@
this.readOnlyFlatFile = flatFile;
}
- DataAVLComparator(SPComparator spComparator, DataStruct dataStruct, RandomAccessFile readOnlyFlatFile) {
+ DataAVLComparator(SPComparator spComparator, DataStruct dataStruct, LBufferedFile readOnlyFlatFile) {
this.spComparator = spComparator;
this.typeCategoryId = dataStruct.getTypeCategoryId();
this.typeId = dataStruct.getTypeId();
Modified: trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/DataStruct.java
===================================================================
--- trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/DataStruct.java 2010-07-06 21:48:03 UTC (rev 1967)
+++ trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/DataStruct.java 2010-07-06 21:50:25 UTC (rev 1968)
@@ -17,7 +17,6 @@
package org.mulgara.store.stringpool.xa11;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
@@ -27,6 +26,7 @@
import org.mulgara.store.stringpool.SPTypedLiteral;
import org.mulgara.store.xa.AVLNode;
import org.mulgara.util.Constants;
+import org.mulgara.util.io.LBufferedFile;
/**
* Similar to a C-struct for storing and retrieving the data being stored by this string pool.
@@ -118,24 +118,19 @@
/**
* Reads a data structure from a file at a given offset.
- * @param file The file to read the structure from.
+ * @param bfile The file to read the structure from.
* @param gNode The gNode of the data to read.
*/
- public DataStruct(RandomAccessFile file, long gNode) throws IOException {
+ public DataStruct(LBufferedFile bfile, long gNode) throws IOException {
long offset = toOffset(gNode);
assert (offset & PADDING_MASK) == 0 : "Bad gNode value: " + gNode;
- synchronized (file) {
- file.seek(offset);
- ByteBuffer header = ByteBuffer.allocate(HEADER);
- file.read(header.array(), 0, HEADER);
- if (0 != header.get(0)) throw new IllegalStateException("Bad data found in Data Pool");
- typeCategoryId = header.get(IDX_TYPE_CATEGORY_B);
- typeId = header.get(IDX_TYPE_ID_B);
- subtypeId = header.get(IDX_SUBTYPE_ID_B);
- dataSize = header.getInt(IDX_DATA_SIZE_B);
- data = ByteBuffer.allocate(dataSize);
- file.readFully(data.array());
- }
+ ByteBuffer header = bfile.read(offset, HEADER);
+ if (0 != header.get(0)) throw new IllegalStateException("Bad data found in Data Pool");
+ typeCategoryId = header.get(IDX_TYPE_CATEGORY_B);
+ typeId = header.get(IDX_TYPE_ID_B);
+ subtypeId = header.get(IDX_SUBTYPE_ID_B);
+ dataSize = header.getInt(IDX_DATA_SIZE_B);
+ data = bfile.read(offset + HEADER, dataSize);
prefixOnly = false;
this.gNode = gNode;
}
@@ -240,23 +235,22 @@
/**
* Gets the remaining data of an object into the buffer.
- * @param file The file to read the data from.
+ * @param bfile The file to read the data from.
*/
- public void getRemainingBytes(RandomAccessFile file) throws IOException {
+ public void getRemainingBytes(LBufferedFile bfile) throws IOException {
// only need to get more if we only have the prefix
if (!prefixOnly) return;
// move the limit out to the end
data.limit(dataSize);
- synchronized (file) {
- // read the file starting at the data, plus the header, plus the already read portion
- file.seek(toOffset(gNode) + HEADER + MAX_DATA_SIZE);
- // read into the buffer, filling at the point where the data had been truncated.
- int remainingBytes = dataSize - MAX_DATA_SIZE;
- assert remainingBytes > 0;
- // we expect read to return everything from a file, so don't use readFully
- int dataRead = file.read(data.array(), MAX_DATA_SIZE, remainingBytes);
- if (dataRead != remainingBytes) throw new IOException("Unable to retrieve data from file.");
- }
+ // read the file starting at the data, plus the header, plus the already read portion
+ long location = toOffset(gNode) + HEADER + MAX_DATA_SIZE;
+ // read into the buffer, filling at the point where the data had been truncated.
+ int remainingBytes = dataSize - MAX_DATA_SIZE;
+ assert remainingBytes > 0;
+ ByteBuffer bb = bfile.read(location, remainingBytes);
+ data.position(MAX_DATA_SIZE);
+ data.put(bb);
+ data.position(0);
}
@@ -326,18 +320,18 @@
/**
* Gets the remaining data of an object into the buffer.
- * @param file The file to read the data from.
+ * @param bfile The file to read the data from.
*/
- public static void getRemainingBytes(ByteBuffer data, RandomAccessFile file, long gNode) throws IOException {
- synchronized (file) {
- // read the file starting at the data, plus the header, plus the already read portion
- file.seek(toOffset(gNode) + HEADER + MAX_DATA_SIZE);
- // read into the buffer, filling at the point where the data had been truncated.
- int remainingBytes = data.limit() - data.position();
- assert remainingBytes > 0;
- int dataRead = file.read(data.array(), MAX_DATA_SIZE, remainingBytes);
- if (dataRead != remainingBytes) throw new IOException("Unable to retrieve data from file.");
- }
+ public static void getRemainingBytes(ByteBuffer data, LBufferedFile bfile, long gNode) throws IOException {
+ // read the file starting at the data, plus the header, plus the already read portion
+ long location = toOffset(gNode) + HEADER + MAX_DATA_SIZE;
+ // read into the buffer, filling at the point where the data had been truncated.
+ int remainingBytes = data.limit() - data.position();
+ assert remainingBytes > 0;
+ ByteBuffer bb = bfile.read(location, remainingBytes);
+ data.position(MAX_DATA_SIZE);
+ data.put(bb);
+ data.position(0);
}
Modified: trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java
===================================================================
--- trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java 2010-07-06 21:48:03 UTC (rev 1967)
+++ trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java 2010-07-06 21:50:25 UTC (rev 1968)
@@ -26,6 +26,7 @@
import java.nio.channels.FileChannel;
import java.util.LinkedList;
import java.util.List;
+import java.util.WeakHashMap;
import org.apache.log4j.Logger;
import org.mulgara.query.Constraint;
@@ -62,6 +63,7 @@
import org.mulgara.util.Constants;
import org.mulgara.util.LongMapper;
import org.mulgara.util.functional.Pair;
+import org.mulgara.util.io.LBufferedFile;
import static org.mulgara.store.stringpool.xa11.DataStruct.*;
@@ -97,8 +99,8 @@
/** The index file for mapping data to a gNode. */
private AVLFile dataToGNode;
- /** The flat file for mapping gNodes to data. */
- private RandomAccessFile gNodeToDataReadOnly;
+ /** The file file reader for mapping gNodes to data. */
+ private LBufferedFile gNodeToDataFile;
/** The object for creating the output appender. */
private FileOutputStream gNodeToDataOutputStream;
@@ -154,6 +156,9 @@
/** A flag used to delay throwing an exception on the file version until it is needed. */
private boolean wrongFileVersion = false;
+ /** Cache the mapping of node IDs to objects */
+ private WeakHashMap<Long,SPObject> nodeCache = new WeakHashMap<Long,SPObject>();
+
/**
* Create a string pool instance using a set of directories.
* @param basenames A list of paths for creating string pool files in.
@@ -176,7 +181,7 @@
dataToGNode = new AVLFile(mainFilename + ".sp_avl", PAYLOAD_SIZE);
gNodeToDataOutputStream = new FileOutputStream(flatDataFilename, true);
gNodeToDataAppender = gNodeToDataOutputStream.getChannel();
- gNodeToDataReadOnly = new RandomAccessFile(flatDataFilename, "r");
+ gNodeToDataFile = LBufferedFile.createReadOnlyLBufferedFile(flatDataFilename);
} catch (IOException ex) {
try {
@@ -186,6 +191,9 @@
}
throw ex;
}
+
+ // clear the cache whenever the whole file is mapped
+ gNodeToDataFile.registerRemapListener(new Runnable() { public void run() { nodeCache.clear(); } });
}
@@ -239,6 +247,7 @@
nextGNodeValue += spObjectData.writeTo(gNodeToDataAppender);
mapObjectToGNode(spObjectData, spObject.getSPComparator());
informNodeListeners(gNode);
+ nodeCache.put(gNode, spObject);
return gNode;
} catch (IOException e) {
throw new StringPoolException("Unable to write to data files.", e);
@@ -309,8 +318,13 @@
if (BlankNodeAllocator.isBlank(node)) return null;
// outside of the allocated range
if (node >= nextGNodeValue) return null;
+
+ // Look aside into the cache first
+ SPObject cached = nodeCache.get(node);
+ if (cached != null) return cached;
+
try {
- return new DataStruct(gNodeToDataReadOnly, node).getSPObject();
+ return new DataStruct(gNodeToDataFile, node).getSPObject();
} catch (IllegalArgumentException iae) {
throw new StringPoolException("Bad node data. gNode = " + node, iae);
} catch (IOException ioe) {
@@ -357,7 +371,7 @@
} catch (IOException ex) {
throw new StringPoolException("I/O error deleting string pool.", ex);
} finally {
- gNodeToDataReadOnly = null;
+ gNodeToDataFile = null;
gNodeToDataAppender = null;
dataToGNode = null;
metarootFile = null;
@@ -794,7 +808,7 @@
unmap();
} finally {
try {
- if (gNodeToDataReadOnly != null) gNodeToDataReadOnly.close();
+ if (gNodeToDataFile != null) gNodeToDataFile.close();
} finally {
try {
if (gNodeToDataAppender != null) gNodeToDataAppender.close();
@@ -1255,7 +1269,7 @@
AVLNode[] findResult = null;
try {
- AVLComparator avlComparator = new DataAVLComparator(spComparator, objectData, gNodeToDataReadOnly);
+ AVLComparator avlComparator = new DataAVLComparator(spComparator, objectData, gNodeToDataFile);
// Find the adjacent nodes.
findResult = avlFilePhase.find(avlComparator, null);
@@ -1315,7 +1329,7 @@
try {
SPComparator spComparator = spObject.getSPComparator();
DataStruct objectData = new DataStruct(spObject);
- AVLComparator avlComparator = new DataAVLComparator(spComparator, objectData, gNodeToDataReadOnly);
+ AVLComparator avlComparator = new DataAVLComparator(spComparator, objectData, gNodeToDataFile);
// Find the SPObject.
findResult = avlFilePhase.find(avlComparator, null);
@@ -1413,7 +1427,7 @@
DataStruct lowData = new DataStruct(lowValue);
SPComparator spComparator = lowValue.getSPComparator();
// lowComparator = new SPAVLComparator(spComparator, typeCategory, typeId, data);
- lowComparator = new DataAVLComparator(spComparator, lowData, gNodeToDataReadOnly);
+ lowComparator = new DataAVLComparator(spComparator, lowData, gNodeToDataFile);
} else {
// Select the first node with the current type.
if (typeCategory == SPObject.TypeCategory.TYPED_LITERAL) {
@@ -1428,7 +1442,7 @@
if (highValue != null) {
DataStruct highData = new DataStruct(highValue);
SPComparator spComparator = highValue.getSPComparator();
- highComparator = new DataAVLComparator(spComparator, highData, gNodeToDataReadOnly);
+ highComparator = new DataAVLComparator(spComparator, highData, gNodeToDataFile);
} else {
// Select the first node past the last one that has the current type.
if (typeCategory == SPObject.TypeCategory.TYPED_LITERAL) {
@@ -1607,7 +1621,7 @@
private SPObject loadSPObject(SPObject.TypeCategory typeCategory, int typeId, AVLNode avlNode) throws StringPoolException {
DataStruct data = new DataStruct(avlNode);
try {
- data.getRemainingBytes(gNodeToDataReadOnly);
+ data.getRemainingBytes(gNodeToDataFile);
} catch (IOException e) {
throw new StringPoolException("Unable to read data pool", e);
}
More information about the Mulgara-svn
mailing list