[Mulgara-svn] r2065 - in trunk/src/jar: resolver/java/org/mulgara/resolver store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11 util/java/org/mulgara/util/io
pag at mulgara.org
pag at mulgara.org
Fri Oct 7 16:56:03 UTC 2011
Author: pag
Date: 2011-10-07 16:56:03 +0000 (Fri, 07 Oct 2011)
New Revision: 2065
Added:
trunk/src/jar/util/java/org/mulgara/util/io/ArrayBufferSetWrapper.java
trunk/src/jar/util/java/org/mulgara/util/io/Bytes.java
trunk/src/jar/util/java/org/mulgara/util/io/FileHashMap.java
trunk/src/jar/util/java/org/mulgara/util/io/FixedLengthSerializable.java
trunk/src/jar/util/java/org/mulgara/util/io/LLHashMap.java
trunk/src/jar/util/java/org/mulgara/util/io/LLHashMapUnitTest.java
trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileRO.java
trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileRW.java
trunk/src/jar/util/java/org/mulgara/util/io/RecordFile.java
trunk/src/jar/util/java/org/mulgara/util/io/RecordFileImpl.java
trunk/src/jar/util/java/org/mulgara/util/io/SetDataConverter.java
Modified:
trunk/src/jar/resolver/java/org/mulgara/resolver/RestoreOperation.java
trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java
trunk/src/jar/util/java/org/mulgara/util/io/IOUtil.java
trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFile.java
trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFileTest.java
trunk/src/jar/util/java/org/mulgara/util/io/LIOBufferedFile.java
trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java
trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileTest.java
trunk/src/jar/util/java/org/mulgara/util/io/LReadOnlyIOBufferedFile.java
Log:
Initial commit of memory mapped hash maps. This version requires fixed size on the keys and values, and only compares for equality on keys
Modified: trunk/src/jar/resolver/java/org/mulgara/resolver/RestoreOperation.java
===================================================================
--- trunk/src/jar/resolver/java/org/mulgara/resolver/RestoreOperation.java 2011-09-29 18:04:55 UTC (rev 2064)
+++ trunk/src/jar/resolver/java/org/mulgara/resolver/RestoreOperation.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -693,7 +693,7 @@
}
/** Need to maintain compatibility with the largest possible items */
- private static final int MAX_LINE = 3 * org.mulgara.util.io.LMappedBufferedFile.PAGE_SIZE;
+ private static final int MAX_LINE = 3 * org.mulgara.util.io.LMappedBufferedFileRO.PAGE_SIZE;
/**
* A wrapper around the {@link org.mulgara.util.io.IOUtil#readLine(BufferedReader, int)}
Modified: trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java
===================================================================
--- trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java 2011-09-29 18:04:55 UTC (rev 2064)
+++ trunk/src/jar/store-stringpool-xa11/java/org/mulgara/store/stringpool/xa11/XA11StringPoolImpl.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -182,7 +182,7 @@
dataToGNode = new AVLFile(mainFilename + ".sp_avl", PAYLOAD_SIZE);
gNodeToDataOutputStream = new FileOutputStream(flatDataFilename, true);
gNodeToDataAppender = gNodeToDataOutputStream.getChannel();
- gNodeToDataFile = LBufferedFile.createReadOnlyLBufferedFile(flatDataFilename);
+ gNodeToDataFile = LBufferedFile.createReadOnly(flatDataFilename);
} catch (IOException ex) {
try {
Added: trunk/src/jar/util/java/org/mulgara/util/io/ArrayBufferSetWrapper.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/ArrayBufferSetWrapper.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/ArrayBufferSetWrapper.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2011 Paul Gearon.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mulgara.util.io;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Set;
+
+/**
+ * A wrapper class for wrapping the sets of data returned from FileHashMap.
+ * The data is either ByteBuffer ({@link FileHashMap#keySet()} or {@link FileHashMap#values()})
+ * or {@link java.util.Map.Entry} of ByteBuffer -> ByteBuffer.
+ */
+public class ArrayBufferSetWrapper<E,SD> implements Set<E> {
+
+ private final Set<SD> dataset;
+
+ private final SetDataConverter<E,SD> serializer;
+
+ public ArrayBufferSetWrapper(Set<SD> ds, SetDataConverter<E,SD> ser) {
+ dataset = ds;
+ serializer = ser;
+ }
+
+ @Override
+ public boolean add(E a) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public boolean addAll(Collection<? extends E> a) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public void clear() {
+ dataset.clear();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public boolean contains(Object a) {
+ return dataset.contains(serializer.toSetData((E)a));
+ }
+
+ @Override
+ public boolean containsAll(Collection<?> a) {
+ for (Object v: a) if (!contains(v)) return false;
+ return true;
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return dataset.isEmpty();
+ }
+
+ @Override
+ public boolean remove(Object arg0) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public boolean removeAll(Collection<?> arg0) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public boolean retainAll(Collection<?> arg0) { throw new UnsupportedOperationException(); }
+
+ @Override
+ public int size() {
+ return dataset.size();
+ }
+
+ @Override
+ public Object[] toArray() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public <T> T[] toArray(T[] arg0) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Iterator<E> iterator() {
+ return new DataIterator(dataset.iterator());
+ }
+
+ /**
+ * Implementation of the iterator.
+ * @param <E> The object type to be returned by the iterator.
+ */
+ private class DataIterator implements Iterator<E> {
+
+ private final Iterator<SD> dataIterator;
+
+ public DataIterator(Iterator<SD> it) {
+ dataIterator = it;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return dataIterator.hasNext();
+ }
+
+ @Override
+ public E next() {
+ return serializer.fromSetData(dataIterator.next());
+ }
+
+ @Override
+ public void remove() { throw new UnsupportedOperationException(); }
+
+ }
+}
Added: trunk/src/jar/util/java/org/mulgara/util/io/Bytes.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/Bytes.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/Bytes.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2011 Paul Gearon.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mulgara.util.io;
+
+/**
+ * Constants pertaining to the bit representation of data types.
+ */
+public class Bytes {
+
+ /** Size of a long, in bytes */
+ public static final int LONG_SIZE = Long.SIZE >> 3;
+
+ /** Size of an int, in bytes */
+ public static final int INT_SIZE = Integer.SIZE >> 3;
+
+ /** Size of a short, in bytes */
+ public static final int SHORT_SIZE = Short.SIZE >> 3;
+
+ /** Size of a byte, in bytes */
+ public static final int BYTE_SIZE = Byte.SIZE >> 3;
+
+ /** Size of a float, in bytes */
+ public static final int FLOAT_SIZE = Float.SIZE >> 3;
+
+ /** Size of a double, in bytes */
+ public static final int DOUBLE_SIZE = Double.SIZE >> 3;
+
+}
Added: trunk/src/jar/util/java/org/mulgara/util/io/FileHashMap.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/FileHashMap.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/FileHashMap.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,1142 @@
+/*
+ * Copyright 2010 Paul Gearon.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mulgara.util.io;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.FloatBuffer;
+import java.nio.IntBuffer;
+import java.nio.LongBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+import static org.mulgara.util.io.Bytes.*;
+
+/**
+ * A file-based hashmap.
+ */
+public class FileHashMap implements Map<ByteBuffer,ByteBuffer>, Closeable {
+
+ /** Logger. */
+ @SuppressWarnings("unused")
+ private static final Logger logger = Logger.getLogger(FileHashMap.class);
+
+ /** A list of prime number sizes, increasing roughly by double. */
+ private static final long[] PRIMES = new long[] { 37L, 67L, 131L, 257L, 521L, 1031L, 2053L, 4099L,
+ 8209L, 16411L, 32771L, 65537L, 131101L, 262147L, 524309L, 1048583L, 2097169L, 4194319L,
+ 8388617L, 16777259L, 33554467L, 67108879L, 134217757L, 268435459L, 536870923L, 1073741827L,
+ 2147483659L, 4294967311L, 8589934609L, 17179869209L, 34359738421L, 68719476767L, 137438953481L,
+ 274877906951L, 549755813911L, 1099511627791L, 2199023255579L, 4398046511119L, 8796093022237L,
+ 17592186044423L, 35184372088891L, 70368744177679L, 140737488355333L, 281474976710677L,
+ // from here on, all the primes are probable
+ 562949953421381L, 1125899906842679L, 2251799813685269L, 4503599627370517L,
+ 9007199254740997L, 18014398509482143L, 36028797018963971L };
+
+ /** The default load factor, of nothing else is specified. */
+ public static final float DEFAULT_LOAD_FACTOR = 0.75f;
+
+ /** The file path for the main file. */
+ private final File path;
+
+ /** The file containing all the records. */
+ private final RecordFile file;
+
+ /** The metadata of the hash table. */
+ private final MetaData md;
+
+ /** The prime index of the current size to use. */
+ private int currentIndex = 0;
+
+ /** Total size of the file in records. fileRecords == PRIMES[currentSize] */
+ private long fileRecords = 0;
+
+ /** The size of the keys in the table. */
+ private final int keySize;
+
+ /** The size of the values in the table. */
+ private final int valueSize;
+
+ /** The size of the records in table. */
+ private final int recordSize;
+
+ /** The number of entries in this table. */
+ private long entries;
+
+ /** The max load in the table */
+ private final float loadFactor;
+
+ /** An empty value byte buffer */
+ private final ByteBuffer empty;
+
+ /** An empty key byte buffer */
+ private final ByteBuffer emptyKey;
+
+ /** An byte buffer representing an all 0 key */
+ private final ByteBuffer zeroKey;
+
+ /**
+ * Constructor with the default load factor.
+ * @param f The file to create.
+ * @param keySize The size of a serialized key, in bytes.
+ * @param valueSize The size of a serialized value, in bytes.
+ * @throws IOException Caused by a file error.
+ */
+ public FileHashMap(File f, int keySize, int valueSize) throws IOException {
+ this(f, keySize, valueSize, DEFAULT_LOAD_FACTOR, 0);
+ }
+
+ /**
+ * Main constructor.
+ * @param f The file to create.
+ * @param keySize The size of a serialized key, in bytes.
+ * @param valueSize The size of a serialized value, in bytes.
+ * @param loadFactor The maximum load factor for the hash table.
+ * @param initialSize The initial number of entries to build capacity for. This is a hint
+ * to help avoid unnecessary rehashing.
+ * @throws IOException Caused by a file error.
+ */
+ public FileHashMap(File f, int keySize, int valueSize, float loadFactor, long initialSize) throws IOException {
+ if (loadFactor <= 0f || loadFactor >= 1f) throw new IllegalArgumentException("Load factor must be between 0 and 1");
+ path = f;
+ boolean create = !f.exists();
+ md = new MetaData(f);
+ md.test(create, keySize, valueSize, loadFactor, f.length());
+ if (create) {
+ md.setKeySize(this.keySize = keySize);
+ md.setValueSize(this.valueSize = valueSize);
+ md.setLoadFactor(this.loadFactor = loadFactor);
+ md.setPrimesIndex(currentIndex = indexOfNextSize(initialSize, loadFactor));
+ md.setEntries(entries = 0L);
+ } else {
+ this.keySize = md.getKeySize();
+ this.valueSize = md.getValueSize();
+ this.loadFactor = md.getLoadFactor();
+ currentIndex = md.getPrimesIndex();
+ entries = md.getEntries();
+ }
+ recordSize = keySize + valueSize;
+ fileRecords = PRIMES[currentIndex];
+ file = new RecordFileImpl(f, recordSize, fileRecords);
+ if (create) file.resize(fileRecords);
+
+ empty = IOUtil.allocate(valueSize).asReadOnlyBuffer(); // all zeros
+ emptyKey = IOUtil.allocate(keySize).asReadOnlyBuffer(); // all zeros
+ ByteBuffer tmpZ = IOUtil.allocate(keySize); // all ones
+ for (int i = 0; i < keySize; i++) tmpZ.put(i, (byte)0xFF);
+ zeroKey = tmpZ.asReadOnlyBuffer();
+ }
+
+ /**
+ * @see java.io.Closeable#close()
+ */
+ public void close() throws IOException {
+ file.close();
+ md.close();
+ }
+
+ /**
+ * @see java.io.Closeable#close()
+ */
+ public void closeAndDelete() throws IOException {
+ file.close();
+ path.delete();
+ md.closeAndDelete();
+ }
+
+ /**
+ * @see java.util.Map#clear()
+ */
+ @Override
+ public void clear() {
+ byte[] empty = new byte[recordSize];
+ try {
+ entries = 0L;
+ md.setEntries(entries);
+ setFileRecords(0);
+ for (long i = 0; i < fileRecords; i++) {
+ ByteBuffer b = file.getBuffer(i);
+ b.clear();
+ b.put(empty);
+ file.put(b, i);
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ /**
+ * @see java.util.Map#containsKey(java.lang.Object)
+ */
+ @Override
+ public boolean containsKey(Object k) {
+ if (k == null) return false;
+ ByteBuffer key = sanitizeKey((ByteBuffer)k);
+ try {
+ long startPos = recordPosition(key);
+ long pos = startPos;
+ ByteBuffer record = file.getBuffer(pos);
+ // search for the next empty position. Sanity guard against wraparound.
+ while (!emptyKey(record) && pos != startPos - 1) {
+ if (equalsKey(record, key)) return true;
+ pos = incPos(pos);
+ }
+ return false;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * This operation is inappropriate for most applications.
+ * @see java.util.Map#containsValue(java.lang.Object)
+ */
+ @Override
+ public boolean containsValue(Object v) {
+ if (v == null) return false;
+ ByteBuffer value = (ByteBuffer)v;
+ try {
+ long entryCount = 0;
+ for (long r = 0; r < PRIMES[currentIndex]; r++) {
+ ByteBuffer record = file.get(r);
+ if (!emptyKey(record)) {
+ entryCount++;
+ record.position(keySize);
+ if (value.equals(record.slice())) return true;
+ }
+ if (entryCount >= entries) break;
+ }
+ return false;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @see java.util.Map#entrySet()
+ */
+ @Override
+ public Set<Map.Entry<ByteBuffer,ByteBuffer>> entrySet() {
+ return new DataSet<Map.Entry<ByteBuffer,ByteBuffer>>(new EntryReader());
+ }
+
+ /**
+ * @see java.util.Map#keySet()
+ */
+ @Override
+ public Set<ByteBuffer> keySet() {
+ return new DataSet<ByteBuffer>(new KeyReader());
+ }
+
+ /**
+ * @see java.util.Map#values()
+ */
+ @Override
+ public Collection<ByteBuffer> values() {
+ return new DataSet<ByteBuffer>(new ValueReader());
+ }
+
+ /**
+ * @see java.util.Map#remove(java.lang.Object)
+ */
+ @Override
+ public ByteBuffer remove(Object k) {
+ if (k == null) return null;
+ ByteBuffer key = sanitizeKey((ByteBuffer)k);
+ try {
+ // get the data to be removed
+ long removePos = recordPosition(key);
+ ByteBuffer removeRecord = file.getBuffer(removePos);
+ if (equalsKey(removeRecord, key)) {
+ removeRecord.position(keySize);
+ ByteBuffer value = copy(removeRecord.slice());
+ removeFromPos(removePos, removeRecord);
+ md.setEntries(--entries);
+ return value;
+ } else {
+ return null;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return entries == 0L;
+ }
+
+ /**
+ * @see java.util.Map#putAll(java.util.Map)
+ */
+ @Override
+ public void putAll(Map<? extends ByteBuffer, ? extends ByteBuffer> src) {
+ for (Map.Entry<? extends ByteBuffer, ? extends ByteBuffer> e: src.entrySet()) {
+ put(e.getKey(), e.getValue());
+ }
+ }
+
+ /**
+ * This will return an incorrect number if the number of entries is larger than Integer.MAX_VALUE
+ * @see java.util.Map#size()
+ */
+ @Override
+ public int size() {
+ return entries > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int)entries;
+ }
+
+ /**
+ * Returns the actual size, which may be larger than {@link Integer#MAX_VALUE}
+ * @return The complete number of entries.
+ */
+ public long realSize() {
+ return entries;
+ }
+
+ /**
+ * @see java.util.Map#get(java.lang.Object)
+ */
+ @Override
+ public ByteBuffer get(Object k) {
+ if (k == null) return null;
+ ByteBuffer key = sanitizeKey((ByteBuffer)k);
+ try {
+ long startPos = recordPosition(key);
+ long pos = startPos;
+ ByteBuffer record = file.getBuffer(pos);
+ // search for the next empty position. Sanity guard against wraparound.
+ while (!emptyKey(record) && pos != startPos - 1) {
+ if (equalsKey(record, key)) {
+ record.position(keySize);
+ return record.slice().asReadOnlyBuffer();
+ }
+ pos = incPos(pos);
+ record = file.getBuffer(pos);
+ }
+ // not found. Return null
+ return null;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * @see java.util.Map#put(java.lang.Object, java.lang.Object)
+ * key and value may not be null as these cannot be round tripped. The best we could do
+ * would be to use an all zero buffer, but that's not the same thing.
+ */
+ @Override
+ public ByteBuffer put(ByteBuffer key, ByteBuffer value) {
+ if (key == null) throw new IllegalArgumentException("Null keys not allowed");
+ if (value == null) throw new IllegalArgumentException("Null values not allowed");
+ if (loadFactor() > loadFactor) {
+ try {
+ rehash();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return put(key, value);
+ }
+
+ key = sanitizeKey(key);
+
+ // find the right space
+ try {
+ ByteBuffer fileData;
+ boolean found = false;
+ // start just before the location, so can increment immediately
+ long startRec = recordPosition(key) - 1;
+ long rec = startRec;
+ do {
+ rec = incPos(rec);
+ assert rec != startRec;
+ fileData = file.get(rec);
+ } while (!emptyKey(fileData) && !(found = equalsKey(fileData, key)));
+
+ if (found) {
+ // get the value buffer
+ fileData.position(keySize);
+ ByteBuffer valueData = fileData.slice();
+ // copy the old value out
+ byte[] resultData = new byte[valueSize];
+ valueData.get(resultData);
+ // write the new value into the buffer
+ valueData.position(0);
+ valueData.put(value);
+ // flush the record buffer back to disk
+ file.put(fileData, rec);
+ return ByteBuffer.wrap(resultData);
+ }
+
+ // write the key and value to the record
+ key.position(0);
+ value.position(0);
+ fileData.put(key);
+ fileData.put(value);
+ file.put(fileData, rec);
+ md.setEntries(++entries);
+ empty.position(0);
+ return empty.duplicate();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Remove an entire record from a position, shifting anything eligible back to take its place.
+ * @param removePos The position to remove from.
+ * @param removeBuffer a buffer associated with removePos.
+ * @throws IOException Due to an error accessing the file.
+ */
+ private void removeFromPos(long removePos, ByteBuffer removeBuffer) throws IOException {
+ // look for things that can come back to this position
+ long position = removePos;
+ ByteBuffer scanRecord;
+ // look until an empty space is found
+ while (!emptyKey(scanRecord = file.get(position = incPos(position)))) {
+ scanRecord.limit(keySize);
+ long scanPos = recordPosition(scanRecord.slice());
+ if (scanPos <= removePos) {
+ // eligible to move back
+ removeBuffer.position(0);
+ scanRecord.clear();
+ removeBuffer.put(scanRecord);
+ file.put(removeBuffer, removePos);
+ // now remove the scanRecord and move anything elegible back into it
+ removeFromPos(position, scanRecord);
+ return;
+ }
+ }
+ // Nothing was moved back here, so clear it out
+ empty.position(0);
+ removeBuffer.position(0);
+ removeBuffer.put(empty);
+ file.put(removeBuffer, removePos);
+ }
+
+ /**
+ * Increment a record position.
+ * @param p The record position to increment.
+ * @return The incremented position.
+ */
+ private long incPos(long p) {
+ if (++p == fileRecords) p = 0;
+ return p;
+ }
+
+ /**
+ * Expand the file and put all existing data into its new position.
+ * @throws IOException If there is an error modifying the file.
+ */
+ private void rehash() throws IOException {
+ long oldRecordCount = fileRecords;
+ setFileRecords(currentIndex + 1);
+ long movedRecords = 0;
+ // check all existing records
+ for (long recPos = 0; recPos < oldRecordCount; recPos++) {
+ if (movedRecords == entries) break;
+ // get the record and the key from the record
+ ByteBuffer record = file.get(recPos);
+ record.limit(keySize);
+ ByteBuffer key = record.slice();
+ // move the non-empty records
+ if (!emptyKey(key)) {
+ // determine the new location
+ long newPos = recordPosition(key);
+ // move the data to that location
+ if (recPos != newPos) {
+ newPos = moveTo(record, recPos, newPos, recPos);
+ // after moving, clear the source
+ if (recPos != newPos) {
+ record.clear();
+ emptyKey.position(0);
+ record.put(emptyKey);
+ file.put(record, recPos);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Move a buffer from a position to a new location, or the first available location after.
+ * @param record The buffer contents to be moved.
+ * @param srcPos The location of the source buffer.
+ * @param toPos The initial location to move to.
+ * @param processedRecords The number of records already processed. 1 past the offset of the last processed record.
+ * @return The final location the buffer was moved to.
+ */
+ private long moveTo(ByteBuffer record, long srcPos, long toPos, long processedRecords) throws IOException {
+ assert srcPos != toPos;
+
+ // move forward through the records until we identify one that is available
+ Availability avail;
+ ByteBuffer destRecord = file.get(toPos);
+ while (AvailState.OCCUPIED == (avail = availability(destRecord, toPos, processedRecords)).state) {
+ toPos = incPos(toPos);
+ // if we wrapped all the way around (eg. first element tries to write to occupied tail) then exit
+ if (toPos == srcPos) return toPos;
+ destRecord = file.get(toPos);
+ }
+
+ // if destination is waiting to move, then move it out of the way
+ if (avail.state == AvailState.WAITING) {
+ moveTo(destRecord, toPos, avail.pos, processedRecords);
+ }
+
+ // destination now clear, write to it
+ destRecord.clear();
+ record.clear();
+ destRecord.put(record);
+ file.put(destRecord, toPos);
+
+ return toPos;
+ }
+
+ /**
+ * Tests if a record location is available.
+ * A record is available if it is empty, or the data in it has not yet been moved in a rehash.
+ * @param record The record to check.
+ * @param recPos The current location of the record.
+ * @param processedRecords The number of records already processed. 1 past the offset of the last processed record.
+ * @return An Availability, containing a state of EMPTY, OCCUPIED or WAITING. The WAITING state
+ * will also contain the new location the record is to move to.
+ */
+ private Availability availability(ByteBuffer record, long recPos, long processedRecords) throws IOException {
+ // check for an empty cell
+ if (emptyKey(record)) return Availability.A_EMPTY;
+
+ // if record is in the already-processed area, or in the new area, then it was placed there during this rehash
+ if (recPos < processedRecords || recPos >= PRIMES[currentIndex - 1]) {
+ return Availability.A_OCCUPIED;
+ }
+
+ // compare the calculated position to the current position
+ record.limit(keySize);
+ long newPos = recordPosition(record.slice());
+ if (newPos == recPos) return Availability.A_OCCUPIED; // record is supposed to be here
+
+ // record might still be in the right place, but he have to check
+
+ // if record is before its desired location, it hasn't been moved yet
+ if (recPos < newPos) return new Availability(AvailState.WAITING, newPos);
+
+ // record is in unprocessed area. Search from desired space forward to the current space.
+ // an empty record means it wasn't supposed to be here.
+ long rec = newPos;
+ while (rec != recPos) {
+ ByteBuffer searchBuffer = file.get(rec);
+ if (emptyKey(searchBuffer)) {
+ // this record is waiting to be moved to newPos
+ return new Availability(AvailState.WAITING, newPos);
+ }
+ rec = incPos(rec);
+ }
+ // unbroken line of records to the current space, so the record is already where it needs to be
+ return Availability.A_OCCUPIED;
+ }
+
+ /** Enumeration of the possible record states during rehashing */
+ private enum AvailState {
+ EMPTY, // nothing here, available to store data in
+ OCCUPIED, // something already at the hoped-for location
+ WAITING // something at the hoped-for location, but it is yet to be moved
+ };
+
+ /** A class for encoding availability state of a record */
+ private static class Availability {
+ public static final Availability A_EMPTY = new Availability(AvailState.EMPTY, 0);
+ public static final Availability A_OCCUPIED = new Availability(AvailState.OCCUPIED, 0);
+ final AvailState state;
+ final long pos;
+ Availability(AvailState s, long p) {
+ state = s;
+ pos = p;
+ }
+ }
+
+ /**
+ * Determines the load factor.
+ * @return The current load factor.
+ */
+ private final float loadFactor() {
+ return (float)((double)entries / fileRecords);
+ }
+
+ /**
+ * Since all zeros in the file means there is no entry, we flip keys with all zeros to all ones.
+ * @param k The original key.
+ * @return If k was all zeros, then a buffer with all ones. Else the original k.
+ */
+ private final ByteBuffer sanitizeKey(ByteBuffer k) {
+ boolean allF = true;
+ for (int i = 0; i < k.capacity(); i++) {
+ byte v = k.get(i);
+ if (allF && v != 0xFF) allF = false;
+ if (v != 0) { // non-zero means key is probably normal
+ if (!allF) return k; // so long as not EVERY bype is 0xFF
+ // continue the search for all bytes of 0xFF
+ for (i++; i < k.capacity(); i++) {
+ if (k.get(i) != 0xFF) return k; // a normal key
+ }
+ throw new RuntimeException("Cannot accept a key of -1");
+ }
+ }
+ return zeroKey.duplicate();
+ }
+
+ /**
+ * If all the bits are one, then return the empty key.
+ * @param k The original key.
+ * @return If k was all ones, then a buffer with all zeros. Else the original k.
+ */
+ private final ByteBuffer desanitizeKey(ByteBuffer k) {
+ for (int i = 0; i < k.capacity(); i++) {
+ if (k.get(i) != 0xFF) return k;
+ }
+ emptyKey.position(0);
+ return emptyKey.duplicate();
+ }
+
+ /**
+ * Tests if the key in a record is empty.
+ * @param kb The data containing the key.
+ * @return <code>true</code> iff the record is empty.
+ */
+ private final boolean emptyKey(ByteBuffer kb) {
+ for (int i = keySize - 1; i >= 0; i--) {
+ if (kb.get(i) != (byte)0) return false;
+ }
+ return true;
+ }
+
+ /**
+ * Tests if a file buffer contains a given key
+ * @param kb The byte buffer of the key.
+ * @param key The key to look for.
+ * @return <code>true</code> iff the buffer contains the key.
+ */
+ private final boolean equalsKey(ByteBuffer kb, ByteBuffer key) {
+ // compare remaining bytes, if any
+ for (int i = keySize - 1; i >= 0; i--) {
+ if (kb.get(i) != key.get(i)) return false;
+ }
+ return true;
+ }
+
+ /**
+ * Sets the number of records to be used by this file, and set the file to use that size.
+ * @param primeIndex The new index into the prime numbers to use.
+ * @return The number of records in the new file.
+ */
+ private final long setFileRecords(int primeIndex) throws IOException {
+ currentIndex = primeIndex;
+ md.setPrimesIndex(currentIndex);
+ fileRecords = PRIMES[currentIndex];
+ file.resize(fileRecords);
+ return fileRecords;
+ }
+
+ /**
+ * Calculate the location for a record, based on its key.
+ * @param key The ByteBuffer for the key.
+ * @return The record number in the file that the key specifies.
+ */
+ private final long recordPosition(ByteBuffer key) {
+ return longHash(key.hashCode()) % (long)fileRecords;
+ }
+
+ /**
+ * Mix an int into a long in a reasonably cheap way.
+ * @param The int to mix.
+ * @return A long value with the integer mixed into it. Positive numbers only.
+ */
+ private static final long longHash(int h) {
+ long v = (long)h;
+ v ^= v << 5;
+ v ^= (v << 11) + 1049;
+ v ^= ((v >> 32) | (v << 32));
+ v ^= (v << 17) + 131041;
+ v ^= ((v >> 56) | (v << 56));
+ v ^= (v << 23) + 8313581;
+ v ^= (((v >> 8) & 0xFF000000L) | ((v << 8) & 0xFF00000000L));
+ v ^= (v << 37) + 2147483659L;
+ v ^= ((v >> 32) | (v << 32));
+ return v & 0x7FFFFFFFFFFFFFFFL;
+ }
+
+ /**
+ * Returns the index of the first prime that is greater than or equal to the requested size.
+ * @param s The requested size.
+ * @return The index into PRIMES to use.
+ */
+ private static final int indexOfNextSize(long s, float loadFactor) {
+ // look for the minimum request
+ if (s == 0) return 0;
+ // increase the required size to include the load factor
+ long sz = (long)(s / (double)loadFactor);
+ // sanity check on the size
+ if (sz <= s) return PRIMES.length - 1;
+ for (int i = 0; i < PRIMES.length; i++) {
+ if (PRIMES[i] >= sz) return i;
+ }
+ return PRIMES.length - 1;
+ }
+
+ /**
+ * Make a copy of a byte buffer.
+ * @param bb The ByteBuffer to copy.
+ * @return A new byte buffer with the same contents as bb.
+ */
+ private static final ByteBuffer copy(ByteBuffer bb) {
+ ByteBuffer cp = IOUtil.allocate(bb.capacity());
+ cp.put(bb);
+ cp.position(0);
+ return cp;
+ }
+
+ /**
+ * An interface for generalizing data access for keys, values, and key/value pairs.
+ * @param <D> The type of data being acccessed.
+ */
+ private interface DataReader<D> {
+ /**
+ * Test is data is available in the collection.
+ * @param val The data to look for.
+ * @return <code>true</code> iff the data is available.
+ */
+ public boolean contains(Object val);
+ /**
+ * Read out the appropriate data type.
+ * @param record The location of the record with the data.
+ * @return The relevant data from the record.
+ */
+ public D read(long record);
+ }
+
+ /**
+ * A class for reading keys for the Key Set.
+ */
+ private class KeyReader implements DataReader<ByteBuffer> {
+ public boolean contains(Object val) {
+ return containsKey(val);
+ }
+ public ByteBuffer read(long recordId) {
+ try {
+ ByteBuffer data = file.get(recordId);
+ data.limit(keySize);
+ return desanitizeKey(data.slice()).asReadOnlyBuffer();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /**
+ * A class for reading values for the Value collection.
+ */
+ private class ValueReader implements DataReader<ByteBuffer> {
+ public boolean contains(Object val) {
+ return containsValue(val);
+ }
+ public ByteBuffer read(long recordId) {
+ try {
+ ByteBuffer data = file.get(recordId);
+ data.position(keySize);
+ return data.slice().asReadOnlyBuffer();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /** A class for reading values for the Entry set. */
+ private class EntryReader implements DataReader<Map.Entry<ByteBuffer,ByteBuffer>> {
+ public boolean contains(Object val) {
+ ByteBuffer b = (ByteBuffer)val;
+ b.limit(keySize);
+ ByteBuffer key = sanitizeKey(b.slice());
+ try {
+ long startPos = recordPosition(key);
+ long pos = startPos;
+ ByteBuffer record = file.getBuffer(pos);
+ // search the valid keys
+ while (!emptyKey(record) && pos != startPos - 1) {
+ // find a matching key
+ if (equalsKey(record, key)) {
+ // check if the value is the same
+ for (int i = keySize; i < recordSize; i++) {
+ if (b.get(i) != record.get(i)) return false;
+ }
+ return true;
+ }
+ pos = incPos(pos);
+ }
+ return false;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ public Map.Entry<ByteBuffer,ByteBuffer> read(long recordId) {
+ try {
+ return new KeyValue(file.get(recordId), recordId);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ /** A {@link Map.Entry} implementation for use with {@link FileHashMap#entrySet()}. */
+ private class KeyValue implements Map.Entry<ByteBuffer, ByteBuffer> {
+ private final ByteBuffer recordBuffer;
+ private final ByteBuffer key;
+ private final ByteBuffer value;
+ private final long recordId;
+ public KeyValue(ByteBuffer bb, long id) {
+ recordBuffer = bb;
+ recordId = id;
+ bb.position(keySize);
+ value = bb.slice();
+ bb.position(0);
+ bb.limit(keySize);
+ key = desanitizeKey(bb.slice()).asReadOnlyBuffer();
+ }
+ public ByteBuffer getKey() { return key; }
+
+ public ByteBuffer getValue() { return value; }
+
+ public ByteBuffer setValue(ByteBuffer v) {
+ byte[] old = new byte[valueSize];
+ value.position(0);
+ value.limit(valueSize);
+ value.get(old);
+ value.position(0);
+ v.position(0);
+ v.limit(valueSize);
+ value.put(v);
+ try {
+ file.put(recordBuffer, recordId);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return ByteBuffer.wrap(old);
+ }
+ }
+
+ class DataSet<T> implements Set<T> {
+
+ private final DataReader<T> reader;
+
+ public DataSet(DataReader<T> reader) {
+ this.reader = reader;
+ }
+
+ public boolean add(T arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean addAll(Collection<? extends T> arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean contains(Object val) {
+ return reader.contains(val);
+ }
+
+ public boolean containsAll(Collection<?> vals) {
+ for (Object v: vals) {
+ if (!contains(v)) return false;
+ }
+ return true;
+ }
+
+ public boolean isEmpty() {
+ return FileHashMap.this.isEmpty();
+ }
+
+ public Iterator<T> iterator() {
+ return new DataIterator();
+ }
+
+ public boolean remove(Object arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean removeAll(Collection<?> arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean retainAll(Collection<?> arg0) {
+ throw new UnsupportedOperationException();
+ }
+
+ public int size() {
+ return FileHashMap.this.size();
+ }
+
+ /**
+ * You're kidding, right?
+ * @see java.util.Set#toArray()
+ */
+ public Object[] toArray() {
+ ByteBuffer[] b = new ByteBuffer[size()];
+ return toArray(b);
+ }
+
+ /**
+ * This is a really bad idea.
+ * @see java.util.Set#toArray(T[])
+ */
+ @SuppressWarnings("unchecked")
+ public <T2> T2[] toArray(T2[] a) {
+ Iterator<T> i = new DataIterator();
+ if (size() > a.length) a = (T2[]) new Object[size()];
+ int pos = 0;
+ try {
+ while (i.hasNext()) {
+ if (pos >= a.length) break;
+ a[pos++] = (T2)i.next();
+ }
+ } catch (ClassCastException e) {
+ throw new ArrayStoreException();
+ }
+ return a;
+ }
+
+ /** Private iterator for the DataSet class. Uses the encapsulated reader. */
+ private class DataIterator implements Iterator<T> {
+
+ private long pos;
+
+ public DataIterator() {
+ pos = 0;
+ nextFull();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return pos < fileRecords;
+ }
+
+ @Override
+ public T next() {
+ T result = reader.read(pos++);
+ nextFull();
+ return result;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ private void nextFull() {
+ try {
+ while (pos < fileRecords && emptyKey(file.get(pos))) pos++;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
+
+ static class MetaData implements Closeable {
+ /** A filename extension for metadata files. */
+ private static final String MD_EXT = ".hmd";
+ /** The metadata offset of the entries value, in bytes */
+ private static final int ENTRY_OFFSET = 0;
+ /** The metadata offset of the entries value, in longs */
+ private static final int ENTRY_OFFSET_L = 0;
+
+ /** The metadata offset of the keySize, in bytes */
+ private static final int KEY_SIZE_OFFSET = ENTRY_OFFSET + LONG_SIZE;
+ /** The metadata offset of the keySize, in ints */
+ private static final int KEY_SIZE_OFFSET_I = KEY_SIZE_OFFSET / INT_SIZE;
+
+ /** The metadata offset of the valueSize, in ints */
+ private static final int VALUE_SIZE_OFFSET = KEY_SIZE_OFFSET + INT_SIZE;
+ /** The metadata offset of the valueSize, in ints */
+ private static final int VALUE_SIZE_OFFSET_I = VALUE_SIZE_OFFSET / INT_SIZE;
+
+ /** The metadata offset of the primes Index, in bytes */
+ private static final int PRIMES_INDEX_OFFSET = VALUE_SIZE_OFFSET + INT_SIZE;
+ /** The metadata offset of the valueSize, in ints */
+ private static final int PRIMES_INDEX_OFFSET_I = PRIMES_INDEX_OFFSET / INT_SIZE;
+
+ /** The metadata offset of the loadFactor, in bytes */
+ private static final int LOAD_FACTOR_OFFSET = PRIMES_INDEX_OFFSET + INT_SIZE;
+
+ /** The total size of the metadata, in bytes */
+ private static final int TOTAL_SIZE = LOAD_FACTOR_OFFSET + FLOAT_SIZE;
+
+ /** The path of this metadata file. */
+ private final File path;
+
+ /** File for metadata about the hash table */
+ private final FileChannel mdFile;
+
+ /** Metadata for the hash table */
+ private ByteBuffer md;
+
+ /** Metadata for the hash table in Longs */
+ private LongBuffer mdLong;
+
+ /** Metadata for the hash table in Integers */
+ private IntBuffer mdInt;
+
+ /** Metadata for the hash table in Floats */
+ private FloatBuffer mdLoadFactor;
+
+ /** Indicates that this object was created fresh */
+ private final boolean created;
+
+ public MetaData(File f) throws IOException {
+ path = new File(f.getAbsolutePath() + MD_EXT);
+ created = !path.exists();
+ RandomAccessFile file = new RandomAccessFile(path, "rw");
+ if (created) {
+ file.setLength(TOTAL_SIZE);
+ } else {
+ long length = path.length();
+ if (length < TOTAL_SIZE) throw new IOException("HashMap Metadata file too short (" + length + ")");
+ if (length > TOTAL_SIZE) throw new IOException("Corrupt Metadata file: too long (" + length + ")");
+ }
+ mdFile = file.getChannel();
+ md = mdFile.map(FileChannel.MapMode.READ_WRITE, 0, TOTAL_SIZE);
+ mdLong = md.asLongBuffer();
+ mdInt = md.asIntBuffer();
+ md.position(LOAD_FACTOR_OFFSET);
+ mdLoadFactor = md.slice().asFloatBuffer();
+ }
+
+ /**
+ * @return <code>true</code> if the metadata was created from scratch.
+ * <code>false</code> if it was loaded from an existing file.
+ */
+ public boolean created() {
+ return created;
+ }
+
+ /**
+ * Test if the metadata matches the given values.
+ * @param tableCreate Indicates that the table is to be created.
+ * @param ks The provided keySize, or 0 if ignored.
+ * @param vs The provided valueSize, or 0 if ignored.
+ * @param l The provided loadFactor, or 0f if ignored.
+ * @param len The provided file length, or 0L if ignored.
+ * @throws IOException If the metadata does not match the provided info.
+ */
+ public void test(boolean tableCreate, int ks, int vs, float l, long len) throws IOException {
+ if (created) {
+ if (!tableCreate) throw new InvalidObjectException("Bad FileHashMap structure. Table exists, but metadata missing.");
+ if (ks == 0) throw new IllegalArgumentException("Key size may not be zero");
+ if (l == 0f || l >= 1f) throw new IllegalArgumentException("Load factor out of bounds");
+ } else {
+ if (tableCreate && getEntries() != 0) {
+ throw new InvalidObjectException("Bad FileHashMap request. Metadata for " + getEntries() + " entries, but missing table file");
+ }
+ if (ks != 0 && ks != getKeySize()) {
+ throw new InvalidObjectException("Bad FileHashMap request. Key size = " + ks + ", but metadata says: " + getKeySize());
+ }
+ if (vs != 0 && vs != getValueSize()) {
+ throw new InvalidObjectException("Bad FileHashMap request. Value size = " + vs + ", but metadata says: " + getValueSize());
+ }
+ if (l != 0f && l != getLoadFactor()) {
+ throw new InvalidObjectException("Bad FileHashMap request. Load Factor = " + vs + ", but metadata says: " + getLoadFactor());
+ }
+ int recordSize = getKeySize() + getValueSize();
+ if (len != 0 && len != PRIMES[getPrimesIndex()] * recordSize) {
+ throw new InvalidObjectException("Bad FileHashMap request. Size = " + len + ", but metadata says: " + PRIMES[getPrimesIndex()]);
+ }
+ }
+ }
+
+ public void close() throws IOException {
+ mdFile.close();
+ md = null;
+ mdLong = null;
+ mdInt = null;
+ mdLoadFactor = null;
+ }
+
+ public void closeAndDelete() throws IOException {
+ close();
+ path.delete();
+ }
+
+ public MetaData setEntries(long entries) {
+ mdLong.put(ENTRY_OFFSET_L, entries);
+ return this;
+ }
+
+ public long getEntries() {
+ return mdLong.get(ENTRY_OFFSET_L);
+ }
+
+ public MetaData setKeySize(int keySize) {
+ mdInt.put(KEY_SIZE_OFFSET_I, keySize);
+ return this;
+ }
+
+ public int getKeySize() {
+ return mdInt.get(KEY_SIZE_OFFSET_I);
+ }
+
+ public MetaData setValueSize(int valueSize) {
+ mdInt.put(VALUE_SIZE_OFFSET_I, valueSize);
+ return this;
+ }
+
+ public int getValueSize() {
+ return mdInt.get(VALUE_SIZE_OFFSET_I);
+ }
+
+ public MetaData setPrimesIndex(int primesIndex) {
+ mdInt.put(PRIMES_INDEX_OFFSET_I, primesIndex);
+ return this;
+ }
+
+ public int getPrimesIndex() {
+ return mdInt.get(PRIMES_INDEX_OFFSET_I);
+ }
+
+ public MetaData setLoadFactor(float loadFactor) {
+ mdLoadFactor.put(0, loadFactor);
+ return this;
+ }
+
+ public float getLoadFactor() {
+ return mdLoadFactor.get(0);
+ }
+ }
+
+ // debug
+ byte[] dump() throws IOException {
+ return ((RecordFileImpl)file).dump();
+ }
+}
Added: trunk/src/jar/util/java/org/mulgara/util/io/FixedLengthSerializable.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/FixedLengthSerializable.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/FixedLengthSerializable.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2010 Paul Gearon.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mulgara.util.io;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+
+/**
+ * A serializable object with a fixed length
+ * @author Paul Gearon
+ */
+public interface FixedLengthSerializable extends Serializable {
+
+ /** Returns the length of the data after serializing */
+ public int getSize();
+
+ /** Write the contents of this object into the buffer */
+ public ByteBuffer writeTo(ByteBuffer data);
+
+ /** Overwrite the contents of this object with the data in the buffer */
+ public FixedLengthSerializable readFrom(ByteBuffer data);
+}
Modified: trunk/src/jar/util/java/org/mulgara/util/io/IOUtil.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/IOUtil.java 2011-09-29 18:04:55 UTC (rev 2064)
+++ trunk/src/jar/util/java/org/mulgara/util/io/IOUtil.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -17,13 +17,79 @@
import java.io.BufferedReader;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.apache.log4j.Logger;
+
/**
* Static utility methods for common actions on IO.
*/
public class IOUtil {
+ /** The logger. */
+ private static final Logger logger = Logger.getLogger(IOUtil.class);
+
+ /** The system property for the byte order. */
+ public static final String BYTE_ORDER_PROPERTY = "mulgara.xa.useByteOrder";
+
+ /** The property for the block type. May be "direct" or "javaHeap" */
+ public static final String MEM_TYPE_PROP = "mulgara.xa.memoryType";
+
+ /** Native ordering of the bytes */
+ public static final ByteOrder NATIVE_ORDER = ByteOrder.nativeOrder();
+
+ /** The endianness of this computer. */
+ private static final ByteOrder byteOrder;
+
+ /** Enumeration of the different memory types for blocks */
+ public enum BlockMemoryType { DIRECT, HEAP };
+
+ /** The default value to use for the block memory type. Used when nothing is configured. */
+ private static final BlockMemoryType DEFAULT = BlockMemoryType.DIRECT;
+
+ /** The configured type of block type to use. */
+ private static final BlockMemoryType BLOCK_TYPE;
+
+ static {
+ // Determine the byte order of this machine, and select an ordering to use. *
+ String useByteOrderProp = System.getProperty(BYTE_ORDER_PROPERTY, "native");
+ ByteOrder bo = ByteOrder.nativeOrder();
+ if (useByteOrderProp != null) {
+ if (useByteOrderProp.equalsIgnoreCase("native")) {
+ bo = ByteOrder.nativeOrder();
+ } else if (useByteOrderProp.equalsIgnoreCase("big_endian")) {
+ bo = ByteOrder.BIG_ENDIAN;
+ } else if (useByteOrderProp.equalsIgnoreCase("little_endian")) {
+ bo = ByteOrder.LITTLE_ENDIAN;
+ } else {
+ logger.warn("Invalid value for property mulgara.xa.useByteOrder: " + useByteOrderProp);
+ }
+ }
+ byteOrder = bo;
+
+ // initialize the type of memory block to be used: heap or direct
+ // configured with mulgara.xa.memoryType
+ String defBlockType = System.getProperty(MEM_TYPE_PROP, DEFAULT.name());
+ if (defBlockType.equalsIgnoreCase(BlockMemoryType.DIRECT.name())) {
+ BLOCK_TYPE = BlockMemoryType.DIRECT;
+ } else if (defBlockType.equalsIgnoreCase(BlockMemoryType.HEAP.name())) {
+ BLOCK_TYPE = BlockMemoryType.HEAP;
+ } else {
+ logger.warn("Invalid value for property " + MEM_TYPE_PROP + ": " + defBlockType);
+ BLOCK_TYPE = DEFAULT;
+ }
+ }
+
/**
+ * Retrieves the configured byte ordering to use. Uses the default if nothing is set.
+ * @return The configured byte order.
+ */
+ public static final ByteOrder getByteOrder() {
+ return byteOrder;
+ }
+
+ /**
* Reads the next non-empty line of text from a buffered reader, up to a given
* number of characters.
*
@@ -51,4 +117,26 @@
}
return s.toString();
}
+
+
+ /**
+ * Allocates data according to the system configured memory model.
+ * @param size The number of bytes in the allocated buffer.
+ * @return the allocated buffer.
+ */
+ public static final ByteBuffer allocate(int size) {
+ return allocate(size, byteOrder);
+ }
+
+ /**
+ * Allocates data according to the system configured memory model.
+ * @param size The number of bytes in the allocated buffer.
+ * @param order The byteorder to use in the buffer.
+ * @return the allocated buffer.
+ */
+ public static final ByteBuffer allocate(int size, ByteOrder order) {
+ return BLOCK_TYPE == BlockMemoryType.DIRECT ?
+ (order == NATIVE_ORDER ? ByteBuffer.allocateDirect(size) : ByteBuffer.allocateDirect(size).order(order)) :
+ (order == NATIVE_ORDER ? ByteBuffer.allocate(size) : ByteBuffer.allocate(size).order(order));
+ }
}
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFile.java 2011-09-29 18:04:55 UTC (rev 2064)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFile.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -16,10 +16,10 @@
package org.mulgara.util.io;
+import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
import org.apache.log4j.Logger;
@@ -32,42 +32,15 @@
private final static Logger logger = Logger.getLogger(LBufferedFile.class);
- /** The property for the block type. May be "direct" or "javaHeap" */
- public static final String MEM_TYPE_PROP = "mulgara.xa.memoryType";
-
/** The property for the io type. May be "mapped" or "explicit" */
public static final String IO_TYPE_PROP = "mulgara.xa.forceIOType";
- /** Enumeration of the different memory types for blocks */
- enum BlockMemoryType { DIRECT, HEAP };
-
/** Enumeration of the different io types */
enum IOType { MAPPED, EXPLICIT };
- /** Native ordering of the bytes */
- ByteOrder NATIVE_ORDER = ByteOrder.nativeOrder();
-
- /** The default value to use for the block memory type. Used when nothing is configured. */
- private static final BlockMemoryType DEFAULT = BlockMemoryType.DIRECT;
-
- /** The configured type of block type to use. */
- private static final BlockMemoryType BLOCK_TYPE;
-
private static final IOType ioType;
static {
- // initialize the type of memory block to be used: heap or direct
- // configured with mulgara.xa.memoryType
- String defBlockType = System.getProperty(MEM_TYPE_PROP, DEFAULT.name());
- if (defBlockType.equalsIgnoreCase(BlockMemoryType.DIRECT.name())) {
- BLOCK_TYPE = BlockMemoryType.DIRECT;
- } else if (defBlockType.equalsIgnoreCase(BlockMemoryType.HEAP.name())) {
- BLOCK_TYPE = BlockMemoryType.HEAP;
- } else {
- logger.warn("Invalid value for property " + MEM_TYPE_PROP + ": " + defBlockType);
- BLOCK_TYPE = DEFAULT;
- }
-
// initialize the type of file access to use: memory mapped files, or explicit read/write operations
// configured with mulgara.xa.forceIOType
String forceIOTypeProp = System.getProperty(IO_TYPE_PROP, "mapped");
@@ -158,25 +131,25 @@
*/
public abstract void registerRemapListener(Runnable l);
- /**
- * Create a buffered file using a filename.
- * @param fileName The name of the file to provide buffered access to.
- * @return The readonly buffered file.
- * @throws IOException An error opening the file.
- */
- public static LBufferedFile createReadOnlyLBufferedFile(String fileName) throws IOException {
- return createReadOnlyLBufferedFile(new RandomAccessFile(fileName, "r"));
+ static LBufferedFile createWritable(RandomAccessFile f, int recordSize) throws IOException {
+ if (ioType == IOType.MAPPED) {
+ return new LMappedBufferedFileRW(f, recordSize);
+ } else if (ioType == IOType.EXPLICIT) {
+ return new LIOBufferedFile(f);
+ } else {
+ throw new IllegalArgumentException("Invalid BlockFile ioType.");
+ }
}
/**
* Create a buffered file, taking over the RandomAccessFile that is provided.
* @param f The file to provide buffered access to.
- * @return The readonly buffered file.
+ * @return The buffered file.
* @throws IOException An error opening the file.
*/
- public static LBufferedFile createReadOnlyLBufferedFile(RandomAccessFile f) throws IOException {
+ static LBufferedFile createReadOnly(RandomAccessFile f) throws IOException {
if (ioType == IOType.MAPPED) {
- return new LMappedBufferedFile(f);
+ return new LMappedBufferedFileRO(f);
} else if (ioType == IOType.EXPLICIT) {
return new LReadOnlyIOBufferedFile(f);
} else {
@@ -185,23 +158,45 @@
}
/**
- * Allocates data according to the system configured memory model.
- * @param size The number of bytes in the allocated buffer.
- * @return the allocated buffer.
+ * Create a buffered file using a filename.
+ * @param fileName The name of the file to provide buffered access to.
+ * @return The readonly buffered file.
+ * @throws IOException An error opening the file.
*/
- protected ByteBuffer allocate(int size) {
- return allocate(size, NATIVE_ORDER);
+ public static LBufferedFile createWritable(String fileName, int recordSize) throws IOException {
+ return createWritable(new RandomAccessFile(fileName, "rw"), recordSize);
}
/**
- * Allocates data according to the system configured memory model.
- * @param size The number of bytes in the allocated buffer.
- * @param order The byteorder to use in the buffer.
- * @return the allocated buffer.
+ * Create a buffered file using a filename.
+ * @param fileName The file to provide buffered access to.
+ * @return The readonly buffered file.
+ * @throws IOException An error opening the file.
*/
- protected ByteBuffer allocate(int size, ByteOrder order) {
- return BLOCK_TYPE == BlockMemoryType.DIRECT ?
- ByteBuffer.allocateDirect(size).order(order) :
- ByteBuffer.allocate(size).order(order);
+ public static LBufferedFile createWritable(File file, int recordSize) throws IOException {
+ return createWritable(new RandomAccessFile(file, "rw"), recordSize);
}
+
+ /**
+ * Create a buffered file using a filename.
+ * @param fileName The name of the file to provide buffered access to.
+ * @return The readonly buffered file.
+ * @throws IOException An error opening the file.
+ */
+ public static LBufferedFile createReadOnly(String fileName) throws IOException {
+ return createReadOnly(new RandomAccessFile(fileName, "r"));
+ }
+
+ /**
+ * Create a buffered file using a filename.
+ * @param file The file to provide buffered access to.
+ * @return The readonly buffered file.
+ * @throws IOException An error opening the file.
+ */
+ public static LBufferedFile createReadOnly(File file) throws IOException {
+ return createReadOnly(new RandomAccessFile(file, "r"));
+ }
+
+ // debug
+ abstract byte[] dump() throws IOException;
}
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFileTest.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFileTest.java 2011-09-29 18:04:55 UTC (rev 2064)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LBufferedFileTest.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -43,7 +43,7 @@
for (byte i = 0; i < data.length; i++) data[i] = i;
raf.write(data);
- System.setProperty(LMappedBufferedFile.PAGE_SIZE_PROP, "4");
+ System.setProperty(LMappedBufferedFileRO.PAGE_SIZE_PROP, "4");
braf = new RandomAccessFile(f, "r");
file = newFile(braf);
}
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LIOBufferedFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LIOBufferedFile.java 2011-09-29 18:04:55 UTC (rev 2064)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LIOBufferedFile.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -40,7 +40,7 @@
@Override
public ByteBuffer read(long offset, int length) throws IOException {
- ByteBuffer data = allocate(length);
+ ByteBuffer data = IOUtil.allocate(length);
synchronized (file) {
file.seek(offset);
file.readFully(data.array());
@@ -50,7 +50,7 @@
@Override
public ByteBuffer allocate(long offset, int length) {
- ByteBuffer data = allocate(length);
+ ByteBuffer data = IOUtil.allocate(length);
offsets.put(new SystemBuffer(data), offset);
return data;
}
@@ -96,4 +96,13 @@
public int hashCode() { return System.identityHashCode(buffer); }
}
+ // debug
+ byte[] dump() throws IOException {
+ byte[] d = new byte[(int)file.length()];
+ long offset = file.getFilePointer();
+ file.seek(0);
+ file.read(d);
+ file.seek(offset);
+ return d;
+ }
}
Added: trunk/src/jar/util/java/org/mulgara/util/io/LLHashMap.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LLHashMap.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LLHashMap.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,261 @@
+/*
+ * Copyright 2011 Paul Gearon.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mulgara.util.io;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A file-based hash map mapping long to long
+ */
+public class LLHashMap implements Map<Long,Long> {
+
+ /** The number of bytes in a LONG */
+ private static final int LONG_BYTES = Long.SIZE >> 3;
+
+ /** The internal map that maps ByteBuffer to ByteBuffer */
+ private final FileHashMap map;
+
+ /**
+ * Creates a hash map for Long to Long.
+ * @param f The file to put the table into.
+ * @throws IOException If there is an error accessing the file.
+ */
+ public LLHashMap(File f) throws IOException {
+ map = new FileHashMap(f, LONG_BYTES, LONG_BYTES);
+ }
+
+ /**
+ * Creates a hash map for Long to Long.
+ * @param f The file to put the table into.
+ * @param loadFactor The fraction of the table above with the table will get expanded.
+ * @param initialSize The initial number of slots available to put data into before being rehashed.
+ * @throws IOException If there is an error accessing the file.
+ */
+ public LLHashMap(File f, float loadFactor, long initialSize) throws IOException {
+ map = new FileHashMap(f, LONG_BYTES, LONG_BYTES, loadFactor, initialSize);
+ }
+
+ /**
+ * @see java.io.Closeable#close()
+ */
+ public void close() throws IOException {
+ map.close();
+ }
+
+ /**
+ * Closes the object and removes it from the filesystem.
+ * @throws IOException If there is an error closing or removing the files.
+ */
+ public void closeAndDelete() throws IOException {
+ map.closeAndDelete();
+ }
+
+ @Override
+ public void clear() {
+ map.clear();
+ }
+
+ @Override
+ public boolean containsKey(Object key) {
+ return map.containsKey(toBytes(key));
+ }
+
+ public boolean containsKey(long key) {
+ return map.containsKey(toBytes(key));
+ }
+
+ @Override
+ public boolean containsValue(Object value) {
+ return map.containsValue(toBytes(value));
+ }
+
+ public boolean containsValue(long value) {
+ return map.containsValue(toBytes(value));
+ }
+
+ @Override
+ public Set<Map.Entry<Long, Long>> entrySet() {
+ map.entrySet();
+ return new ArrayBufferSetWrapper<Map.Entry<Long,Long>,Map.Entry<ByteBuffer,ByteBuffer>>(map.entrySet(),
+ new SetDataConverter<Map.Entry<Long,Long>, Map.Entry<ByteBuffer,ByteBuffer>>() {
+ public Map.Entry<ByteBuffer,ByteBuffer> toSetData(Map.Entry<Long,Long> d) {
+ return new BBKeyValue(toBytes(d.getKey()), toBytes(d.getValue()));
+ }
+ public Map.Entry<Long,Long> fromSetData(Map.Entry<ByteBuffer,ByteBuffer> b) {
+ return new KeyValue<Long,Long>(fromBytes(b.getKey()), fromBytes(b.getValue()));
+ }
+ });
+ }
+
+ @Override
+ public Long get(Object key) {
+ ByteBuffer b = map.get(toBytes(key));
+ return b == null ? null : fromBytes(b);
+ }
+
+ public long get(long key) {
+ return fromBytes(map.get(toBytes(key)));
+ }
+
+ @Override
+ public boolean isEmpty() {
+ return map.isEmpty();
+ }
+
+ @Override
+ public Set<Long> keySet() {
+ return new ArrayBufferSetWrapper<Long,ByteBuffer>(map.keySet(),
+ new SetDataConverter<Long, ByteBuffer>() {
+ public ByteBuffer toSetData(Long d) { return toBytes(d); }
+ public Long fromSetData(ByteBuffer b) { return fromBytes(b); }
+ });
+ }
+
+ @Override
+ public Long put(Long key, Long value) {
+ ByteBuffer[] pair = toBytesPair(key, value);
+ return fromBytes(map.put(pair[0], pair[1]));
+ }
+
+ public long put(long key, long value) {
+ ByteBuffer[] pair = toBytesPair(key, value);
+ return fromBytes(map.put(pair[0], pair[1]));
+ }
+
+ @Override
+ public void putAll(Map<? extends Long, ? extends Long> m) {
+ for (java.util.Map.Entry<? extends Long, ? extends Long> e: m.entrySet()) {
+ put(e.getKey(), e.getValue());
+ }
+ }
+
+ @Override
+ public Long remove(Object key) {
+ return fromBytes(map.remove(toBytes(key)));
+ }
+
+ public long remove(long key) {
+ return fromBytes(map.remove(toBytes(key)));
+ }
+
+ @Override
+ public int size() {
+ return map.size();
+ }
+
+ @Override
+ public Collection<Long> values() {
+ return new ArrayBufferSetWrapper<Long,ByteBuffer>((Set<ByteBuffer>)map.values(),
+ new SetDataConverter<Long, ByteBuffer>() {
+ public ByteBuffer toSetData(Long d) { return toBytes(d); }
+ public Long fromSetData(ByteBuffer b) { return fromBytes(b); }
+ });
+ }
+
+ /**
+ * Convenience method to avoid casting Object that will then be sent
+ * for auto unboxing.
+ * @param l A Long value as an Object.
+ * @return A new {@link ByteBuffer} containing the number. Allocated according to the type
+ * configured with {@link IOUtil#MEM_TYPE_PROP}
+ */
+ private static final ByteBuffer toBytes(Object l) {
+ ByteBuffer b = IOUtil.allocate(LONG_BYTES);
+ b.asLongBuffer().put(0, ((Long)l).longValue());
+ return b;
+ }
+
+ /**
+ * Converts a pair of long numbers into a pair of ByteBuffers.
+ * @param first The first number to convert.
+ * @param second The second number to convert.
+ * @return An array containing the new {@link ByteBuffer}s containing the numbers.
+ * Allocated according to the type configured with {@link IOUtil#MEM_TYPE_PROP}
+ */
+ private static final ByteBuffer[] toBytesPair(long first, long second) {
+ ByteBuffer[] pair = new ByteBuffer[2];
+ ByteBuffer b = toBytes(first, second);
+ b.position(LONG_BYTES);
+ pair[1] = b.slice();
+ b.flip();
+ pair[0] = b.slice();
+ return pair;
+ }
+
+ /**
+ * Converts a pair of long numbers into a ByteBuffer.
+ * @param first The first number to convert.
+ * @param second The second number to convert.
+ * @return A {@link ByteBuffer}s containing the numbers.
+ * Allocated according to the type configured with {@link IOUtil#MEM_TYPE_PROP}
+ */
+ private static final ByteBuffer toBytes(long first, long second) {
+ ByteBuffer b = IOUtil.allocate(LONG_BYTES << 1);
+ b.asLongBuffer().put(0, first).put(1, second);
+ return b;
+ }
+
+ /**
+ * Converts a ByteBuffer into a long value.
+ * @param bb The buffer to convert.
+ * @return The long value from the buffer.
+ */
+ private static final long fromBytes(ByteBuffer bb) {
+ return bb == null ? -1L : bb.asLongBuffer().get(0);
+ }
+
+ /** A {@link Map.Entry} implementation for use with {@link FileHashMap#entrySet()}. */
+ public static class KeyValue<K,V> implements Map.Entry<K,V> {
+ private final K key;
+ protected V value;
+ public KeyValue(K k, V v) { key = k; value = v; }
+ public K getKey() { return key; }
+ public V getValue() { return value; }
+ public V setValue(V v) {
+ V old = value;
+ value = v;
+ return old;
+ }
+ }
+
+ /** A {@link Map.Entry} implementation for use with {@link FileHashMap#entrySet()}. */
+ public static class BBKeyValue extends KeyValue<ByteBuffer,ByteBuffer> {
+ public BBKeyValue(ByteBuffer k, ByteBuffer v) { super(k, v); }
+ public ByteBuffer setValue(ByteBuffer v) {
+ int size = value == null ? v.capacity() : value.capacity();
+ byte[] old = new byte[size];
+ value.position(0);
+ value.limit(size);
+ value.get(old);
+ value.position(0);
+ v.position(0);
+ v.limit(size);
+ value.put(v);
+ return ByteBuffer.wrap(old);
+ }
+ }
+
+
+ // debug
+ byte[] dump() throws IOException {
+ return map.dump();
+ }
+}
Added: trunk/src/jar/util/java/org/mulgara/util/io/LLHashMapUnitTest.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LLHashMapUnitTest.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LLHashMapUnitTest.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,300 @@
+package org.mulgara.util.io;
+
+import java.io.File;
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+public class LLHashMapUnitTest extends TestCase {
+
+ File f;
+ LLHashMap map;
+
+ protected void setUp() throws Exception {
+ super.setUp();
+ f = new File("/tmp/i2i");
+ clean(f);
+ map = new LLHashMap(f);
+ map.clear();
+ }
+
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ map.clear();
+ map.close();
+ clean(f);
+ }
+
+ public void testLLHashMapFile() throws IOException {
+ File f = new File("/tmp/int2int");
+ clean(f);
+ LLHashMap map = new LLHashMap(f);
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ assertEquals(11, map.get(1));
+ assertEquals(12, map.get(2));
+ assertEquals(13, map.get(3));
+ assertEquals(-1L, map.get(4));
+ assertEquals(null, map.get(Long.valueOf(4)));
+ map.close();
+ assertEquals(37 * 16, f.length());
+ clean(f);
+ }
+
+ public void testLLHashMapFileFloatLong() throws IOException {
+ File f = new File("/tmp/int2intl");
+ clean(f);
+ LLHashMap map = new LLHashMap(f, 0.5f, 50L);
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ assertEquals(13, map.get(3));
+ assertEquals(12, map.get(2));
+ assertEquals(11, map.get(1));
+ assertEquals(-1L, map.get(4));
+ assertEquals(null, map.get(Long.valueOf(4)));
+ map.close();
+ assertEquals(131 * 16, f.length());
+ clean(f);
+ }
+
+ public void testClear() {
+ assertEquals(0, map.size());
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ assertEquals(3, map.size());
+ assertEquals(13, map.get(3));
+ assertEquals(12, map.get(2));
+ assertEquals(11, map.get(1));
+ assertEquals(-1L, map.get(4));
+ assertEquals(null, map.get(Long.valueOf(4)));
+ map.clear();
+ assertEquals(0, map.size());
+ assertEquals(-1L, map.get(3));
+ assertEquals(-1L, map.get(2));
+ assertEquals(-1L, map.get(1));
+ assertEquals(-1L, map.get(4));
+ assertEquals(null, map.get(Long.valueOf(4)));
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ assertEquals(3, map.size());
+ assertEquals(13, map.get(3));
+ assertEquals(12, map.get(2));
+ assertEquals(11, map.get(1));
+ assertEquals(-1L, map.get(4));
+ assertEquals(null, map.get(Long.valueOf(4)));
+ }
+
+ public void testContainsKeyObject() {
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ assertTrue(map.containsKey(Long.valueOf(1)));
+ assertTrue(map.containsKey(Long.valueOf(2)));
+ assertTrue(map.containsKey(Long.valueOf(3)));
+ assertFalse(map.containsKey(Long.valueOf(4)));
+ assertFalse(map.containsKey(Long.valueOf(0)));
+ }
+
+ public void testContainsKeyLong() {
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ assertTrue(map.containsKey(1));
+ assertTrue(map.containsKey(2));
+ assertTrue(map.containsKey(3));
+ assertFalse(map.containsKey(4));
+ assertFalse(map.containsKey(0));
+ }
+
+ public void testContainsValueObject() {
+ assertFalse(map.containsValue(Long.valueOf(11)));
+ assertFalse(map.containsValue(Long.valueOf(0)));
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ assertTrue(map.containsValue(Long.valueOf(11)));
+ assertFalse(map.containsValue(Long.valueOf(0)));
+ }
+
+ public void testContainsValueLong() {
+ assertFalse(map.containsValue(11));
+ assertFalse(map.containsValue(0));
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ assertTrue(map.containsValue(11));
+ assertFalse(map.containsValue(0));
+ }
+
+ public void testEntrySet() {
+// fail("Not yet implemented");
+ }
+
+ public void testGetObject() {
+// fail("Not yet implemented");
+ }
+
+ public void testGetLong() {
+ assertEquals(-1, map.get(1));
+ assertEquals(-1, map.get(11));
+ assertEquals(-1, map.get(0));
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ assertEquals(11, map.get(1));
+ assertEquals(-1, map.get(11));
+ assertEquals(-1, map.get(0));
+ assertEquals(12, map.get(2));
+ assertEquals(13, map.get(3));
+ }
+
+ public void testIsEmpty() {
+ assertTrue(map.isEmpty());
+ map.put(1, 11);
+ assertFalse(map.isEmpty());
+ map.put(2, 12);
+ assertFalse(map.isEmpty());
+ map.put(3, 13);
+ assertFalse(map.isEmpty());
+ map.clear();
+ assertTrue(map.isEmpty());
+ }
+
+ public void testKeySet() {
+// fail("Not yet implemented");
+ }
+
+ public void testPutLongLong() throws Exception {
+ map.put(1, 11);
+ map.put(2, 12);
+ map.put(3, 13);
+ map.put(0, 10);
+ assertEquals(4, map.size());
+ assertEquals(11, map.get(1));
+ assertEquals(12, map.get(2));
+ assertEquals(13, map.get(3));
+ assertEquals(10, map.get(0));
+ assertEquals(-1L, map.get(4));
+ assertEquals(null, map.get(Long.valueOf(4)));
+ }
+
+ public void testRehash() throws Exception {
+ File f = new File("/tmp/int2int_small");
+ clean(f);
+ LLHashMap map = new LLHashMap(f);
+ for (long i = 0L; i < 27L; i++) {
+ map.put(i, i * i);
+ }
+ assertEquals(27, map.size());
+ for (long i = 0L; i < 27L; i++) {
+ assertEquals(i * i, map.get(i));
+ }
+ map.close();
+ assertEquals(37 * 16, f.length());
+ clean(f);
+ }
+
+ public void testHighLoad() throws Exception {
+ File f = new File("/tmp/int2int_big");
+ clean(f);
+ LLHashMap map = new LLHashMap(f);
+ long i = 0;
+ try {
+ for (i = 0L; i < 10000L; i++) {
+ map.put(i, i * i);
+ }
+ } catch (Exception ex) {
+ System.err.println("Exception on: " + i);
+ throw ex;
+ } catch (Error e) {
+ System.err.println("Error on: " + i);
+ throw e;
+ }
+ assertEquals(10000L, map.size());
+ for (i = 0L; i < 10000L; i++) {
+ assertEquals(i * i, map.get(i));
+ }
+ map.close();
+ assertEquals(16411L * 16, f.length());
+ clean(f);
+ }
+
+ public void testExpHighLoad() throws Exception {
+ File f = new File("/tmp/int2int_big2");
+ clean(f);
+ LLHashMap map = new LLHashMap(f, 0.75f, 10000);
+ long i = 0;
+ try {
+ for (i = 0L; i < 10000L; i++) {
+ map.put(i, i * i);
+ }
+ } catch (Exception ex) {
+ System.err.println("Exception on: " + i);
+ throw ex;
+ } catch (Error e) {
+ System.err.println("Error on: " + i);
+ throw e;
+ }
+ assertEquals(10000L, map.size());
+ for (i = 0L; i < 10000L; i++) {
+ assertEquals(i * i, map.get(i));
+ }
+ map.close();
+ assertEquals(16411L * 16, f.length());
+ clean(f);
+ }
+
+ public void testPutLongLong1() {
+// fail("Not yet implemented");
+ }
+
+ public void testPutAll() {
+// fail("Not yet implemented");
+ }
+
+ public void testRemoveObject() {
+// fail("Not yet implemented");
+ }
+
+ public void testRemoveLong() {
+// fail("Not yet implemented");
+ }
+
+ public void testSize() {
+ assertEquals(0, map.size());
+ map.put(1, 11);
+ assertEquals(1, map.size());
+ map.put(2, 12);
+ assertEquals(2, map.size());
+ map.put(3, 13);
+ assertEquals(3, map.size());
+ map.put(3, 14);
+ assertEquals(3, map.size());
+ map.remove(1);
+ assertEquals(2, map.size());
+ map.remove(1);
+ assertEquals(2, map.size());
+ map.remove(2);
+ assertEquals(1, map.size());
+ map.remove(3);
+ assertEquals(0, map.size());
+ map.remove(3);
+ assertEquals(0, map.size());
+ }
+
+ public void testValues() {
+// fail("Not yet implemented");
+ }
+
+ static void clean(File file) throws IOException {
+ if (file.exists()) {
+ file.delete();
+ File md = new File(file.getPath() + ".hmd");
+ if (md.exists()) md.delete();
+ }
+ }
+}
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java 2011-09-29 18:04:55 UTC (rev 2064)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFile.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2011 Paul Gearon
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.mulgara.util.io;
import java.io.IOException;
@@ -11,106 +27,63 @@
import org.apache.log4j.Logger;
/**
- * A memory mapped read-only version of LBufferedFile
- * @author pag
+ * Represents a memory mapped file.
*/
-public class LMappedBufferedFile extends LBufferedFile {
+public abstract class LMappedBufferedFile extends LBufferedFile {
/** Logger. */
- private static final Logger logger = Logger.getLogger(LMappedBufferedFile.class);
+ static final Logger logger = Logger.getLogger(LMappedBufferedFile.class);
/** The name of the property for setting the page size */
- static final String PAGE_SIZE_PROP = "org.mulgara.io.pagesize";
+ protected static final String PAGE_SIZE_PROP = "org.mulgara.io.pagesize";
- /** The size of a page to be mapped */
- private static final int DEFAULT_PAGE_SIZE = 33554432; // 32 MB
-
- /** The page size to use. */
- public static final int PAGE_SIZE;
-
/** The objects that want to know when the file gets remapped */
- private List<Runnable> listeners = new ArrayList<Runnable>();
-
- static {
- String pageSizeStr = System.getProperty(PAGE_SIZE_PROP);
- int tmp = DEFAULT_PAGE_SIZE;
- try {
- if (pageSizeStr != null) tmp = Integer.parseInt(pageSizeStr);
- } catch (NumberFormatException e) {
- logger.warn("Property [" + PAGE_SIZE_PROP + "] is not a number [" + pageSizeStr + "]. Using default: " + tmp);
- }
- PAGE_SIZE = tmp;
- }
-
+ protected List<Runnable> listeners = new ArrayList<Runnable>();
+
/** The channel to the file being accessed */
- private FileChannel fc;
+ protected FileChannel fc;
/** All the pages of mapped buffers */
- private MappedByteBuffer[] buffers;
+ protected MappedByteBuffer[] buffers;
/**
- * Create a new buffered file for Long seeks.
- * @param file The file to buffer.
- * @throws IOException There was an error setting up initial mapping of the file.
+ * Create a mapped file.
+ * @param file The open file to be accessed by mapping.
*/
- public LMappedBufferedFile(RandomAccessFile file) throws IOException {
+ public LMappedBufferedFile(RandomAccessFile file) {
super(file);
- fc = file.getChannel();
- mapFile();
- if (logger.isDebugEnabled()) logger.debug("Mapping files with pages of: " + PAGE_SIZE);
}
+ /**
+ * Gets a buffer that reflects the actual contents within a file.
+ * @see org.mulgara.util.io.LBufferedFile#read(long, int)
+ */
@Override
- public ByteBuffer read(long offset, int length) throws IOException {
- // get the offset into the last buffer, this may be negative if before the last page
- long lastPageOffset = (offset + length) - (PAGE_SIZE * (buffers.length - 1));
- // if the offset is larger than the final page, then remap
- if (lastPageOffset > PAGE_SIZE || lastPageOffset > buffers[buffers.length - 1].limit()) {
- mapFile();
- }
- int page = (int)(offset / PAGE_SIZE);
- int page_offset = (int)(offset % PAGE_SIZE);
- if (page_offset + length <= PAGE_SIZE) {
- ByteBuffer bb = buffers[page].asReadOnlyBuffer();
- bb.position(page_offset);
- bb.limit(page_offset + length);
- if (page == buffers.length - 1) {
- // In the final page, so make a copy
- // This is needed in case of rollback, because the final page
- // has the potential of being needed by a read-only transaction
- // even when we need to remove that page to truncate the file.
- // We have not synched on this page, but the GC will retry removing it
- // with short delays, so it should get picked up despite our brief use.
- ByteBuffer data = ByteBuffer.allocate(length);
- bb.get(data.array(), 0, length);
- return data;
- } else {
- // normal return of the mapped region slice
- return bb.slice();
- }
- } else {
- ByteBuffer data = ByteBuffer.allocate(length);
- byte[] dataArray = data.array();
- ByteBuffer tmp = buffers[page].asReadOnlyBuffer();
- tmp.position(page_offset);
- int firstSlice = PAGE_SIZE - page_offset;
- tmp.get(dataArray, 0, firstSlice);
- tmp = buffers[page + 1].asReadOnlyBuffer();
- tmp.get(dataArray, firstSlice, length - firstSlice);
- return data;
- }
- }
+ abstract public ByteBuffer read(long offset, int length) throws IOException;
+ /**
+ * Gets a buffer that reflects the actual contents within a file without trying to read it.
+ * Identical to {@link #read(long, int)} for a mapped file.
+ * @see org.mulgara.util.io.LBufferedFile#read(long, int)
+ */
@Override
public ByteBuffer allocate(long offset, int length) throws IOException {
return read(offset, length);
}
+ /**
+ * Pushes the buffer data to disk
+ * @see org.mulgara.util.io.LBufferedFile#write(java.nio.ByteBuffer)
+ */
@Override
public void write(ByteBuffer data) throws IOException {
// no-op, even if read/write
}
+ /**
+ * Controls where in a file to read or write a buffer. Irrelevant for mapped buffers.
+ * @see org.mulgara.util.io.LBufferedFile#seek(long)
+ */
@Override
public void seek(long offset) throws IOException {
// no-op
@@ -134,26 +107,27 @@
for (Runnable listener: listeners) listener.run();
return;
}
-
+
// if truncating to an address above the mapping, then return
int fullBuffers = buffers.length - 1;
- if (fullBuffers >= 0 && size >= fullBuffers * PAGE_SIZE + buffers[fullBuffers].limit()) return;
-
+ int pageSize = getPageSize();
+ if (fullBuffers >= 0 && size >= fullBuffers * pageSize + buffers[fullBuffers].limit()) return;
+
// get all the pages, including a possible partial page
- int pages = (int)((size + PAGE_SIZE - 1) / PAGE_SIZE);
+ int pages = (int)((size + pageSize - 1) / pageSize);
// get all the full pages. Either pages or pages-1
- int fullPages = (int)(size / PAGE_SIZE);
-
+ int fullPages = (int)(size / pageSize);
+
if (logger.isDebugEnabled()) logger.debug("Existing file holds " + buffers.length + " pages. Truncated file needs " + pages + " pages (" + fullPages + " full pages)");
// if the data is fully mapped then there is nothing to do
if (fullPages == buffers.length) return;
-
+
// check that this really is a truncation
assert pages <= buffers.length;
-
+
// This will be the set of buffers to use in the end
MappedByteBuffer[] newBuffers;
-
+
if (pages == buffers.length) {
// can't be on a page boundary, else that would mean there was no truncation
// since it would either be truncated to the same size, or larger.
@@ -171,57 +145,49 @@
System.arraycopy(buffers, 0, newBuffers, 0, fullPages);
if (logger.isDebugEnabled()) logger.debug("dropped " + (buffers.length - fullPages) + " pages, saved " + fullPages);
}
-
+
// if there's a partial page at the end, then map it
if (fullPages < pages) {
assert fullPages == pages - 1;
- newBuffers[fullPages] = fc.map(FileChannel.MapMode.READ_ONLY, fullPages * PAGE_SIZE, size % PAGE_SIZE);
+ newBuffers[fullPages] = fc.map(getMode(), fullPages * pageSize, size % pageSize);
if (logger.isDebugEnabled()) logger.debug("Remapped final partial page");
}
-
+
// switch over from the previous buffers to the new ones
// anyone reading from anything but the partial buffer at the end is
// accessing the wrong transaction.
MappedByteBuffer[] tmpBuffers = buffers;
buffers = newBuffers;
-
+
// We're about to lose the last reference to these buffers when tmpBuffers
// goes away, but by putting the following code here anyone looking can see
// which buffers we're trying to eliminate.
// Note that this will remove any partial buffer at the end, even if most
// of it has been remapped into a new partial buffer.
-
+
// Setting these buffers to null is a belt and suspenders approach.
// This code is informative, rather than necessary
if (tmpBuffers != buffers) {
for (int b = fullPages; b < tmpBuffers.length; b++) tmpBuffers[b] = null;
}
if (logger.isDebugEnabled()) logger.debug("Removed " + (tmpBuffers.length - fullPages) + " pages");
-
+
// tell the listeners that we've remapped
for (Runnable listener: listeners) listener.run();
}
- @Override
- public void registerRemapListener(Runnable l) {
- listeners.add(l);
- }
-
- @Override
- public int getPageSize() {
- return PAGE_SIZE;
- }
-
/**
- * Map the entire file
- * @throws IOException If there is an error mapping the file
+ * Map the entire file, using the given page size.
+ * @param pageSize The size of the pages when mapping.
+ * @throws IOException Due to an error in file access.
*/
synchronized void mapFile() throws IOException {
+ int pageSize = getPageSize();
long size = fc.size();
// get all the pages, including a possible partial page
- int pages = (int)((size + PAGE_SIZE - 1) / PAGE_SIZE);
+ int pages = (int)((size + pageSize - 1) / pageSize);
// get all the full pages. Either pages or pages-1
- int fullPages = (int)(size / PAGE_SIZE);
+ int fullPages = (int)(size / pageSize);
// create a larger buffers array, with all the original buffers in it
// except any partial pages
@@ -229,7 +195,7 @@
int start = 0;
if (buffers != null) {
int topBuffer = buffers.length - 1;
- if (topBuffer == -1 || buffers[topBuffer].limit() == PAGE_SIZE) {
+ if (topBuffer == -1 || buffers[topBuffer].limit() == pageSize) {
// last buffer full
topBuffer++;
} else {
@@ -241,11 +207,12 @@
}
// fill in the rest of the new array
+ FileChannel.MapMode mode = getMode();
for (int page = start; page < fullPages; page++) {
- newBuffers[page] = fc.map(FileChannel.MapMode.READ_ONLY, page * PAGE_SIZE, PAGE_SIZE);
+ newBuffers[page] = fc.map(mode, page * pageSize, pageSize);
}
// if there's a partial page at the end, then map it
- if (fullPages < pages) newBuffers[fullPages] = fc.map(FileChannel.MapMode.READ_ONLY, fullPages * PAGE_SIZE, size % PAGE_SIZE);
+ if (fullPages < pages) newBuffers[fullPages] = fc.map(mode, fullPages * pageSize, size % pageSize);
buffers = newBuffers;
@@ -253,4 +220,25 @@
for (Runnable listener: listeners) listener.run();
}
-}
+ /**
+ * Registers a listener for the remap event.
+ * @see org.mulgara.util.io.LBufferedFile#registerRemapListener(java.lang.Runnable)
+ */
+ @Override
+ public void registerRemapListener(Runnable l) {
+ listeners.add(l);
+ }
+
+ /**
+ * Returns the size of each block of mapped data.
+ * @see org.mulgara.util.io.LBufferedFile#getPageSize()
+ */
+ @Override
+ public abstract int getPageSize();
+
+ /**
+ * Describes the read/write mode of the file.
+ * @return either {@link FileChannel.MapMode#READ_WRITE} or {@link FileChannel.MapMode#READ_ONLY}.
+ */
+ abstract FileChannel.MapMode getMode();
+}
\ No newline at end of file
Added: trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileRO.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileRO.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileRO.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2011 Paul Gearon
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.mulgara.util.io;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
+import org.apache.log4j.Logger;
+
+/**
+ * A memory mapped read-only version of LBufferedFile
+ * @author pag
+ */
+public class LMappedBufferedFileRO extends LMappedBufferedFile {
+
+ /** Logger. */
+ static final Logger logger = Logger.getLogger(LMappedBufferedFileRO.class);
+
+ /** The page size to use. */
+ public static final int PAGE_SIZE;
+
+ /** The size of a page to be mapped */
+ private static final int DEFAULT_PAGE_SIZE = 33554432; // 32 MB
+
+ static {
+ String pageSizeStr = System.getProperty(PAGE_SIZE_PROP);
+ int tmp = DEFAULT_PAGE_SIZE;
+ try {
+ if (pageSizeStr != null) tmp = Integer.parseInt(pageSizeStr);
+ } catch (NumberFormatException e) {
+ logger.warn("Property [" + PAGE_SIZE_PROP + "] is not a number [" + pageSizeStr + "]. Using default: " + tmp);
+ }
+ PAGE_SIZE = tmp;
+ }
+
+ /**
+ * Create a new buffered file for Long seeks.
+ * @param file The file to buffer.
+ * @throws IOException There was an error setting up initial mapping of the file.
+ */
+ public LMappedBufferedFileRO(RandomAccessFile file) throws IOException {
+ super(file);
+ fc = file.getChannel();
+ mapFile();
+ if (logger.isDebugEnabled()) logger.debug("Mapping files with pages of: " + PAGE_SIZE);
+ }
+
+ @Override
+ public int getPageSize() {
+ return PAGE_SIZE;
+ }
+
+ @Override
+ public ByteBuffer read(long offset, int length) throws IOException {
+ // get the offset into the last buffer, this may be negative if before the last page
+ long lastPageOffset = (offset + length) - (PAGE_SIZE * (buffers.length - 1));
+ // if the offset is larger than the final page, then remap
+ if (lastPageOffset > PAGE_SIZE || lastPageOffset > buffers[buffers.length - 1].limit()) {
+ mapFile();
+ }
+ int page = (int)(offset / PAGE_SIZE);
+ int page_offset = (int)(offset % PAGE_SIZE);
+ if (page_offset + length <= PAGE_SIZE) {
+ ByteBuffer bb = buffers[page].asReadOnlyBuffer();
+ bb.position(page_offset);
+ bb.limit(page_offset + length);
+ if (page == buffers.length - 1) {
+ // In the final page, so make a copy
+ // This is needed in case of rollback, because the final page
+ // has the potential of being needed by a read-only transaction
+ // even when we need to remove that page to truncate the file.
+ // We have not synched on this page, but the GC will retry removing it
+ // with short delays, so it should get picked up despite our brief use.
+ ByteBuffer data = ByteBuffer.allocate(length);
+ bb.get(data.array(), 0, length);
+ return data;
+ } else {
+ // normal return of the mapped region slice
+ return bb.slice();
+ }
+ } else {
+ // TODO data must become a new write-through type
+ ByteBuffer data = ByteBuffer.allocate(length);
+ byte[] dataArray = data.array();
+ ByteBuffer tmp = buffers[page].asReadOnlyBuffer();
+ tmp.position(page_offset);
+ int firstSlice = PAGE_SIZE - page_offset;
+ tmp.get(dataArray, 0, firstSlice);
+ tmp = buffers[page + 1].asReadOnlyBuffer();
+ tmp.get(dataArray, firstSlice, length - firstSlice);
+ return data;
+ }
+ }
+
+ @Override
+ FileChannel.MapMode getMode() {
+ return FileChannel.MapMode.READ_ONLY;
+ }
+
+ byte[] dump() {
+ byte[] d = new byte[buffers[0].capacity()];
+ int p = buffers[0].position();
+ buffers[0].position(0);
+ buffers[0].get(d);
+ buffers[0].position(p);
+ return d;
+ }
+}
Added: trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileRW.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileRW.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileRW.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2011 Paul Gearon
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.mulgara.util.io;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
+import org.apache.log4j.Logger;
+
+/**
+ * A memory mapped read-write version of LBufferedFile. This is not safe for transactional work.
+ * @author pag
+ */
+public class LMappedBufferedFileRW extends LMappedBufferedFile {
+
+ /** Logger. */
+ static final Logger logger = Logger.getLogger(LMappedBufferedFileRW.class);
+
+ /** The page size to use. */
+ public static final int PREFERRED_PAGE_SIZE;
+
+ /** The size of the page for this object. */
+ protected final int pageSize;
+
+ /** The size of a page to be mapped */
+ private static final int DEFAULT_PAGE_SIZE = 33554432; // 32 MB
+
+ static {
+ String pageSizeStr = System.getProperty(PAGE_SIZE_PROP);
+ int tmp = DEFAULT_PAGE_SIZE;
+ try {
+ if (pageSizeStr != null) tmp = Integer.parseInt(pageSizeStr);
+ } catch (NumberFormatException e) {
+ logger.warn("Property [" + PAGE_SIZE_PROP + "] is not a number [" + pageSizeStr + "]. Using default: " + tmp);
+ }
+ PREFERRED_PAGE_SIZE = tmp;
+ }
+
+ /**
+ * Create a new buffered file for Long seeks.
+ * @param file The file to buffer.
+ * @throws IOException There was an error setting up initial mapping of the file.
+ */
+ public LMappedBufferedFileRW(RandomAccessFile file, int recordSize) throws IOException {
+ super(file);
+ fc = file.getChannel();
+ int recordsPerPage = PREFERRED_PAGE_SIZE / recordSize;
+ pageSize = recordsPerPage * recordSize;
+ mapFile();
+ if (logger.isDebugEnabled()) {
+ logger.debug("Mapping files for R/W with pages of " + pageSize + "bytes at "+ recordsPerPage + " records/page");
+ }
+ }
+
+ @Override
+ public int getPageSize() {
+ return pageSize;
+ }
+
+ @Override
+ public ByteBuffer read(long offset, int length) throws IOException {
+ // get the offset into the last buffer, this may be negative if before the last page
+ long lastPageOffset = (offset + (long)length) - (pageSize * (long)(buffers.length - 1));
+ // if the offset is larger than the final page, then remap
+ if (lastPageOffset > pageSize || lastPageOffset > buffers[buffers.length - 1].limit() || buffers.length == 0) {
+ mapFile();
+ }
+ int page = (int)(offset / pageSize);
+ int page_offset = (int)(offset % pageSize);
+
+ assert page_offset + length <= pageSize : "Access to block outside of record boundaries";
+
+ ByteBuffer bb = buffers[page];
+ bb.position(page_offset);
+ bb.limit(page_offset + length);
+ return bb.slice();
+ }
+
+ @Override
+ FileChannel.MapMode getMode() {
+ return FileChannel.MapMode.READ_WRITE;
+ }
+
+ byte[] dump() {
+ byte[] d = new byte[buffers[0].capacity()];
+ int p = buffers[0].position();
+ buffers[0].position(0);
+ buffers[0].get(d);
+ buffers[0].position(p);
+ return d;
+ }
+}
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileTest.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileTest.java 2011-09-29 18:04:55 UTC (rev 2064)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LMappedBufferedFileTest.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -20,7 +20,7 @@
}
LBufferedFile newFile(RandomAccessFile r) throws IOException {
- return new LMappedBufferedFile(r);
+ return new LMappedBufferedFileRO(r);
}
boolean readOnly() {
Modified: trunk/src/jar/util/java/org/mulgara/util/io/LReadOnlyIOBufferedFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/LReadOnlyIOBufferedFile.java 2011-09-29 18:04:55 UTC (rev 2064)
+++ trunk/src/jar/util/java/org/mulgara/util/io/LReadOnlyIOBufferedFile.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -70,4 +70,14 @@
return 0;
}
+ // debug
+ byte[] dump() throws IOException {
+ byte[] d = new byte[(int)file.length()];
+ long offset = file.getFilePointer();
+ file.seek(0);
+ file.read(d);
+ file.seek(offset);
+ return d;
+ }
+
}
Added: trunk/src/jar/util/java/org/mulgara/util/io/RecordFile.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/RecordFile.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/RecordFile.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2011 Paul Gearon.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mulgara.util.io;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * A file for storing records of fixed length.
+ */
+public interface RecordFile extends Closeable {
+
+ /**
+ * Retrieves a ByteBuffer containing a given record number.
+ * This is a read-only operation, and will not grow the file
+ * if the record does not exist.
+ * @param id The record number.
+ * @return The record data.
+ * @throws IndexOutOfBoundsException if the id is out of the range of the file.
+ * @throws IOException if there is an error accessing the file.
+ */
+ public ByteBuffer get(long id) throws IndexOutOfBoundsException, IOException;
+
+ /**
+ * Retrieves a block associated with a record in the file.
+ * The data in the ByteBuffer is not defined.
+ * @param id The record number.
+ * @return A ByteBuffer associated with the record. The contents may be anything.
+ * @throws IndexOutOfBoundsException if the if is out of the range of the file.
+ * @throws IOException if there is an error accessing the file.
+ */
+ public ByteBuffer getBuffer(long id) throws IOException;
+
+ /**
+ * Puts a record into a file. The ByteBuffer <em>must</em> already be associated
+ * with the block ID. This is not checked, due to performance, but if it is not
+ * true, then the operation of this method is undefined.
+ * @param buffer The data to write to the record.
+ * @param id The record number.
+ * @return The ByteBuffer that was written.
+ * @throws IndexOutOfBoundsException If the id is out of range for the file.
+ * @throws IOException If there is an error writing to the file.
+ */
+ public ByteBuffer put(ByteBuffer buffer, long id) throws IndexOutOfBoundsException, IOException;
+
+ /**
+ * Changes the file size to contain the number of request records.
+ * Any buffer associations may be invalidated.
+ * @param records The number of records to hold in the file.
+ * @return The original RecordFile object.
+ * @throws IOException If there is an error modifying the file.
+ */
+ public RecordFile resize(long records) throws IOException;
+}
Added: trunk/src/jar/util/java/org/mulgara/util/io/RecordFileImpl.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/RecordFileImpl.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/RecordFileImpl.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2010 Paul Gearon.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mulgara.util.io;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+
+/**
+ * An implementation of RecordFile.
+ * @author Paul Gearon
+ */
+public class RecordFileImpl implements RecordFile {
+
+ /** The buffered file containing the records. */
+ private final LBufferedFile recordFile;
+
+ /** The RandomAccessFile the recordFile is based on. */
+ private final RandomAccessFile raf;
+
+ /** The size of the file records. */
+ private final int recordSize;
+
+ /**
+ * Creates a new record file.
+ * @param filename The name of the file to create.
+ * @param recordSize The size of the records in the file.
+ * @param initialBlocks The initial size of the file, in records.
+ * @throws IOException If there was an error creating the file.
+ */
+ public RecordFileImpl(String filename, int recordSize, long initialBlocks) throws IOException {
+ this(new File(filename), recordSize, initialBlocks);
+ }
+
+ /**
+ * Creates a new record file.
+ * @param file The file to create.
+ * @param recordSize The size of the records in the file.
+ * @param initialBlocks The initial size of the file, in records.
+ * @throws IOException If there was an error creating the file.
+ */
+ public RecordFileImpl(File file, int recordSize, long initialBlocks) throws IOException {
+ raf = new RandomAccessFile(file, "rw");
+ recordFile = LBufferedFile.createWritable(raf, recordSize);
+ this.recordSize = recordSize;
+ resize(initialBlocks);
+ }
+
+ /**
+ * @see java.io.Closeable#close()
+ */
+ public void close() throws IOException {
+ raf.getChannel().force(true);
+ raf.close();
+ }
+
+ /**
+ * @see org.mulgara.util.io.RecordFile#get(long)
+ */
+ @Override
+ public ByteBuffer get(long id) throws IndexOutOfBoundsException, IOException {
+ return recordFile.read(id * recordSize, recordSize);
+ }
+
+ /**
+ * @see org.mulgara.util.io.RecordFile#getBuffer(long)
+ */
+ @Override
+ public ByteBuffer getBuffer(long id) throws IOException {
+ return recordFile.allocate(id * recordSize, recordSize);
+ }
+
+ /**
+ * @see org.mulgara.util.io.RecordFile#put(java.nio.ByteBuffer, long)
+ */
+ public ByteBuffer put(ByteBuffer buffer, long id)
+ throws IndexOutOfBoundsException, IOException {
+ recordFile.write(buffer);
+ return buffer;
+ }
+
+ /**
+ * @see org.mulgara.util.io.RecordFile#resize(long)
+ */
+ public RecordFile resize(long records) throws IOException {
+ long fileSize = records * recordSize;
+ // if the file is to be shrunk, then prepare the file for truncation
+ if (fileSize < raf.length()) recordFile.truncate(fileSize);
+ raf.setLength(fileSize);
+ return this;
+ }
+
+ // debug
+ byte[] dump() throws IOException {
+ return recordFile.dump();
+ }
+}
Added: trunk/src/jar/util/java/org/mulgara/util/io/SetDataConverter.java
===================================================================
--- trunk/src/jar/util/java/org/mulgara/util/io/SetDataConverter.java (rev 0)
+++ trunk/src/jar/util/java/org/mulgara/util/io/SetDataConverter.java 2011-10-07 16:56:03 UTC (rev 2065)
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2011 Paul Gearon.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.mulgara.util.io;
+
+/**
+ * An interface for serializing data into and out of set data.
+ * Set data is either the raw data of a Map-key, Map-value, or Map.Entry.
+ */
+public interface SetDataConverter<T, SD> {
+
+ /**
+ * Convert data to set-data .
+ * @param d The data to convert.
+ * @return The set-data representing this data.
+ */
+ public SD toSetData(T d);
+
+ /**
+ * Convert set-data to data.
+ * @param d The set-data containing the data.
+ * @return The data interpreted from the set-data.
+ */
+ public T fromSetData(SD d);
+
+}
More information about the Mulgara-svn
mailing list