[Mulgara-svn] r868 - projects/xa2/object-pool/src
andrae at mulgara.org
andrae at mulgara.org
Tue Apr 29 05:36:36 UTC 2008
Author: andrae
Date: 2008-04-28 22:36:35 -0700 (Mon, 28 Apr 2008)
New Revision: 868
Modified:
projects/xa2/object-pool/src/CompBlockTrieTest.java
Log:
Updated random-file test to test BlockTrie.
Modified: projects/xa2/object-pool/src/CompBlockTrieTest.java
===================================================================
--- projects/xa2/object-pool/src/CompBlockTrieTest.java 2008-04-29 04:37:47 UTC (rev 867)
+++ projects/xa2/object-pool/src/CompBlockTrieTest.java 2008-04-29 05:36:35 UTC (rev 868)
@@ -22,24 +22,23 @@
*/
public class CompBlockTrieTest {
public static void main(String[] args) throws Exception {
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), 1);
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), 2);
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), 3);
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), 4);
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), 5);
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), 6);
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), 7);
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), 8);
- // testWithLimitedFile(new File("../scratch/propernames.uniq"), 9);
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), 10);
-// testWithLimitedFile(new File("../scratch/propernames.uniq"), Integer.MAX_VALUE);
-// testWithLimitedFile(new File("../scratch/connectives.uniq"), Integer.MAX_VALUE);
-// testWithLimitedFile(new File("../scratch/web2a.uniq"), Integer.MAX_VALUE);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 1);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 2);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 3);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 4);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 5);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 6);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 7);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 8);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 9);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), 10);
+ testWithLimitedFile(new File("../scratch/propernames.uniq"), Integer.MAX_VALUE);
+ testWithLimitedFile(new File("../scratch/connectives.uniq"), Integer.MAX_VALUE);
+ testWithLimitedFile(new File("../scratch/web2a.uniq"), Integer.MAX_VALUE);
testWithLimitedFile(new File("../scratch/web2.uniq"), Integer.MAX_VALUE);
-// testWithRandomFileTuned(new File("../bulk/random70M"), 12, 10);
-// testWithRandomFileTuned(new File("../bulk/random70M"), 250, 10);
-// testWithRandomFileTuned(new File("../bulk/random70M"), 5000, 10);
-// testWithRandomFile(new File("/dev/urandom"));
+ testWithRandomFileTuned(new File("../bulk/random70M"), 12, 10);
+ testWithRandomFileTuned(new File("../bulk/random70M"), 250, 10);
+ testWithRandomFileTuned(new File("../bulk/random70M"), 5000, 10);
// testLoadWithRandomFile(new File("/dev/urandom"));
}
@@ -152,6 +151,10 @@
System.arraycopy(bytes, 0, this.bytes, 0, bytes.length);
}
+ public byte[] getBytes() {
+ return bytes;
+ }
+
public boolean equals(Object o) {
if (o instanceof Bytes) {
return Arrays.equals(bytes, ((Bytes)o).bytes);
@@ -164,107 +167,27 @@
int hc = Arrays.hashCode(bytes);
return hc;
}
- }
- public static void testWithRandomFile(File file) throws Exception {
- Map<Bytes, Long> namesMap = new HashMap<Bytes, Long>();
- CompMemTrie namesTrie = new CompMemTrie();
-
- System.out.println("Inserting random bytes from " + file);
- FileChannel chan = new FileInputStream(file).getChannel();
- ByteBuffer buffer = ByteBuffer.allocate(134*1024);
-
- long n = 0;
- byte[] key;
- // Set limit to 0 so initial compact works correctly.
- buffer.clear().limit(0);
- long _startInsert = System.currentTimeMillis();
- long _aggregateL = 0;
- long _avlL = 0;
- long _aggregateS = 0;
- long _avlS = 0;
- int nL = 0;
- int nS = 0;
-
- while (n < 5000) {
- if (buffer.remaining() < 67*1024 && chan.read(buffer.compact()) == -1) {
- break;
- }
- buffer.flip();
-
- key = new byte[((int)buffer.getShort()) & 0x0000FFFF];
- buffer.get(key);
- Bytes keyBytes = new Bytes(key);
-
- if (namesMap.containsKey(keyBytes)) {
-// namesTrie.insert(key, namesMap.get(keyBytes));
- } else {
- n++;
- namesMap.put(keyBytes, n);
- long _si = System.currentTimeMillis();
- namesTrie.insert(key, n);
- _aggregateL += System.currentTimeMillis() - _si;
- _avlL += key.length;
- nL++;
-
- if (namesTrie.lookup(key) != n) {
- throw new IllegalStateException("lookup failed key: " + Arrays.toString(key) + " value: " + n);
- }
- }
-
- for (int i = 0; i < 10; i++) {
- key = new byte[((int)buffer.get()) & 0x000000FF];
- buffer.get(key);
- keyBytes = new Bytes(key);
- if (namesMap.containsKey(keyBytes)) {
-// namesTrie.insert(key, namesMap.get(keyBytes));
- } else {
- n++;
- namesMap.put(keyBytes, n);
- long _ss = System.currentTimeMillis();
- namesTrie.insert(key, n);
- _aggregateS += System.currentTimeMillis() - _ss;
- _avlS += key.length;
- nS++;
-
- if (namesTrie.lookup(key) != n) {
- throw new IllegalStateException("lookup failed key: " + Arrays.toString(key) + " value: " + n);
- }
- }
- }
+ public String toString() {
+ return Arrays.toString(bytes);
}
- long _endInsert = System.currentTimeMillis();
- System.out.println("Input " + namesMap.size() + " entries");
- System.out.printf(" %d long entries ave: %d in: %dms; ave %dus per entry, %dns per byte\n",
- nL, (_avlL / nL), _aggregateL, (_aggregateL * 1000 / nL), (_aggregateL * 1000000 / _avlL));
- System.out.printf(" %d short entries ave: %d in: %dms; ave %dus per entry, %dns per byte\n",
- nS, (_avlS / nS), _aggregateS, (_aggregateS * 1000 / nS), (_aggregateS * 1000000 / _avlS));
- System.out.println(chan.position() + " bytes read from " + file);
- chan.close();
-
- long _startLookup = System.currentTimeMillis();
- System.out.println("Checking random bytes from " + file);
- for (Bytes k : namesMap.keySet()) {
- int i = 0;
- if (namesTrie.lookup(k.bytes) != namesMap.get(k)) {
- throw new IllegalStateException("Trie doesn't match Map on entry: " + i + " key:" + Arrays.toString(k.bytes) + " value: " + namesTrie.lookup(k.bytes) + " value': " + namesMap.get(k));
- }
- i++;
- }
- long _endLookup = System.currentTimeMillis();
-
- System.out.println("Test Succeeded with " + file +
- " insert:" + (_endInsert - _startInsert) + " lookup:" + (_endLookup - _startLookup));
}
public static void testWithRandomFileTuned(File file, int max, int small) throws Exception {
Map<Bytes, Long> namesMap = new HashMap<Bytes, Long>();
- CompMemTrie namesTrie = new CompMemTrie();
+ CompBlockTrie namesTrie = new CompBlockTrie(32*1024);
System.out.println("Inserting random bytes from " + file + " to max records " + max + " with small " + small);
FileChannel chan = new FileInputStream(file).getChannel();
ByteBuffer buffer = ByteBuffer.allocate(134*1024);
+ FileChannel indexFile = new FileOutputStream("tmp/sp_idx").getChannel();
+ FileChannel dataFile = new FileOutputStream("tmp/sp_dat").getChannel();
+ ByteBuffer indexBuffer = ByteBuffer.allocateDirect(BLOCK_SIZE);
+
+ ArrayList<CompBlockTrie> tries = new ArrayList<CompBlockTrie>();
+ CompBlockTrie trie = new CompBlockTrie(BLOCK_SIZE);
+
long n = 0;
byte[] key;
// Set limit to 0 so initial compact works correctly.
@@ -287,43 +210,60 @@
buffer.get(key);
Bytes keyBytes = new Bytes(key);
- if (namesMap.containsKey(keyBytes)) {
-// namesTrie.insert(key, namesMap.get(keyBytes));
- } else {
+ if (!namesMap.containsKey(keyBytes)) {
n++;
namesMap.put(keyBytes, n);
long _si = System.currentTimeMillis();
- namesTrie.insert(key, n);
+
+ if (!trie.insert(key, n)) {
+ trie.write((ByteBuffer)indexBuffer.clear(), dataFile);
+ indexFile.write((ByteBuffer)indexBuffer.flip());
+ tries.add(trie);
+ trie = new CompBlockTrie(BLOCK_SIZE);
+ trie.insert(key, n);
+ }
+
+ n++;
+
_aggregateL += System.currentTimeMillis() - _si;
_avlL += key.length;
nL++;
-
- if (namesTrie.lookup(key) != n) {
- throw new IllegalStateException("lookup failed key: " + Arrays.toString(key) + " value: " + n);
- }
}
for (int i = 0; i < small; i++) {
key = new byte[((int)buffer.get()) & 0x000000FF];
buffer.get(key);
keyBytes = new Bytes(key);
- if (namesMap.containsKey(keyBytes)) {
-// namesTrie.insert(key, namesMap.get(keyBytes));
- } else {
+ if (!namesMap.containsKey(keyBytes)) {
n++;
namesMap.put(keyBytes, n);
long _ss = System.currentTimeMillis();
- namesTrie.insert(key, n);
+
+ if (!trie.insert(key, n)) {
+ trie.write((ByteBuffer)indexBuffer.clear(), dataFile);
+ indexFile.write((ByteBuffer)indexBuffer.flip());
+ tries.add(trie);
+ trie = new CompBlockTrie(BLOCK_SIZE);
+ trie.insert(key, n);
+ }
+ n++;
+
_aggregateS += System.currentTimeMillis() - _ss;
_avlS += key.length;
nS++;
-
- if (namesTrie.lookup(key) != n) {
- throw new IllegalStateException("lookup failed key: " + Arrays.toString(key) + " value: " + n);
- }
}
}
}
+
+ trie.write((ByteBuffer)indexBuffer.clear(), dataFile);
+ indexFile.write((ByteBuffer)indexBuffer.flip());
+ tries.add(trie);
+
+ indexFile.force(true);
+ dataFile.force(true);
+ indexFile.close();
+ dataFile.close();
+
long _endInsert = System.currentTimeMillis();
System.out.println("Input " + namesMap.size() + " entries");
System.out.printf(" %d long entries ave: %d in: %dms; ave %dus per entry, %dns per byte\n",
@@ -333,19 +273,49 @@
System.out.println(chan.position() + " bytes read from " + file);
chan.close();
+ indexFile = new FileInputStream("tmp/sp_idx").getChannel();
+ dataFile = new FileInputStream("tmp/sp_dat").getChannel();
+ indexBuffer = ByteBuffer.allocateDirect(BLOCK_SIZE);
+ ArrayList<CompBlockTrie> readTries = new ArrayList<CompBlockTrie>();
+
+ long _startRead = System.currentTimeMillis();
+
+ while (indexFile.read((ByteBuffer)indexBuffer.clear()) != -1) {
+ indexBuffer.flip();
+ CompBlockTrie ctrie = new CompBlockTrie(indexBuffer, dataFile);
+ readTries.add(ctrie);
+ }
+
+ long _endRead = System.currentTimeMillis();
+
long _startLookup = System.currentTimeMillis();
System.out.println("Checking random bytes from " + file);
- for (Bytes k : namesMap.keySet()) {
+
+ boolean[] valid = new boolean[1];
+
+ for (Bytes keyBytes : namesMap.keySet()) {
int i = 0;
- if (namesTrie.lookup(k.bytes) != namesMap.get(k)) {
- throw new IllegalStateException("Trie doesn't match Map on entry: " + i + " key:" + Arrays.toString(k.bytes) + " value: " + namesTrie.lookup(k.bytes) + " value': " + namesMap.get(k));
+ boolean found = false;
+ for (CompBlockTrie t : readTries) {
+ long value = t.lookup(keyBytes.getBytes(), valid);
+ if (valid[0] && value == namesMap.get(keyBytes)) {
+ found = true;
+ break;
+ }
}
+ if (!found) {
+ throw new IllegalStateException("Trie doesn't match Map on entry: " + i
+ + " key:" + keyBytes + " value: " + namesMap.get(keyBytes));
+ }
i++;
}
+
long _endLookup = System.currentTimeMillis();
System.out.println("Test Succeeded with " + file +
- " insert:" + (_endInsert - _startInsert) + " lookup:" + (_endLookup - _startLookup));
+ " insert:" + (_endInsert - _startInsert) +
+ " read:" + (_endRead - _startRead) +
+ " lookup:" + (_endLookup - _startLookup));
}
public static void testLoadWithRandomFile(File file) throws Exception {
More information about the Mulgara-svn
mailing list