[Mulgara-svn] r387 - in projects/xa2: . draft draft/stuff lib src testdata
andrae at mulgara.org
andrae at mulgara.org
Tue Aug 28 07:39:57 UTC 2007
Author: andrae
Date: 2007-08-28 02:39:57 -0500 (Tue, 28 Aug 2007)
New Revision: 387
Added:
projects/xa2/build.sh
projects/xa2/build.xml
projects/xa2/draft/
projects/xa2/draft/ArraysSortTest.java
projects/xa2/draft/BrokenArraySort.java
projects/xa2/draft/CompressedBlock.java
projects/xa2/draft/CompressedBlockSeq.java
projects/xa2/draft/LongBufferOperations.java
projects/xa2/draft/LongBufferOperationsTest.java
projects/xa2/draft/PhaseDigit.java
projects/xa2/draft/QuadOperations.java
projects/xa2/draft/Quads.java
projects/xa2/draft/RandomQuads.java
projects/xa2/draft/RandomTriples.java
projects/xa2/draft/SortedQuadBuffer.java
projects/xa2/draft/TestHarness.java
projects/xa2/draft/TestSort.java
projects/xa2/draft/TestWrite1.java
projects/xa2/draft/TestWrite2.java
projects/xa2/draft/TestWrite3.java
projects/xa2/draft/TestWrite4.java
projects/xa2/draft/TestWrite5.java
projects/xa2/draft/TestWrite6.java
projects/xa2/draft/Triples.java
projects/xa2/draft/TupleBuffer.java
projects/xa2/draft/Tuples.java
projects/xa2/draft/TuplesWrapsQuads.java
projects/xa2/draft/UncompressedBlock.java
projects/xa2/draft/UncompressedPhaseDigit.java
projects/xa2/draft/calcSizes.py
projects/xa2/draft/stuff/
projects/xa2/draft/stuff/BufferedWriter.java
projects/xa2/draft/stuff/Header.java
projects/xa2/draft/stuff/ReadBuffer.java
projects/xa2/draft/stuff/WriteBuffer.java
projects/xa2/lib/
projects/xa2/lib/ant
projects/xa2/lib/ant-junit.jar
projects/xa2/lib/ant-launcher.jar
projects/xa2/lib/ant.jar
projects/xa2/lib/junit-4.1.jar
projects/xa2/src/
projects/xa2/src/ByteBufferPool.java
projects/xa2/src/ComparableQuadStream.java
projects/xa2/src/CompressedBlock.java
projects/xa2/src/CompressedBlockSeq.java
projects/xa2/src/CompressedBlockSeqUnitTest.java
projects/xa2/src/CompressedBlockUnitTest.java
projects/xa2/src/DigitMerger.java
projects/xa2/src/FileQuadStream.java
projects/xa2/src/GenerateFileQuads.java
projects/xa2/src/GenerateLargeRandomQuads.java
projects/xa2/src/IndexListener.java
projects/xa2/src/LicensedTemplate.java
projects/xa2/src/LongArrayOperations.java
projects/xa2/src/MergedQuadStreams.java
projects/xa2/src/PhaseDigit.java
projects/xa2/src/PhaseDigitEmptyEscape.java
projects/xa2/src/PhaseElement.java
projects/xa2/src/PhaseElementUnitTest.java
projects/xa2/src/PhaseSet.java
projects/xa2/src/PhaseSetUnitTest.java
projects/xa2/src/QuadQSort.java
projects/xa2/src/QuadStream.java
projects/xa2/src/ScaleTest1.java
projects/xa2/testdata/
projects/xa2/testdata/10000random.quads
projects/xa2/testdata/10000sequential.quads
projects/xa2/testdata/10000sorted.quads
Log:
Current status of XA2 migrated from netpr to the mulgara repository.
Added: projects/xa2/build.sh
===================================================================
--- projects/xa2/build.sh 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/build.sh 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+unset ANT_HOME
+CLASSPATH=$CLASSPATH:lib/ant.jar:lib/ant-junit.jar:lib/ant-launcher.jar lib/ant $@
Property changes on: projects/xa2/build.sh
___________________________________________________________________
Name: svn:executable
+ *
Added: projects/xa2/build.xml
===================================================================
--- projects/xa2/build.xml 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/build.xml 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<project name="XA2-Design" default="dist" basedir=".">
+ <description>simple example build file</description>
+ <!-- set global properties for this build -->
+ <property name="name" value="xa2-design"/>
+ <property name="src" location="src"/>
+ <property name="lib" location="lib"/>
+ <property name="build" location="classes"/>
+ <property name="dist" location="dist"/>
+ <property name="test" location="test"/>
+
+ <target name="init">
+ <!-- Create the time stamp -->
+ <tstamp/>
+ <!-- Create the build directory structure used by compile -->
+ <mkdir dir="${build}"/>
+ <mkdir dir="${dist}"/>
+ <mkdir dir="${test}"/>
+ </target>
+
+ <target name="compile" depends="init" description="compile the source ">
+ <javac srcdir="${src}" destdir="${build}" debug="yes">
+ <classpath>
+ <fileset dir="${lib}">
+ <include name="**/*.jar"/>
+ </fileset>
+ </classpath>
+ </javac>
+ </target>
+
+ <target name="dist" depends="compile" description="generate the distribution">
+ <jar jarfile="${dist}/${name}.jar" basedir="${build}"/>
+ </target>
+
+ <target name="test" depends="compile" description="Run the unit tests">
+ <delete dir="${test}"/>
+ <mkdir dir="${test}"/>
+ <junit printsummary="yes" haltonfailure="no" showoutput="yes">
+ <classpath>
+ <pathelement path="${build}"/>
+ <pathelement location="/Developer/Java/Ant/lib/ant-junit.jar"/>
+ <fileset dir="${lib}">
+ <include name="**/*.jar"/>
+ </fileset>
+ </classpath>
+
+ <formatter type="plain"/>
+
+ <batchtest todir="${test}">
+ <fileset dir="${src}">
+ <include name="**/*UnitTest.java"/>
+ </fileset>
+ </batchtest>
+ </junit>
+ </target>
+
+ <target name="scale" depends="compile" description="Run the scale tests">
+ <delete dir="${test}"/>
+ <mkdir dir="${test}"/>
+ <junit printsummary="yes" haltonfailure="no" showoutput="yes">
+ <classpath>
+ <pathelement path="${build}"/>
+ <pathelement location="/Developer/Java/Ant/lib/ant-junit.jar"/>
+ <fileset dir="${lib}">
+ <include name="**/*.jar"/>
+ </fileset>
+ </classpath>
+
+ <formatter type="plain"/>
+
+ <batchtest todir="${test}">
+ <fileset dir="${src}">
+ <include name="**/ScaleTest*.java"/>
+ </fileset>
+ </batchtest>
+ </junit>
+ </target>
+
+ <target name="clean" description="clean up" >
+ <delete dir="${build}"/>
+ <delete dir="${dist}"/>
+ <delete dir="${test}"/>
+ </target>
+
+ <target name="tags" description="Build ctags file for source">
+ <exec executable="ctags">
+ <arg value="-R"/>
+ <arg value="src/"/>
+ </exec>
+ </target>
+</project>
Added: projects/xa2/draft/ArraysSortTest.java
===================================================================
--- projects/xa2/draft/ArraysSortTest.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/ArraysSortTest.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,121 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.util.Arrays;
+
+/**
+ * Writes tuples to buffer, and sorts them.
+ *
+ * FIXME: Loops should probably use arraycopy.
+ */
+public class ArraysSortTest {
+ /** Number of Triples per buffer */
+ /** Total buffer size in bytes */
+
+ private long[] buffer;
+ private long[] buffer2;
+ private int length;
+ private int width;
+ private int maxTuples;
+ private int bufferSize;
+
+ private int current;
+ private Object sorted;
+
+ public static void main(String[] arg) {
+ ArraysSortTest test = new ArraysSortTest();
+ System.out.println("Test completed");
+ }
+
+ /** loads the next maxTuples worth of tuples/width from the tuples object */
+ public ArraysSortTest() {
+ this.buffer = new long[40];
+ this.width = 4;
+
+ this.length = load(buffer, 4, 10);
+ this.buffer2 = new long[length];
+ dump("Pre-clone");
+
+ buffer2 = (long[])buffer.clone();
+ dump("Post-clone");
+
+ Arrays.sort(buffer2);
+
+ dump("Post-resort");
+
+ LongBufferOperations.sort(buffer, width, length);
+
+ if (!LongBufferOperations.check(buffer, width, length)) {
+ throw new IllegalStateException("Test Failed");
+ }
+ current = -1;
+ }
+
+ /** loads maxTuples tuples from 'tuples' into the buffer. */
+ private int load(long[] buffer, int width, int maxTuples) {
+ long[] test = new long[] {
+ 5, 6, 13, 12,
+ 10, 996, 997, 998,
+ 21, 20, 19, 18,
+ 27, 26, 980, 981,
+ 44, 43, 42, 964,
+ 50, 956, 957, 958,
+ 966, 967, 53, 52,
+ 972, 973, 974, 975,
+ 983, 37, 36, 35,
+ 989, 990, 991, 29 };
+
+ System.arraycopy(test, 0, buffer, 0, 40);
+
+ return buffer.length; // buffer is full.
+ }
+/*
+ public void beforeFirst() {
+ dump("beforeFirst");
+ current = -1;
+ }
+
+ public boolean next() {
+ if (current+1 >= length / 4) {
+ return false;
+ } else {
+ current++;
+ return true;
+ }
+ }
+
+ public long getElement(int n) {
+ check();
+ return buffer[current * width + n];
+ }
+
+ public long[] getTuple(long[] quad) {
+ check();
+ assert quad != null;
+ assert quad.length == width;
+
+ System.arraycopy(buffer, current * width, quad, 0, width);
+
+ return quad;
+ }
+
+ private void check() {
+ if (current < 0) {
+ throw new IllegalStateException("Iteration attempted before call to beforeFirst");
+ }
+ if (current >= maxTuples) {
+ throw new IllegalStateException("Iterated beyond end of array");
+ }
+ }
+*/
+ private void dump(String label) {
+ System.err.println("Dump: " + label);
+ for (int i = 0; i < length / width; i++) {
+ System.err.format("Tuple: %d [ %d %d %d %d ]\n", i*width, buffer[i*width], buffer[i*width + 1], buffer[i*width + 2], buffer[i*width + 3]);
+ }
+ for (int i = 0; i < length / width; i++) {
+ System.err.format("Tuple2: %d [ %d %d %d %d ]\n", i*width, buffer2[i*width], buffer2[i*width + 1], buffer2[i*width + 2], buffer2[i*width + 3]);
+ }
+ }
+}
Added: projects/xa2/draft/BrokenArraySort.java
===================================================================
--- projects/xa2/draft/BrokenArraySort.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/BrokenArraySort.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,128 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.util.Arrays;
+
+/**
+ * Writes tuples to buffer, and sorts them.
+ *
+ * FIXME: Loops should probably use arraycopy.
+ */
+public class BrokenArraySort implements Tuples, Cloneable {
+ /** Number of Triples per buffer */
+ /** Total buffer size in bytes */
+
+ private long[] buffer;
+ private long[] buffer2;
+ private int length;
+ private int width;
+ private int maxTuples;
+ private int bufferSize;
+
+ private int current;
+ private Object sorted;
+
+ /** loads the next maxTuples worth of tuples/width from the tuples object */
+ public BrokenArraySort(Tuples tuples, int width, int maxTuples) {
+ this.width = width;
+ this.maxTuples = maxTuples;
+ this.buffer = new long[maxTuples * width];
+
+ long preload = System.currentTimeMillis();
+ this.length = load(buffer, width, maxTuples, tuples);
+ long postload = System.currentTimeMillis();
+ System.err.format("Total sort-load = %d\n", postload - preload);
+ this.buffer2 = new long[length];
+ dump("Pre-clone");
+
+ long[] buffer2 = (long[])buffer.clone();
+ this.buffer2 = buffer2;
+ dump("Post-clone");
+
+ long preresort = System.currentTimeMillis();
+
+ Arrays.sort(buffer2);
+
+ this.buffer2 = buffer2;
+
+ long postresort = System.currentTimeMillis();
+
+ dump("Post-resort");
+
+ System.err.format("Raw sort = %d\n", postresort-preresort);
+
+ long presort = System.currentTimeMillis();
+ LongBufferOperations.sort(buffer, width, length);
+ long postsort = System.currentTimeMillis();
+ System.err.format("Total sort-sort = %d\n", postsort - presort);
+
+ if (!LongBufferOperations.check(buffer, width, length)) {
+ throw new IllegalStateException("Test Failed");
+ }
+ current = -1;
+ }
+
+ /** loads maxTuples tuples from 'tuples' into the buffer. */
+ private int load(long[] buffer, int width, int maxTuples, Tuples tuples) {
+ long[] tmp = new long[width];
+ for (int nTuples = 0; nTuples < maxTuples; nTuples++) {
+ if (tuples.next()) {
+ tmp = tuples.getTuple(tmp);
+ System.err.format("Loading tuple: [ %d %d %d %d ]\n", tmp[0], tmp[1], tmp[2], tmp[3]);
+ System.arraycopy(tmp, 0, buffer, nTuples*width, width);
+ } else {
+ return nTuples * width;
+ }
+ }
+ return buffer.length; // buffer is full.
+ }
+
+ public void beforeFirst() {
+ dump("beforeFirst");
+ current = -1;
+ }
+
+ public boolean next() {
+ if (current+1 >= length / 4) {
+ return false;
+ } else {
+ current++;
+ return true;
+ }
+ }
+
+ public long getElement(int n) {
+ check();
+ return buffer[current * width + n];
+ }
+
+ public long[] getTuple(long[] quad) {
+ check();
+ assert quad != null;
+ assert quad.length == width;
+
+ System.arraycopy(buffer, current * width, quad, 0, width);
+
+ return quad;
+ }
+
+ private void check() {
+ if (current < 0) {
+ throw new IllegalStateException("Iteration attempted before call to beforeFirst");
+ }
+ if (current >= maxTuples) {
+ throw new IllegalStateException("Iterated beyond end of array");
+ }
+ }
+
+ private void dump(String label) {
+ System.err.println("Dump: " + label);
+ for (int i = 0; i < length / width; i++) {
+ System.err.format("Tuple: %d [ %d %d %d %d ]\n", i*width, buffer[i*width], buffer[i*width + 1], buffer[i*width + 2], buffer[i*width + 3]);
+ }
+ for (int i = 0; i < length / width; i++) {
+ System.err.format("Tuple2: %d [ %d %d %d %d ]\n", i*width, buffer2[i*width], buffer2[i*width + 1], buffer2[i*width + 2], buffer2[i*width + 3]);
+ }
+ }
+}
Added: projects/xa2/draft/CompressedBlock.java
===================================================================
--- projects/xa2/draft/CompressedBlock.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/CompressedBlock.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,141 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * Basic SL-structure.
+ *
+ * 1-byte header : 2-bits-field-3, 2-bits-field-2, 2-bits-field-1, 2-bits-field-0
+ * fields : 00 - field 1-byte delta
+ * 01 - field 2-byte delta
+ * 10 - field 4-byte delta
+ * 11 - field 8-byte literal long
+ * consider for field-0 - more likely to contain 0-byte duplicates this might
+ * also apply for field-1:
+ * fields : 00 - field 0-byte delta
+ * 01 - field 1-byte delta
+ * 10 - field 2-byte delta
+ * 11 - field 8-byte literal long
+ */
+
+public class CompressedBlock {
+ public static final byte BYTE = 0x00;
+ public static final byte SHORT = 0x01;
+ public static final byte INT = 0x02;
+ public static final byte LONG = 0x03;
+
+ private ByteBuffer bb;
+
+ private long[] previous;
+ private long[] head;
+ private long[] tail;
+
+ transient private long[] delta;
+ transient private byte[] header;
+
+ /** Due to reset this buffer is normally backed by disk. */
+ public CompressedBlock() {
+ this.previous = new long[] { 0, 0, 0, 0};
+ this.head = null;
+ this.tail = null;
+ this.delta = new long[4];
+ this.header = new byte[4];
+ }
+
+ public void reset(ByteBuffer bb) {
+ this.bb = bb;
+ }
+
+ public boolean write(long[] quad) {
+// System.err.format("writing quad: [ %d %d %d %d ]\n", quad[0], quad[1], quad[2], quad[3]);
+ if (bb == null) {
+ return false;
+ }
+ if (head == null) {
+ head = quad;
+ }
+
+ delta = QuadOperations.subtract(delta, quad, previous);
+ calcHeader(header, delta);
+// System.err.format("Header: [ 0x%02x 0x%02x 0x%02x 0x%02x ]\n", header[0], header[1], header[2], header[3]);
+
+ if (tupleLength(header) > bb.remaining()) {
+// System.err.format("bb.remaining = %d tupleLength = %d aborting write\n", bb.remaining(), tupleLength(header));
+ return false;
+ }
+
+ writeTuple(header, delta);
+
+ System.arraycopy(quad, 0, previous, 0, 4);
+
+ return true;
+ }
+
+ public ByteBuffer getBuffer() {
+ return bb;
+ }
+
+ protected void writeTuple(byte[] header, long[] delta) {
+ byte headerByte = (byte)(((byte)header[3] << 6) |
+ ((byte)header[2] << 4) |
+ ((byte)header[1] << 2) |
+ ((byte)header[0] << 0));
+
+// System.err.format("Writing header byte: 0x%02x\n", headerByte);
+ bb.put(headerByte);
+
+ writeField(header[0], delta[0]);
+ writeField(header[1], delta[1]);
+ writeField(header[2], delta[2]);
+ writeField(header[3], delta[3]);
+ }
+
+ private void calcHeader(byte[] fields, long[] delta) {
+ fields[0] = calcField(delta[0]);
+ fields[1] = calcField(delta[1]);
+ fields[2] = calcField(delta[2]);
+ fields[3] = calcField(delta[3]);
+ }
+
+ private byte calcField(long delta) {
+ if (delta <= Byte.MAX_VALUE) {
+ return BYTE;
+ } else if (delta <= Short.MAX_VALUE) {
+ return SHORT;
+ } else if (delta <= Integer.MAX_VALUE) {
+ return INT;
+ } else {
+ return LONG;
+ }
+ }
+
+ private void writeField(byte header, long field) {
+ switch (header) {
+ case BYTE: bb.put((byte)field); break;
+ case SHORT: bb.putShort((short)field); break;
+ case INT: bb.putInt((int)field); break;
+ case LONG: bb.putLong(field); break;
+ }
+ }
+
+ private int tupleLength(byte[] header) {
+ return fieldLength(header[0]) +
+ fieldLength(header[1]) +
+ fieldLength(header[2]) +
+ fieldLength(header[3]) + 1;
+ }
+
+ private int fieldLength(byte field) {
+ switch (field) {
+ case BYTE: return 1;
+ case SHORT: return 2;
+ case INT: return 4;
+ case LONG: return 8;
+ }
+
+ throw new IllegalArgumentException("Invalid field type: " + field);
+ }
+}
Added: projects/xa2/draft/CompressedBlockSeq.java
===================================================================
--- projects/xa2/draft/CompressedBlockSeq.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/CompressedBlockSeq.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,100 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.RandomAccessFile;
+import java.io.File;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+
+public class CompressedBlockSeq implements Quads {
+ private RandomAccessFile file;
+ private FileChannel channel;
+ private int blocksize;
+ private Quads quads;
+ private long[] quad;
+ private long[] tmp;
+ private ByteBuffer bb ;
+ private CompressedBlock block;
+ private long count = 0;
+
+ public CompressedBlockSeq(File target, int blocksize) throws Exception {
+ this.file = new RandomAccessFile(target, "rw");
+ this.channel = file.getChannel();
+ this.blocksize = blocksize;
+ this.channel.position(blocksize);
+ }
+
+ public void reset(Quads quads) {
+ this.quads = quads;
+ }
+
+ public void close() throws Exception {
+ this.bb.flip();
+ while (this.bb.hasRemaining()) {
+ this.channel.write(bb);
+ }
+ this.channel.force(true);
+ this.file.close();
+ this.bb = null;
+ this.block = null;
+ System.out.format("CompressedBlockSeq wrote out %d quads\n", count);
+ }
+
+ public void beforeFirst() {
+ bb = ByteBuffer.allocate(blocksize);
+ block = new CompressedBlock();
+ quads.beforeFirst();
+ block.reset(bb);
+ quad = new long[4];
+ tmp = new long[4];
+ }
+
+
+ public boolean next() {
+ try {
+ if (!quads.next()) {
+ return false;
+ }
+ quad = quads.getQuad(quad);
+
+ do {
+ tmp = quads.getQuad(tmp);
+ count++;
+ if (!block.write(tmp)) {
+ bb.flip();
+ while (bb.hasRemaining()) {
+ channel.write(bb);
+ }
+ bb.clear();
+ return true;
+ }
+ } while (quads.next());
+
+ return true;
+ } catch (Exception e) {
+ throw new IllegalStateException("Error in next", e);
+ }
+ }
+
+ public long getSubject() {
+ return quad[0];
+ }
+
+ public long getPredicate() {
+ return quad[1];
+ }
+
+ public long getObject() {
+ return quad[2];
+ }
+
+ public long getModel() {
+ return quad[3];
+ }
+
+ public long[] getQuad(long[] quad) {
+ System.arraycopy(this.quad, 0, quad, 0, 4);
+ return quad;
+ }
+}
Added: projects/xa2/draft/LongBufferOperations.java
===================================================================
--- projects/xa2/draft/LongBufferOperations.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/LongBufferOperations.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,197 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+public class LongBufferOperations {
+ public static void sort(long[] buffer, int stride, int length) {
+ if (length % stride != 0) {
+ throw new IllegalArgumentException("Invalid length: " + length + " stride: " + stride);
+ }
+
+// qsort(buffer, stride, 0, length - stride); // 0-index, so length - 1triple
+// dump("Presort", buffer, 4, length);
+ qsort(buffer, 0, (length / 4)); // 0-index, so length - 1triple
+// dump("Postsort", buffer, 4, length);
+ }
+
+/*
+ private static void qsort(long[] buffer, int stride, int left, int right) {
+ if ((right - left) % stride != 0) {
+ throw new IllegalArgumentException(
+ "Invalid sort length, right: " + right + " left: " + left);
+ }
+
+ if ((right - left) <= 0) {
+ return;
+ } else if ((right - left) == stride) {
+ if (comparela(buffer, stride, left, right) > 0) {
+ swap(buffer, stride, left, right);
+ }
+ } else {
+ int centre = left + ((right - left) / 6) * stride; // triples:(/stride) centre:(/2) index:(*stride)
+ // pivot on centre to avoid pathological sort case
+ swap(buffer, stride, left, centre);
+ centre = partition(buffer, stride, left, left + stride, right);
+ qsort(buffer, stride, left, centre - stride);
+ qsort(buffer, stride, centre + stride, right);
+ }
+ }
+*/
+ private static void qsort(long[] buffer, int start, int end) {
+ if (end - start < 2) {
+ return;
+ }
+
+ int pivot = partition(buffer, start, end);
+
+ /* If pivot <= start + 1, then (start, pivot) form a
+ * sorted pair */
+ if (pivot > (start + 1)) {
+ qsort(buffer, start, pivot);
+ }
+ /* Last element is end - 1; if
+ * pivot <= last - 1, then
+ * (pivot, last) form a sorted
+ * pair */
+ if (pivot < (end - 2)) {
+ qsort(buffer, pivot + 1, end);
+ }
+ }
+
+ private static int partition(long[] buffer, int start, int end) {
+ final int size = end - start;
+ int pivot = (size / 2) + start;
+ int lhs = start;
+ int rhs = end - 1;
+
+ for (;;) {
+ while ((lhs < pivot) && (comparela(buffer, pivot*4, lhs*4) > 0)) {
+ lhs++;
+ }
+ while ((rhs > pivot) && (comparela(buffer, pivot*4, rhs*4) < 0)) {
+ rhs--;
+ }
+
+ if (lhs >= rhs) {
+ return pivot;
+ }
+
+ swap(buffer, 4, lhs*4, rhs*4);
+
+ if (lhs == pivot) {
+ lhs++;
+ pivot = rhs;
+ } else if (rhs == pivot) {
+ pivot = lhs;
+ rhs--;
+ } else {
+ lhs++;
+ rhs--;
+ }
+ }
+ }
+
+
+ /*
+ private static int partition(long[] buffer, int stride, int pivot, int left, int right) {
+ for (;;) {
+ // Rather than inverting the test, this is phrased in terms of the
+ // modifying left/right until the relevant invariant holds.
+
+ // Invariant: left is the lowest index > pivot
+ while (!(comparela(buffer, stride, left, pivot) > 0)) {
+ left += stride;
+ if (left >= right) {
+ break;
+ }
+ }
+ // Invariant: right is the lowest index <= pivot
+ while (!(comparela(buffer, stride, right, pivot) <= 0)) {
+ right -= stride;
+ }
+ if (left >= right) {
+ swap(buffer, stride, pivot, right);
+
+ return right;
+ } else {
+ swap(buffer, stride, left, right);
+ }
+ }
+ }
+*/
+// private static int comparela(long[] buffer, int stride, int left, int right) {
+ private static int comparela(long[] buffer, int left, int right) {
+ if (buffer[left] < buffer[right]) {
+ return -1;
+ } else if (buffer[left] > buffer[right]) {
+ return +1;
+ } else if (buffer[left+1] > buffer[right+1]) {
+ return -1;
+ } else if (buffer[left+1] < buffer[right+1]) {
+ return +1;
+ } else if (buffer[left+2] > buffer[right+2]) {
+ return -1;
+ } else if (buffer[left+2] < buffer[right+2]) {
+ return +1;
+ } else if (buffer[left+3] > buffer[right+3]) {
+ return -1;
+ } else if (buffer[left+3] < buffer[right+3]) {
+ return +1;
+ } else {
+ return 0;
+ }
+ }
+
+ private static int comparel(long left, long right) {
+ if (left < right) {
+ return -1;
+ } else if (left > right) {
+ return +1;
+ } else {
+ return 0;
+ }
+ }
+
+ private static void swap(long[] buffer, int stride, int left, int right) {
+ for (int i = 0; i < stride; i++) {
+ long tmp = buffer[left + i];
+ buffer[left + i] = buffer[right + i];
+ buffer[right + i] = tmp;
+ }
+ }
+
+ private static void dump(String label, long[] buffer, int stride, int length) {
+ System.err.format("Sort-Dump: %s (%d %d)\n", label, stride, length);
+ for (int i = 0; i < length / stride; i++) {
+ System.err.format("Tuple %d: [ %d %d %d %d ]\n", i*stride, buffer[i*stride], buffer[i*stride + 1], buffer[i*stride + 2], buffer[i*stride + 3]);
+ }
+ }
+
+/*
+ public static boolean check(long[] buffer, int stride, int length) {
+ for (int i = 0; i < length / stride - 1; i++) {
+ if (comparela(buffer, stride, i*stride, (i+1)*stride) > 0) {
+ dump("SORT-ERROR - " + i*stride, buffer, stride, length);
+ return false;
+ }
+ }
+ return true;
+ }
+*/
+ public static boolean check(long[] buffer, int stride, int length) {
+ for (int i = 0; i < length / stride - 1; i++) {
+ if (comparela(buffer, i*stride, (i+1)*stride) > 0) {
+ dumpError("SORT-ERROR - " + i*stride, i, buffer, stride, length);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static void dumpError(String label, int n, long[] buffer, int stride, int length) {
+ System.err.format("Sort-Dump: %s (%d %d)\n", label, stride, length);
+ for (int i = n - 10; i < length / stride && i < n + 10; i++) {
+ System.err.format("Tuple %d: [ %d %d %d %d ]\n", i*stride, buffer[i*stride], buffer[i*stride + 1], buffer[i*stride + 2], buffer[i*stride + 3]);
+ }
+ }
+}
Added: projects/xa2/draft/LongBufferOperationsTest.java
===================================================================
--- projects/xa2/draft/LongBufferOperationsTest.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/LongBufferOperationsTest.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,20 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.File;
+
+public class LongBufferOperationsTest {
+ public static void main(String[] args) {
+ int total = 10000;
+ for (int i = 0; i < total; i++) {
+ if (i % (total / 10) == 0) {
+ System.err.print("#");
+ }
+ RandomTriples source = new RandomTriples(1010L);
+ source.beforeFirst();
+ TripleBuffer buffer = new TripleBuffer(source, 1L);
+ }
+ System.err.println(" Test Succeeded");
+ }
+}
Added: projects/xa2/draft/PhaseDigit.java
===================================================================
--- projects/xa2/draft/PhaseDigit.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/PhaseDigit.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,47 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.RandomAccessFile;
+import java.io.File;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+
+public class PhaseDigit {
+ public static int HEADER_SIZE = 32*1024;
+ public static int BLOCK_SIZE = 32*1024;
+
+ private RandomAccessFile file;
+ private FileChannel channel;
+
+ public PhaseDigit(File target) throws Exception {
+ this.file = new RandomAccessFile(target, "rw");
+ this.channel = file.getChannel();
+ this.channel.position(HEADER_SIZE);
+ }
+
+ public void write(Quads quads) throws Exception {
+ ByteBuffer buffer = ByteBuffer.allocate(BLOCK_SIZE);
+ CompressedBlock compressed = new CompressedBlock();
+ quads.beforeFirst();
+ compressed.reset(buffer);
+ while (quads.next()) {
+ if (!compressed.write(quads.getQuad(new long[4]))) {
+ buffer.flip();
+ while (buffer.hasRemaining()) {
+ channel.write(buffer);
+ buffer.compact();
+ }
+ }
+ }
+ }
+
+ public void write(Quads[] addends) {
+ throw new IllegalStateException("Merge not supported yet");
+ }
+
+ public void close() throws Exception {
+ this.channel.force(true);
+ this.file.close();
+ }
+}
Added: projects/xa2/draft/QuadOperations.java
===================================================================
--- projects/xa2/draft/QuadOperations.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/QuadOperations.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,34 @@
+/**
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+/**
+ * Represents operations on quads (represented by long[4]'s)
+ */
+public class QuadOperations {
+ /**
+ * diff - difference
+ * sub - subtrahend
+ * min - minuend
+ */
+ public static long[] subtract(long[] diff, long[]sub, long[] min) {
+ isQuad(diff);
+ isQuad(sub);
+ isQuad(min);
+
+ diff[0] = sub[0] - min[0];
+ diff[1] = sub[1] - min[1];
+ diff[2] = sub[2] - min[2];
+ diff[3] = sub[3] - min[3];
+
+ return diff;
+ }
+
+ public static void isQuad(long[] quad) {
+ if (quad == null) {
+ throw new IllegalArgumentException("Quad argument null");
+ } else if (quad.length != 4) {
+ throw new IllegalArgumentException("Quad argument not length 4: " + quad);
+ }
+ }
+}
Added: projects/xa2/draft/Quads.java
===================================================================
--- projects/xa2/draft/Quads.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/Quads.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,18 @@
+/**
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+/**
+ * Represents a stream of 4-tuples.
+ */
+public interface Quads {
+ public void beforeFirst();
+ public boolean next();
+
+ public long getSubject();
+ public long getPredicate();
+ public long getObject();
+ public long getModel();
+
+ public long[] getQuad(long[] quad);
+}
Added: projects/xa2/draft/RandomQuads.java
===================================================================
--- projects/xa2/draft/RandomQuads.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/RandomQuads.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,85 @@
+/**
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.util.Random;
+
+/**
+ * Represents a stream of 4-tuples.
+ */
+public class RandomQuads implements Quads {
+ private static final long BASE_CAP = 1000;
+ private static final long MAX_CAP = 100000;
+ private static final long INC_RATE = 10;
+ private static final long HEAD = 2; // Headroom for model / unbound / etc.
+ private static final long MERSENNE_61 = 2305843009213693951L;
+
+ private long seed;
+ private long current;
+ private long cap;
+ private long[] quad;
+ private int size;
+ private int n;
+
+ public RandomQuads(int size) {
+ this(size, MERSENNE_61);
+ }
+
+ public RandomQuads(int size, long seed) {
+ this.size = size;
+ this.seed = seed;
+ this.quad = new long[4];
+ }
+
+ private long nextLong() {
+ current += MERSENNE_61;
+ return current > 0 ? current : -current;
+ }
+
+ public void beforeFirst() {
+ n = 0;
+ this.cap = BASE_CAP;
+ this.current = seed;
+ }
+
+ public boolean next() {
+ if (n >= size) {
+ return false;
+ }
+ if (nextLong() % 100 < INC_RATE && cap < MAX_CAP) {
+ cap++;
+ }
+ if (cap == 0) {
+ System.err.println("ERROR CAP == 0");
+ return false;
+ }
+ quad[0] = StrictMath.abs(nextLong()) % cap + HEAD;
+ quad[1] = StrictMath.abs(nextLong()) % cap + HEAD;
+ quad[2] = StrictMath.abs(nextLong()) % cap + HEAD;
+ quad[3] = StrictMath.abs(nextLong()) % cap + HEAD;
+ n++;
+ return true;
+ }
+
+ public long getSubject() {
+ return quad[0];
+ }
+
+ public long getPredicate() {
+ return quad[1];
+ }
+
+ public long getObject() {
+ return quad[2];
+ }
+
+ public long getModel() {
+ return quad[3];
+ }
+
+ public long[] getQuad(long[] quad) {
+ System.arraycopy(this.quad, 0, quad, 0, 4);
+
+ return quad;
+ }
+}
Added: projects/xa2/draft/RandomTriples.java
===================================================================
--- projects/xa2/draft/RandomTriples.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/RandomTriples.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,70 @@
+/**
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.util.Random;
+
+/**
+ * Represents a stream of 4-tuples.
+ */
+public class RandomTriples implements Triples {
+ private static final long BASE_CAP = 1000;
+ private static final long INC_RATE = 10;
+ private static final long HEAD = 2; // Headroom for model / unbound / etc.
+
+ private long seed;
+ private Random random;
+ private long cap;
+ private long[] triple = {0L, 0L, 0L};
+ private long size;
+ private long n;
+
+ public RandomTriples(long size) {
+ this(size, (new Random()).nextLong());
+ }
+
+ public RandomTriples(long size, long seed) {
+ this.size = size;
+ this.seed = seed;
+ }
+
+ public void beforeFirst() {
+ this.random = new Random(seed);
+ cap = BASE_CAP;
+ n = 0;
+ }
+
+ public boolean next() {
+ if (n >= size) {
+ return false;
+ }
+
+ triple[0] = StrictMath.abs(random.nextLong()) % cap + HEAD;
+ triple[1] = StrictMath.abs(random.nextLong()) % cap + HEAD;
+ triple[2] = StrictMath.abs(random.nextLong()) % cap + HEAD;
+ if (random.nextFloat() < (1.0 / INC_RATE)) {
+ cap++;
+ }
+ n++;
+
+ return true;
+ }
+
+ public long getSubject() {
+ return triple[0];
+ }
+
+ public long getPredicate() {
+ return triple[1];
+ }
+
+ public long getObject() {
+ return triple[2];
+ }
+
+ public long[] getTriple(long[] triple) {
+ System.arraycopy(this.triple, 0, triple, 0, 3);
+
+ return triple;
+ }
+}
Added: projects/xa2/draft/SortedQuadBuffer.java
===================================================================
--- projects/xa2/draft/SortedQuadBuffer.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/SortedQuadBuffer.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,46 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+/**
+ * Writes tuples to buffer, and sorts them.
+ *
+ * FIXME: Loops should probably use arraycopy.
+ */
+public class SortedQuadBuffer implements Quads, Cloneable {
+ private TupleBuffer buffer;
+
+ /** loads the next maxTuples worth of tuples/width from the tuples object */
+ public SortedQuadBuffer(Quads quads) {
+ buffer = new TupleBuffer(new TuplesWrapsQuads(quads), 4, 1000000);
+ }
+
+ public void beforeFirst() {
+ System.err.println("Called beforeFirst on SortedQuadBuffer");
+ buffer.beforeFirst();
+ }
+
+ public boolean next() {
+ return buffer.next();
+ }
+
+ public long getSubject() {
+ return buffer.getElement(0);
+ }
+
+ public long getPredicate() {
+ return buffer.getElement(1);
+ }
+
+ public long getObject() {
+ return buffer.getElement(2);
+ }
+
+ public long getModel() {
+ return buffer.getElement(3);
+ }
+
+ public long[] getQuad(long []quad) {
+ return buffer.getTuple(quad);
+ }
+}
Added: projects/xa2/draft/TestHarness.java
===================================================================
--- projects/xa2/draft/TestHarness.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TestHarness.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,34 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.File;
+
+public class TestHarness {
+ public static void main(String[] args) {
+ try {
+ File target = new File("./test");
+ if (target.exists()) {
+ target.delete();
+ }
+ File targetuc = new File("./testuc");
+ if (targetuc.exists()) {
+ targetuc.delete();
+ }
+
+ RandomTriples source = new RandomTriples(1010L);
+ source.beforeFirst();
+ TupleBuffer = new TupleBuffer(source, 1L);
+ buffer.beforeFirst();
+ PhaseDigit digit = new PhaseDigit(target);
+ digit.write(buffer);
+ digit.close();
+ buffer.beforeFirst();
+ UncompressedPhaseDigit ucdigit = new UncompressedPhaseDigit(targetuc);
+ ucdigit.write(buffer);
+ ucdigit.close();
+ } catch (Exception e) {
+ throw new RuntimeException("Exception caught at toplevel", e);
+ }
+ }
+}
Added: projects/xa2/draft/TestSort.java
===================================================================
--- projects/xa2/draft/TestSort.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TestSort.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,26 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.util.Arrays;
+
+public class TestSort {
+ public static void main(String[] args) {
+ long[] test = new long[] { 5, 4, 3, 2, 1, 6, 7, 8, 9, 0 };
+ System.out.print("test = [");
+ for (int i = 0; i < test.length; i++) {
+ System.out.format(" %d ", test[i]);
+ }
+ System.out.println(" ]");
+
+ long[] test2 = (long[])test.clone();
+
+ Arrays.sort(test2);
+
+ System.out.print("test = [");
+ for (int i = 0; i < test2.length; i++) {
+ System.out.format(" %d ", test2[i]);
+ }
+ System.out.println(" ]");
+ }
+}
Added: projects/xa2/draft/TestWrite1.java
===================================================================
--- projects/xa2/draft/TestWrite1.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TestWrite1.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,36 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+
+public class TestWrite1 {
+ public static void main(String[] args) {
+ try {
+ File target = new File("./testwrite1");
+
+ RandomAccessFile raf = new RandomAccessFile(target, "rw");
+ FileChannel fc = raf.getChannel();
+ fc.position(0);
+
+ ByteBuffer bb = ByteBuffer.allocate(4096);
+
+ for (int i = 0; i < 1024; i++) {
+ bb.putInt(i);
+ }
+
+ bb.flip();
+ while (bb.hasRemaining()) {
+ fc.write(bb);
+ }
+
+ fc.close();
+ raf.close();
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
Added: projects/xa2/draft/TestWrite2.java
===================================================================
--- projects/xa2/draft/TestWrite2.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TestWrite2.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,44 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+
+public class TestWrite2 {
+ public static void main(String[] args) {
+ try {
+ File target = new File("./testwrite2");
+
+ RandomAccessFile raf = new RandomAccessFile(target, "rw");
+ FileChannel fc = raf.getChannel();
+ fc.position(0);
+
+ ByteBuffer bb = ByteBuffer.allocate(2*1024*1024);
+
+ long[] quad = new long[] { 0, 0, 0, 0 };
+ CompressedBlock block = new CompressedBlock();
+ block.reset(bb);
+
+ int count = 0;
+ while (block.write(quad)) {
+ count++;
+ quad[0] += 0x0000000000000001L;
+ quad[1] += 0x0000000000000123L;
+ quad[2] += 0x0000000001234567L;
+ quad[3] += 0x0000000076543210L;
+ }
+
+ bb.flip();
+ fc.write(bb);
+
+ fc.close();
+ raf.close();
+ System.out.format("Wrote %d quads\n", count);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
Added: projects/xa2/draft/TestWrite3.java
===================================================================
--- projects/xa2/draft/TestWrite3.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TestWrite3.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,56 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+
+public class TestWrite3 {
+ public static void main(String[] args) {
+ try {
+ File target = new File("./testwrite3");
+
+ RandomAccessFile raf = new RandomAccessFile(target, "rw");
+ FileChannel fc = raf.getChannel();
+ fc.position(0);
+
+ ByteBuffer bb = ByteBuffer.allocate(30*1024*1024);
+
+ long start = System.currentTimeMillis();
+ RandomQuads quads = new RandomQuads(4200000);
+ long preload = System.currentTimeMillis();
+ CompressedBlock block = new CompressedBlock();
+ block.reset(bb);
+
+ int count = -1;
+ long[] quad = new long[4];
+ quads.beforeFirst();
+ do {
+ count++;
+ quads.next();
+ } while (block.write(quads.getQuad(quad)));
+ long postload = System.currentTimeMillis();
+
+ bb.flip();
+ fc.write(bb);
+ long postwrite = System.currentTimeMillis();
+
+ fc.force(true);
+ long postflush = System.currentTimeMillis();
+ fc.close();
+ raf.close();
+ long end = System.currentTimeMillis();
+ System.out.format("Wrote %d quads\n", count);
+ System.out.format("Total Time = %d\n", end - start);
+ System.out.format("Total Setup = %d\n", preload - start);
+ System.out.format("Total Load = %d\n", postload - preload);
+ System.out.format("Total Write = %d\n", postwrite - postload);
+ System.out.format("Total Flush = %d\n", postflush - postwrite);
+ System.out.format("Total Close = %d\n", end - postflush);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
Added: projects/xa2/draft/TestWrite4.java
===================================================================
--- projects/xa2/draft/TestWrite4.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TestWrite4.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,39 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+
+public class TestWrite4 {
+ public static void main(String[] args) {
+ try {
+ long start = System.currentTimeMillis();
+ File target = new File("./testwrite4");
+ int count = 100000;
+ RandomQuads quads = new RandomQuads(count);
+ CompressedBlockSeq seq = new CompressedBlockSeq(target, 128*1024);
+ seq.reset(quads);
+ long preload = System.currentTimeMillis();
+
+ long[] quad = new long[4];
+ seq.beforeFirst();
+ while (seq.next()) {
+ quad = seq.getQuad(quad);
+// System.out.format("Index quad: [ %d %d %d %d ]\n", quad[0], quad[1], quad[2], quad[2]);
+ }
+ long postload = System.currentTimeMillis();
+ seq.close();
+ long end = System.currentTimeMillis();
+ System.out.format("Wrote %d quads\n", count);
+ System.out.format("Total Time = %d\n", end - start);
+ System.out.format("Total Setup = %d\n", preload - start);
+ System.out.format("Total Load = %d\n", postload - preload);
+ System.out.format("Total Close = %d\n", end - postload);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
Added: projects/xa2/draft/TestWrite5.java
===================================================================
--- projects/xa2/draft/TestWrite5.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TestWrite5.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,54 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+
+public class TestWrite5 {
+ public static void main(String[] args) {
+ try {
+ long start = System.currentTimeMillis();
+ File target = new File("./testwrite5");
+ if (target.exists()) {
+ target.delete();
+ }
+ int count = 1000000;
+ Quads quads = new SortedQuadBuffer(new RandomQuads(count));
+// quads.beforeFirst();
+/*
+ long[] tmp = new long[] { 1, 2, 3, 4 };
+ for (int i = 0; i < 100; i++) {
+ quads.next();
+ tmp = quads.getQuad(tmp);
+ System.out.format("sorted %d: [ %d %d %d %d ]\n", i, tmp[0], tmp[1], tmp[2], tmp[3]);
+ }
+
+ return;
+*/
+ long preload = System.currentTimeMillis();
+ System.out.format("Total Setup = %d\n", preload - start);
+ CompressedBlockSeq seq = new CompressedBlockSeq(target, 4*1024);
+ seq.reset(quads);
+
+ long[] quad = new long[4];
+ seq.beforeFirst();
+ while (seq.next()) {
+ quad = seq.getQuad(quad);
+// System.out.format("Index quad: [ %d %d %d %d ]\n", quad[0], quad[1], quad[2], quad[2]);
+ }
+ long postload = System.currentTimeMillis();
+ System.out.format("Total Write = %d\n", postload - preload);
+ seq.close();
+ long end = System.currentTimeMillis();
+ System.out.format("Total Close = %d\n", end - postload);
+
+ System.out.format("\nWrote %d quads\n", count);
+ System.out.format("Total Time = %d\n", end - start);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
Added: projects/xa2/draft/TestWrite6.java
===================================================================
--- projects/xa2/draft/TestWrite6.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TestWrite6.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,51 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+
+public class TestWrite6 {
+ public static void main(String[] args) {
+ try {
+ long start = System.currentTimeMillis();
+ int count = 100*1000*1000;
+ Quads quads = new RandomQuads(count);
+ quads.beforeFirst();
+ long fileNr = 0;
+ while (quads.next()) {
+ long presort = System.currentTimeMillis();
+ Quads sortedQuads = new SortedQuadBuffer(quads);
+
+ long preload = System.currentTimeMillis();
+ System.out.format("Total Sort = %d\n", preload - presort);
+ File target = new File("./testwrite6/blk-" + fileNr++);
+ if (target.exists()) {
+ target.delete();
+ }
+ CompressedBlockSeq seq = new CompressedBlockSeq(target, 4*1024);
+ seq.reset(sortedQuads);
+
+ seq.beforeFirst();
+ long[] quad = new long[4];
+ while (seq.next()) {
+ quad = seq.getQuad(quad);
+ }
+ long postload = System.currentTimeMillis();
+ System.out.format("Total Write = %d\n", postload - preload);
+ seq.close();
+ long postwrite = System.currentTimeMillis();
+ System.out.format("Total Close = %d\n", postwrite - postload);
+ }
+
+ long end = System.currentTimeMillis();
+
+ System.out.format("\nWrote %d quads\n", count);
+ System.out.format("Total Time = %d\n", end - start);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
Added: projects/xa2/draft/Triples.java
===================================================================
--- projects/xa2/draft/Triples.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/Triples.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,16 @@
+/**
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+/**
+ * Represents a stream of 3-tuples.
+ */
+interface Triples {
+ public void beforeFirst();
+ public boolean next();
+ public long getSubject();
+ public long getPredicate();
+ public long getObject();
+
+ public long[] getTriple(long[] triple);
+}
Added: projects/xa2/draft/TupleBuffer.java
===================================================================
--- projects/xa2/draft/TupleBuffer.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TupleBuffer.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,105 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.util.Arrays;
+
+/**
+ * Writes tuples to buffer, and sorts them.
+ *
+ * FIXME: Loops should probably use arraycopy.
+ */
+public class TupleBuffer implements Tuples, Cloneable {
+ /** Number of Triples per buffer */
+ /** Total buffer size in bytes */
+
+ private long[] buffer;
+ private int length;
+ private int width;
+ private int maxTuples;
+ private int bufferSize;
+
+ private int current;
+ private Object sorted;
+
+ /** loads the next maxTuples worth of tuples/width from the tuples object */
+ public TupleBuffer(Tuples tuples, int width, int maxTuples) {
+ this.width = width;
+ this.maxTuples = maxTuples;
+ this.buffer = new long[maxTuples * width];
+
+ long preload = System.currentTimeMillis();
+ this.length = load(buffer, width, maxTuples, tuples);
+ long postload = System.currentTimeMillis();
+ System.err.format("Total sort-load = %d\n", postload - preload);
+
+ long presort = System.currentTimeMillis();
+ LongBufferOperations.sort(buffer, width, length);
+ long postsort = System.currentTimeMillis();
+ System.err.format("Total sort-sort = %d\n", postsort - presort);
+
+ if (!LongBufferOperations.check(buffer, width, length)) {
+ throw new IllegalStateException("Test Failed");
+ }
+ current = -1;
+ }
+
+ /** loads maxTuples tuples from 'tuples' into the buffer. */
+ private int load(long[] buffer, int width, int maxTuples, Tuples tuples) {
+ long[] tmp = new long[width];
+ for (int nTuples = 0; nTuples < maxTuples; nTuples++) {
+ if (tuples.next()) {
+ tmp = tuples.getTuple(tmp);
+// System.err.format("Loading tuple: [ %d %d %d %d ]\n", tmp[0], tmp[1], tmp[2], tmp[3]);
+ System.arraycopy(tmp, 0, buffer, nTuples*width, width);
+ } else {
+ return nTuples * width;
+ }
+ }
+ return buffer.length; // buffer is full.
+ }
+
+ public void beforeFirst() {
+ current = -1;
+ }
+
+ public boolean next() {
+ if (current+1 >= length / 4) {
+ return false;
+ } else {
+ current++;
+ return true;
+ }
+ }
+
+ public long getElement(int n) {
+ check();
+ return buffer[current * width + n];
+ }
+
+ public long[] getTuple(long[] quad) {
+ check();
+ assert quad != null;
+ assert quad.length == width;
+
+ System.arraycopy(buffer, current * width, quad, 0, width);
+
+ return quad;
+ }
+
+ private void check() {
+ if (current < 0) {
+ throw new IllegalStateException("Iteration attempted before call to beforeFirst");
+ }
+ if (current >= maxTuples) {
+ throw new IllegalStateException("Iterated beyond end of array");
+ }
+ }
+
+ private void dump(String label) {
+ System.err.println("Dump: " + label);
+ for (int i = 0; i < length / width; i++) {
+ System.err.format("Tuple: %d [ %d %d %d %d ]\n", i*width, buffer[i*width], buffer[i*width + 1], buffer[i*width + 2], buffer[i*width + 3]);
+ }
+ }
+}
Added: projects/xa2/draft/Tuples.java
===================================================================
--- projects/xa2/draft/Tuples.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/Tuples.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,13 @@
+/**
+ * (c) Jan 2007 Andrae Muys - Licenced GPL v2
+ */
+
+/**
+ * Represents a stream of n-tuples.
+ */
+interface Tuples {
+ public void beforeFirst();
+ public boolean next();
+
+ public long[] getTuple(long[] triple);
+}
Added: projects/xa2/draft/TuplesWrapsQuads.java
===================================================================
--- projects/xa2/draft/TuplesWrapsQuads.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/TuplesWrapsQuads.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,33 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+public class TuplesWrapsQuads implements Tuples {
+ private Quads quads;
+ private long count = 0;
+
+ public TuplesWrapsQuads(Quads quads) {
+ this.quads = quads;
+ }
+
+ public void beforeFirst() {
+ quads.beforeFirst();
+ }
+
+ public boolean next() {
+ boolean result = quads.next();
+ if (!result) {
+// System.err.format("TWQ.next() returned false after %d tuples\n", count);
+ return false;
+ }
+// if (count % 100000 == 0) {
+// System.err.format("TWQ.next() returned true after %d tuples\n", count);
+// }
+// count++;
+ return true;
+ }
+
+ public long[] getTuple(long[] quad) {
+ return quads.getQuad(quad);
+ }
+}
Added: projects/xa2/draft/UncompressedBlock.java
===================================================================
--- projects/xa2/draft/UncompressedBlock.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/UncompressedBlock.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,36 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.nio.ByteBuffer;
+import java.nio.LongBuffer;
+
+public class UncompressedBlock {
+ private LongBuffer lb;
+ private ByteBuffer bb;
+
+ public UncompressedBlock() { }
+
+ public void reset(ByteBuffer bb) {
+ this.bb = bb;
+ this.lb = bb.asLongBuffer();
+ }
+
+ public boolean write(long[] quad) {
+ if (lb == null) {
+ throw new IllegalStateException("Attempt to write to UncompressedBlock without reset");
+ }
+
+ if (lb.remaining() < 4) {
+ return false;
+ }
+
+ lb.put(quad);
+
+ return true;
+ }
+
+ public ByteBuffer getBuffer() {
+ return bb;
+ }
+}
Added: projects/xa2/draft/UncompressedPhaseDigit.java
===================================================================
--- projects/xa2/draft/UncompressedPhaseDigit.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/UncompressedPhaseDigit.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,47 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+import java.io.RandomAccessFile;
+import java.io.File;
+import java.nio.channels.FileChannel;
+import java.nio.ByteBuffer;
+
+public class UncompressedPhaseDigit {
+ public static int HEADER_SIZE = 32*1024;
+ public static int BLOCK_SIZE = 32*1024;
+
+ private RandomAccessFile file;
+ private FileChannel channel;
+
+ public UncompressedPhaseDigit(File target) throws Exception {
+ this.file = new RandomAccessFile(target, "rw");
+ this.channel = file.getChannel();
+ this.channel.position(HEADER_SIZE);
+ }
+
+ public void write(Quads quads) throws Exception {
+ ByteBuffer buffer = ByteBuffer.allocate(BLOCK_SIZE);
+ UncompressedBlock block = new UncompressedBlock();
+ quads.beforeFirst();
+ block.reset(buffer);
+ while (quads.next()) {
+ if (!block.write(quads.getQuad(new long[4]))) {
+ buffer.flip();
+ while (buffer.hasRemaining()) {
+ channel.write(buffer);
+ buffer.compact();
+ }
+ }
+ }
+ }
+
+ public void write(Quads[] addends) {
+ throw new IllegalStateException("Merge not supported yet");
+ }
+
+ public void close() throws Exception {
+ this.channel.force(true);
+ this.file.close();
+ }
+}
Added: projects/xa2/draft/calcSizes.py
===================================================================
--- projects/xa2/draft/calcSizes.py 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/calcSizes.py 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+# (c) Dec 2006 Andrae Muys - Licenced GPL v2
+
+NTriples = 2**34
+BlockSize = 2**15
+Base = 4
+NBytes = NTriples * 32
+NBlocks = NBytes / BlockSize
+RefsPerBlock = BlockSize / 32
+NRefBlocks = [NBlocks / RefsPerBlock]
+for x in range(1,100):
+ NRefBlocksN = NRefBlocks[x-1] / RefsPerBlock
+ if NRefBlocksN > 0:
+ NRefBlocks += [NRefBlocksN]
+ else:
+ break
+
+print "Using base: ", Base
+print "Using block size: ", BlockSize
+print "Number of Triples: ", NTriples
+print ""
+print "NBytes (uncompressed): ", NTriples * 32
+print "NBytes (compressed): ", NBytes
+print "Skip-Index levels: ", len(NRefBlocks)
+for (n,ref) in enumerate(NRefBlocks):
+ print "NRefBlocks%d: %d" % (n, ref)
+ print "NRefBytes%d: %d" % (n, (ref * BlockSize))
Added: projects/xa2/draft/stuff/BufferedWriter.java
===================================================================
--- projects/xa2/draft/stuff/BufferedWriter.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/stuff/BufferedWriter.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,50 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+/**
+ * Writes single buffers (30MB).
+ */
+public class BufferedWriter {
+ public static final int SIMPLE_WRITE = 0;
+ public static final int NEW_BLOCK = 1;
+ public static final int FULL_BUFFER = 2;
+
+ private static final int BUFFER_SIZE = 30*1024*1024;
+ private static final int BLOCK_SIZE = 32*1024;
+
+ ByteBuffer bb;
+ WriteBuffer wb;
+ List[] skipIndex;
+
+ public BufferedWriter() {
+ bb = ByteBuffer.allocate(BUFFER_SIZE);
+ wb = new WriteBuffer();
+ }
+
+ private ByteBuffer newSlice() {
+ if (bb.remaining() < BLOCK_SIZE) {
+ return null;
+ }
+
+ return bb.slice().limit(BLOCK_SIZE);
+ }
+
+ /**
+ * returns SIMPLE_WRITE if the write completes in the current block;
+ * NEW_BLOCK if the write completes in a new block within the current buffer
+ * FULL_BUFFER if the write cannot complete because the buffer is full.
+ */
+ private int writeTuple(Tuple tuple) {
+ if (wb.write(tuple)) {
+ return SIMPLE_WRITE;
+ }
+
+ wb.reset(newSlice());
+ if (wb.writeTuple(tuple)) {
+ return FULL_BUFFER;
+ } else {
+ return NEW_BLOCK;
+ }
+ }
+}
Added: projects/xa2/draft/stuff/Header.java
===================================================================
--- projects/xa2/draft/stuff/Header.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/stuff/Header.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,17 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+
+
+public class Header {
+ public int magic = 0x07DFDA7A;
+ public int version = 2;
+ public int blocksize;
+ public long index1offset;
+ public int index1blocks;
+ public long index2offset;
+ public int index2blocks;
+ public long index3offset;
+ public int index3blocks;
+}
Added: projects/xa2/draft/stuff/ReadBuffer.java
===================================================================
--- projects/xa2/draft/stuff/ReadBuffer.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/stuff/ReadBuffer.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,62 @@
+/**
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+/**
+ * Basic SL-structure.
+ *
+ * 1-byte header : 2-bits-field-3, 2-bits-field-2, 2-bits-field-1, 2-bits-field-0
+ * fields : 00 - field 1-byte delta
+ * 01 - field 2-byte delta
+ * 10 - field 4-byte delta
+ * 11 - field 8-byte literal long
+ * consider for field-0 - more likely to contain 0-byte duplicates this might
+ * also apply for field-1:
+ * fields : 00 - field 0-byte delta
+ * 01 - field 1-byte delta
+ * 10 - field 2-byte delta
+ * 11 - field 8-byte literal long
+ */
+
+public class ReadBuffer {
+ public final static byte BYTE = 0x00;
+ public final static byte SHORT = 0x01;
+ public final static byte INT = 0x10;
+ public final static byte LONG = 0x11;
+
+ private ByteBuffer bb;
+ private Tuple previous;
+
+ public ReadBuffer(ByteBuffer bb) {
+ this.previous = new Tuple();
+ this.bb = bb;
+ }
+
+ public Tuple read(int position) {
+ bb.position(position);
+ return readNext();
+ }
+
+ public Tuple readNext() {
+ byte header = bb.get();
+ Tuple current = new Tuple(previous);
+
+ readField(current, 0);
+ readField(current, 1);
+ readField(current, 2);
+ readField(current, 3);
+
+ previous = current;
+ return current;
+ }
+
+ private void readField(Tuple t, int field) {
+ switch ((header >> field) & 0x03) {
+// case BYTE: current[field] = previous + bb.get(); break;
+ case BYTE: current.setField(field, bb.get()); break;
+ case SHORT: current.setField(field, bb.getShort()); break;
+ case INT: current.setField(field, bb.getInt()); break;
+ case LONG: current.setField(field, bb.getLong()); break;
+ }
+ }
+}
Added: projects/xa2/draft/stuff/WriteBuffer.java
===================================================================
--- projects/xa2/draft/stuff/WriteBuffer.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/draft/stuff/WriteBuffer.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,104 @@
+/*
+ * (c) Dec 2006 Andrae Muys - Licenced GPL v2
+ */
+
+/**
+ * Basic SL-structure.
+ *
+ * 1-byte header : 2-bits-field-3, 2-bits-field-2, 2-bits-field-1, 2-bits-field-0
+ * fields : 00 - field 1-byte delta
+ * 01 - field 2-byte delta
+ * 10 - field 4-byte delta
+ * 11 - field 8-byte literal long
+ * consider for field-0 - more likely to contain 0-byte duplicates this might
+ * also apply for field-1:
+ * fields : 00 - field 0-byte delta
+ * 01 - field 1-byte delta
+ * 10 - field 2-byte delta
+ * 11 - field 8-byte literal long
+ */
+
+public class WriteBuffer {
+ public final static byte BYTE = 0x00;
+ public final static byte SHORT = 0x01;
+ public final static byte INT = 0x10;
+ public final static byte LONG = 0x11;
+
+ private ByteBuffer bb;
+ private Tuple previous;
+
+ public WriteBuffer() {
+ this.previous = new Tuple();
+ }
+
+ public void reset(ByteBuffer bb) {
+ this.bb = bb;
+ }
+
+ public boolean write(Tuple tuple) {
+ if (bb == null) {
+ return false;
+ }
+
+ long[] delta = { 0L, 0L, 0L, 0L };
+ tuple.calcDelta(delta, previous);
+
+ byte[] header = { 0, 0, 0, 0 };
+ calcHeader(header, delta);
+
+ if (tupleLength(header) > bb.remaining()) {
+ return false;
+ }
+
+ writeTuple(header, delta);
+
+ previous = tuple;
+ }
+
+ protected void writeTuple(byte[] header, long[] delta) {
+ bb.put((header[3] << 6) &
+ (header[2] << 4) &
+ (header[1] << 2) &
+ (header[0] << 0));
+
+ writeField(header[0], delta[0]);
+ writeField(header[1], delta[1]);
+ writeField(header[2], delta[2]);
+ writeField(header[3], delta[3]);
+ }
+
+ private void calcHeader(byte[] fields, long[] delta) {
+ fields[0] = calcField(delta[0]);
+ fields[1] = calcField(delta[1]);
+ fields[2] = calcField(delta[2]);
+ fields[3] = calcField(delta[3]);
+ }
+
+ private byte calcField(long delta) {
+ if (delta <= Byte.MAX_VALUE) {
+ return BYTE;
+ } else if (delta <= Short.MAX_VALUE) {
+ return SHORT;
+ } else if (delta <= Int.MAX_VALUE) {
+ return INT;
+ } else {
+ return LONG;
+ }
+ }
+
+ private void writeField(byte header, long field) {
+ switch (header) {
+ case BYTE: bb.put(field); break;
+ case SHORT: bb.putShort(field); break;
+ case INT: bb.putInt(field); break;
+ case LONG: bb.putLong(field); break;
+ }
+ }
+
+ private int tupleLength(byte[] header) {
+ return fieldLength(header[0]) +
+ fieldLength(header[1]) +
+ fieldLength(header[2]) +
+ fieldLength(header[3]);
+ }
+}
Added: projects/xa2/lib/ant
===================================================================
--- projects/xa2/lib/ant 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/lib/ant 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,326 @@
+#! /bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Extract launch and ant arguments, (see details below).
+ant_exec_args=
+no_config=false
+use_jikes_default=false
+ant_exec_debug=false
+show_help=false
+for arg in "$@" ; do
+ if [ "$arg" = "--noconfig" ] ; then
+ no_config=true
+ elif [ "$arg" = "--usejikes" ] ; then
+ use_jikes_default=true
+ elif [ "$arg" = "--execdebug" ] ; then
+ ant_exec_debug=true
+ elif [ my"$arg" = my"--h" -o my"$arg" = my"--help" ] ; then
+ show_help=true
+ ant_exec_args="$ant_exec_args -h"
+ else
+ if [ my"$arg" = my"-h" -o my"$arg" = my"-help" ] ; then
+ show_help=true
+ fi
+ ant_exec_args="$ant_exec_args \"$arg\""
+ fi
+done
+
+# Source/default ant configuration
+if $no_config ; then
+ rpm_mode=false
+ usejikes=$use_jikes_default
+else
+ # load system-wide ant configuration (ONLY if ANT_HOME has NOT been set)
+ if [ -z "$ANT_HOME" -o "$ANT_HOME" = "/usr/share/ant" ]; then
+ if [ -f "/etc/ant.conf" ] ; then
+ . /etc/ant.conf
+ fi
+ fi
+
+ # load user ant configuration
+ if [ -f "$HOME/.ant/ant.conf" ] ; then
+ . $HOME/.ant/ant.conf
+ fi
+ if [ -f "$HOME/.antrc" ] ; then
+ . "$HOME/.antrc"
+ fi
+
+ # provide default configuration values
+ if [ -z "$rpm_mode" ] ; then
+ rpm_mode=false
+ fi
+ if [ -z "$usejikes" ] ; then
+ usejikes=$use_jikes_default
+ fi
+fi
+
+# Setup Java environment in rpm mode
+if $rpm_mode ; then
+ if [ -f /usr/share/java-utils/java-functions ] ; then
+ . /usr/share/java-utils/java-functions
+ set_jvm
+ set_javacmd
+ fi
+fi
+
+# OS specific support. $var _must_ be set to either true or false.
+cygwin=false;
+darwin=false;
+case "`uname`" in
+ CYGWIN*) cygwin=true ;;
+ Darwin*) darwin=true
+ if [ -z "$JAVA_HOME" ] ; then
+ JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Home
+ fi
+ ;;
+esac
+
+if [ -z "$ANT_HOME" -o ! -d "$ANT_HOME" ] ; then
+ ## resolve links - $0 may be a link to ant's home
+ PRG="$0"
+ progname=`basename "$0"`
+
+ # need this for relative symlinks
+ while [ -h "$PRG" ] ; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`"/$link"
+ fi
+ done
+
+ ANT_HOME=`dirname "$PRG"`/..
+
+ # make it fully qualified
+ ANT_HOME=`cd "$ANT_HOME" && pwd`
+fi
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin ; then
+ [ -n "$ANT_HOME" ] &&
+ ANT_HOME=`cygpath --unix "$ANT_HOME"`
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+fi
+
+# set ANT_LIB location
+ANT_LIB="${ANT_HOME}/lib"
+
+if [ -z "$JAVACMD" ] ; then
+ if [ -n "$JAVA_HOME" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ JAVACMD="$JAVA_HOME/jre/sh/java"
+ elif [ -x "$JAVA_HOME/jre/bin/java" ] ; then
+ JAVACMD="$JAVA_HOME/jre/bin/java"
+ else
+ JAVACMD="$JAVA_HOME/bin/java"
+ fi
+ else
+ JAVACMD=`which java 2> /dev/null `
+ if [ -z "$JAVACMD" ] ; then
+ JAVACMD=java
+ fi
+ fi
+fi
+
+if [ ! -x "$JAVACMD" ] ; then
+ echo "Error: JAVA_HOME is not defined correctly."
+ echo " We cannot execute $JAVACMD"
+ exit 1
+fi
+
+# Build local classpath using just the launcher in non-rpm mode or
+# use the Jpackage helper in rpm mode with basic and default jars
+# specified in the ant.conf configuration. Because the launcher is
+# used, libraries linked in ANT_HOME/lib will also be included, but this
+# is discouraged as it is not java-version safe. A user should
+# request optional jars and their dependencies via the OPT_JAR_LIST
+# variable
+if $rpm_mode && [ -x /usr/bin/build-classpath ] ; then
+ LOCALCLASSPATH="$(/usr/bin/build-classpath ant ant-launcher jaxp_parser_impl xml-commons-apis)"
+
+ # If no optional jars have been specified then build the default list
+ if [ -z "$OPT_JAR_LIST" ] ; then
+ for file in /etc/ant.d/*; do
+ if [ -f "$file" ]; then
+ case "$file" in
+ *~) ;;
+ *#*) ;;
+ *.rpmsave) ;;
+ *.rpmnew) ;;
+ *)
+ for dep in `cat "$file"`; do
+ case "$OPT_JAR_LIST" in
+ *"$dep"*) ;;
+ *) OPT_JAR_LIST="$OPT_JAR_LIST${OPT_JAR_LIST:+ }$dep"
+ esac
+ done
+ esac
+ fi
+ done
+ fi
+
+ # If the user requested to try to add some other jars to the classpath
+ if [ -n "$OPT_JAR_LIST" ] ; then
+ _OPTCLASSPATH="$(/usr/bin/build-classpath $OPT_JAR_LIST 2> /dev/null)"
+ if [ -n "$_OPTCLASSPATH" ] ; then
+ LOCALCLASSPATH="$LOCALCLASSPATH:$_OPTCLASSPATH"
+ fi
+ fi
+
+ # Explicitly add javac path to classpath, assume JAVA_HOME set
+ # properly in rpm mode
+ if [ -f "$JAVA_HOME/lib/tools.jar" ] ; then
+ LOCALCLASSPATH="$LOCALCLASSPATH:$JAVA_HOME/lib/tools.jar"
+ fi
+ if [ -f "$JAVA_HOME/lib/classes.zip" ] ; then
+ LOCALCLASSPATH="$LOCALCLASSPATH:$JAVA_HOME/lib/classes.zip"
+ fi
+
+ # if CLASSPATH_OVERRIDE env var is set, LOCALCLASSPATH will be
+ # user CLASSPATH first and ant-found jars after.
+ # In that case, the user CLASSPATH will override ant-found jars
+ #
+ # if CLASSPATH_OVERRIDE is not set, we'll have the normal behaviour
+ # with ant-found jars first and user CLASSPATH after
+ if [ -n "$CLASSPATH" ] ; then
+ # merge local and specified classpath
+ if [ -z "$LOCALCLASSPATH" ] ; then
+ LOCALCLASSPATH="$CLASSPATH"
+ elif [ -n "$CLASSPATH_OVERRIDE" ] ; then
+ LOCALCLASSPATH="$CLASSPATH:$LOCALCLASSPATH"
+ else
+ LOCALCLASSPATH="$LOCALCLASSPATH:$CLASSPATH"
+ fi
+
+ # remove class path from launcher -cp option
+ CLASSPATH=""
+ fi
+else
+ # not using rpm_mode; use launcher to determine classpaths
+ if [ -z "$LOCALCLASSPATH" ] ; then
+ LOCALCLASSPATH=$ANT_LIB/ant-launcher.jar
+ else
+ LOCALCLASSPATH=$ANT_LIB/ant-launcher.jar:$LOCALCLASSPATH
+ fi
+fi
+
+if [ -n "$JAVA_HOME" ] ; then
+ # OSX hack to make Ant work with jikes
+ if $darwin ; then
+ OSXHACK="${JAVA_HOME}/../Classes"
+ if [ -d "${OSXHACK}" ] ; then
+ for i in "${OSXHACK}"/*.jar
+ do
+ JIKESPATH="$JIKESPATH:$i"
+ done
+ fi
+ fi
+fi
+
+# Allow Jikes support (off by default)
+if $usejikes; then
+ ANT_OPTS="$ANT_OPTS -Dbuild.compiler=jikes"
+fi
+
+# For Cygwin, switch paths to appropriate format before running java
+# For PATHs convert to unix format first, then to windows format to ensure
+# both formats are supported. Probably this will fail on directories with ;
+# in the name in the path. Let's assume that paths containing ; are more
+# rare than windows style paths on cygwin.
+if $cygwin; then
+ if [ "$OS" = "Windows_NT" ] && cygpath -m .>/dev/null 2>/dev/null ; then
+ format=mixed
+ else
+ format=windows
+ fi
+ ANT_HOME=`cygpath --$format "$ANT_HOME"`
+ ANT_LIB=`cygpath --$format "$ANT_LIB"`
+ JAVA_HOME=`cygpath --$format "$JAVA_HOME"`
+ LCP_TEMP=`cygpath --path --unix "$LOCALCLASSPATH"`
+ LOCALCLASSPATH=`cygpath --path --$format "$LCP_TEMP"`
+ if [ -n "$CLASSPATH" ] ; then
+ CP_TEMP=`cygpath --path --unix "$CLASSPATH"`
+ CLASSPATH=`cygpath --path --$format "$CP_TEMP"`
+ fi
+ CYGHOME=`cygpath --$format "$HOME"`
+fi
+
+# Show script help if requested
+if $show_help ; then
+ echo $0 '[script options] [options] [target [target2 [target3] ..]]'
+ echo 'Script Options:'
+ echo ' --help, --h print this message and ant help'
+ echo ' --noconfig suppress sourcing of /etc/ant.conf,'
+ echo ' $HOME/.ant/ant.conf, and $HOME/.antrc'
+ echo ' configuration files'
+ echo ' --usejikes enable use of jikes by default, unless'
+ echo ' set explicitly in configuration files'
+ echo ' --execdebug print ant exec line generated by this'
+ echo ' launch script'
+ echo ' '
+fi
+# add a second backslash to variables terminated by a backslash under cygwin
+if $cygwin; then
+ case "$ANT_HOME" in
+ *\\ )
+ ANT_HOME="$ANT_HOME\\"
+ ;;
+ esac
+ case "$CYGHOME" in
+ *\\ )
+ CYGHOME="$CYGHOME\\"
+ ;;
+ esac
+ case "$JIKESPATH" in
+ *\\ )
+ JIKESPATH="$JIKESPATH\\"
+ ;;
+ esac
+ case "$LOCALCLASSPATH" in
+ *\\ )
+ LOCALCLASSPATH="$LOCALCLASSPATH\\"
+ ;;
+ esac
+ case "$CLASSPATH" in
+ *\\ )
+ CLASSPATH="$CLASSPATH\\"
+ ;;
+ esac
+fi
+# Execute ant using eval/exec to preserve spaces in paths,
+# java options, and ant args
+ant_sys_opts=
+if [ -n "$CYGHOME" ]; then
+ if [ -n "$JIKESPATH" ]; then
+ ant_sys_opts="-Djikes.class.path=\"$JIKESPATH\" -Dcygwin.user.home=\"$CYGHOME\""
+ else
+ ant_sys_opts="-Dcygwin.user.home=\"$CYGHOME\""
+ fi
+else
+ if [ -n "$JIKESPATH" ]; then
+ ant_sys_opts="-Djikes.class.path=\"$JIKESPATH\""
+ fi
+fi
+ant_exec_command="exec \"$JAVACMD\" $ANT_OPTS -classpath \"$LOCALCLASSPATH\" -Dant.home=\"$ANT_HOME\" -Dant.library.dir=\"$ANT_LIB\" $ant_sys_opts org.apache.tools.ant.launch.Launcher $ANT_ARGS -cp \"$CLASSPATH\" $ant_exec_args"
+if $ant_exec_debug ; then
+ echo $ant_exec_command
+fi
+eval $ant_exec_command
Property changes on: projects/xa2/lib/ant
___________________________________________________________________
Name: svn:executable
+ *
Added: projects/xa2/lib/ant-junit.jar
===================================================================
(Binary files differ)
Property changes on: projects/xa2/lib/ant-junit.jar
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: projects/xa2/lib/ant-launcher.jar
===================================================================
(Binary files differ)
Property changes on: projects/xa2/lib/ant-launcher.jar
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: projects/xa2/lib/ant.jar
===================================================================
(Binary files differ)
Property changes on: projects/xa2/lib/ant.jar
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: projects/xa2/lib/junit-4.1.jar
===================================================================
(Binary files differ)
Property changes on: projects/xa2/lib/junit-4.1.jar
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: projects/xa2/src/ByteBufferPool.java
===================================================================
--- projects/xa2/src/ByteBufferPool.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/ByteBufferPool.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,66 @@
+/*
+ * ByteBufferPool.java - Handles a pool of direct ByteBuffers.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.util.LinkedList;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+/**
+ * Represents a pool of direct ByteBuffers.
+ *
+ * @created 2007-02-06
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+
+public class ByteBufferPool {
+ private final int bufferSize;
+ private final int highWater;
+ private LinkedList<ByteBuffer> pool;
+
+ public ByteBufferPool(int bufferSize, int highWater) {
+ this.bufferSize = bufferSize;
+ this.highWater = highWater;
+ this.pool = new LinkedList<ByteBuffer>();
+ }
+
+ public synchronized ByteBuffer obtainBuffer() {
+ if (pool.isEmpty()) {
+ return (ByteBuffer)ByteBuffer.allocateDirect(bufferSize).clear();
+ } else {
+ return (ByteBuffer)pool.remove().clear();
+ }
+ }
+
+ public synchronized void returnBuffer(ByteBuffer bb) {
+ if (pool.size() < highWater) {
+ bb.order(ByteOrder.BIG_ENDIAN);
+ pool.addFirst(bb);
+ }
+ }
+
+ public int getBufferSize() {
+ return bufferSize;
+ }
+}
Added: projects/xa2/src/ComparableQuadStream.java
===================================================================
--- projects/xa2/src/ComparableQuadStream.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/ComparableQuadStream.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,34 @@
+/*
+ * ComparableQuadStream.java - A stream of quads, that can be compared.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * A stream of quads, that can be compared.
+ *
+ * @created 2007-02-13
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+
+public interface ComparableQuadStream extends
+ QuadStream, Comparable<ComparableQuadStream> { }
Added: projects/xa2/src/CompressedBlock.java
===================================================================
--- projects/xa2/src/CompressedBlock.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/CompressedBlock.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,360 @@
+/*
+ * CompressedBlock.java - Handles reading and writing a compressed block.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
+/**
+ * Represents a single disk-block of compressed quads.
+ *
+ * Basic structure.
+ * Each block has a 4 byte header consisting of the number of uncompressed quads
+ * stored compressed in this block.
+ *
+ * Each commpressed quad is stored as a 1 byte header, followed by 4 fields
+ * representing a delta relative to the same field in the previous quad.
+ *
+ * 1-byte header : 2-bits-field-3, 2-bits-field-2, 2-bits-field-1, 2-bits-field-0
+ * fields : 00 - field 1-byte delta
+ * 01 - field 2-byte delta
+ * 10 - field 4-byte delta
+ * 11 - field 8-byte literal long
+ * consider for field-0 - more likely to contain 0-byte duplicates this might
+ * also apply for field-1:
+ * fields : 00 - field 0-byte delta
+ * 01 - field 1-byte delta
+ * 10 - field 2-byte delta
+ * 11 - field 8-byte literal long
+ *
+ * The first quad of a block is stored relative to a virtual previous quad with
+ * the value [ 0 0 0 0 ].
+ *
+ * @created 2007-02-06
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+
+public class CompressedBlock {
+ private static final byte BYTE = 0x00;
+ private static final byte SHORT = 0x01;
+ private static final byte INT = 0x02;
+ private static final byte LONG = 0x03;
+
+ private static final boolean debug = false;
+
+ private ByteBufferPool bufferPool;
+
+ /** Due to reset this buffer is normally backed by disk. */
+ public CompressedBlock(ByteBufferPool bufferPool) {
+ this.bufferPool = bufferPool;
+ }
+
+ /**
+ * Write as many quads from the QuadStream as will fit in this block.
+ *
+ * Note: Assumes both beforeFirst() and at least one call to next() have been
+ * made on the QuadStream prior to call.
+ */
+ public boolean write(QuadStream quads, FileChannel fc) throws IOException {
+ ByteBuffer bb = bufferPool.obtainBuffer();
+ bb.clear();
+ bb.position(4);
+
+ if (prepareBlock(quads, bb)) {
+ bb.flip();
+ fc.write(bb);
+ bufferPool.returnBuffer(bb);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * A version of write() that is optimised for working on a long array.
+ */
+ public int write(long[] quads, int offset, int length, FileChannel fc)
+ throws IOException {
+ ByteBuffer bb = bufferPool.obtainBuffer();
+ bb.clear();
+ bb.position(4);
+
+ if (length == -1) {
+ length = quads.length - offset;
+ }
+ if (length < 0) {
+ throw new IllegalArgumentException("Attempt to write < 0 nodes");
+ }
+
+ if (quads.length < offset + length) {
+ throw new IllegalArgumentException(String.format(
+ "Insufficient quads provided to write q.len: %d, off: %d, len: %d",
+ quads.length, offset, length));
+ }
+
+ offset = prepareBlock(quads, offset, length, bb);
+
+ bb.flip();
+ fc.write(bb);
+ bufferPool.returnBuffer(bb);
+
+ return offset;
+ }
+
+ private boolean prepareBlock(QuadStream quads, ByteBuffer bb) {
+ long[] delta = new long[4];
+ long[] quad;
+ long[] prev = new long[4];
+
+ int quadCount = 0;
+ do {
+ quad = quads.getQuad();
+ if (quad == null) {
+ break; // Ran out of quads with the previous block.
+ }
+ if (quadCount == 0) {
+ System.arraycopy(quad, 0, delta, 0, 4);
+ } else {
+ LongArrayOperations.subtract(4, quad, 0, prev, 0, delta, 0);
+ }
+ if (!(delta[0] == 0 && delta[1] == 0 && delta[2] == 0 && delta[3] == 0)) {
+ if (!write(delta, bb)) {
+ break;
+ } else {
+ quadCount++;
+ }
+ }
+ System.arraycopy(quad, 0, prev, 0, 4);
+ } while(quads.next());
+
+ if (quadCount == 0) {
+ return false;
+ }
+
+ bb.putInt(0, quadCount);
+
+ while (bb.hasRemaining()) {
+ bb.put((byte)0x00);
+ }
+ return true;
+ }
+
+ /**
+ * A version of prepareBlock optimised for working on a long array.
+ */
+ private int prepareBlock(long[] quads, int start, int length, ByteBuffer bb) {
+ long[] delta = new long[4];
+
+ int quadCount = 0;
+ int offset = start;
+ while (offset <= start + length - 4) {
+ if (quadCount == 0) {
+ System.arraycopy(quads, offset, delta, 0, 4);
+ } else {
+ LongArrayOperations.subtract(4, quads, offset, quads, offset - 4, delta, 0);
+ }
+ if (delta[0] == 0 && delta[1] == 0 && delta[2] == 0 && delta[3] == 0) {
+ offset += 4;
+ continue;
+ } else if (!write(delta, bb)) {
+ break;
+ } else {
+ quadCount++;
+ offset += 4;
+ }
+ }
+
+ bb.putInt(0, quadCount);
+
+ while (bb.hasRemaining()) {
+ bb.put((byte)0x00);
+ }
+
+ return offset;
+ }
+
+ public boolean write(long[] delta, ByteBuffer bb) {
+ byte[] header = calcHeader(delta);
+
+ if (tupleLength(header) > bb.remaining()) {
+ return false;
+ }
+
+ writeTuple(header, delta, bb);
+
+ return true;
+ }
+
+ private byte[] calcHeader(long[] delta) {
+ byte[] fields = new byte[4];
+ fields[0] = calcField(delta[0]);
+ fields[1] = calcField(delta[1]);
+ fields[2] = calcField(delta[2]);
+ fields[3] = calcField(delta[3]);
+
+ return fields;
+ }
+
+ private byte calcField(long delta) {
+ if (delta <= Byte.MAX_VALUE && delta >= Byte.MIN_VALUE) {
+ return BYTE;
+ } else if (delta <= Short.MAX_VALUE && delta >= Short.MIN_VALUE) {
+ return SHORT;
+ } else if (delta <= Integer.MAX_VALUE && delta >= Integer.MIN_VALUE) {
+ return INT;
+ } else {
+ return LONG;
+ }
+ }
+
+ private int tupleLength(byte[] header) {
+ return fieldLength(header[0]) +
+ fieldLength(header[1]) +
+ fieldLength(header[2]) +
+ fieldLength(header[3]) + 1;
+ }
+
+ private int fieldLength(byte field) {
+ switch (field) {
+ case BYTE: return Byte.SIZE / 8;
+ case SHORT: return Short.SIZE / 8;
+ case INT: return Integer.SIZE / 8;
+ case LONG: return Long.SIZE / 8;
+ default: throw new IllegalArgumentException("Invalid field type: " + field);
+ }
+ }
+
+ protected void writeTuple(byte[] header, long[] delta, ByteBuffer bb) {
+ byte headerByte = (byte)((header[3] << 6) |
+ (header[2] << 4) |
+ (header[1] << 2) |
+ (header[0] << 0));
+
+ bb.put(headerByte);
+
+ writeField(header[0], delta[0], bb);
+ writeField(header[1], delta[1], bb);
+ writeField(header[2], delta[2], bb);
+ writeField(header[3], delta[3], bb);
+ }
+
+ private void writeField(byte header, long field, ByteBuffer bb) {
+ switch (header) {
+ case BYTE: bb.put((byte)field); break;
+ case SHORT: bb.putShort((short)field); break;
+ case INT: bb.putInt((int)field); break;
+ case LONG: bb.putLong(field); break;
+ }
+ }
+
+
+ public long[] read(FileChannel fc) throws IOException {
+ ByteBuffer bb = bufferPool.obtainBuffer();
+
+ while (bb.hasRemaining()) {
+ if (fc.read(bb) == -1) {
+ return null;
+ }
+ }
+
+ bb.flip();
+
+ int count = bb.getInt();
+ long[] quads = new long[count*4];
+
+ int offset = 0;
+ while (offset < count && read(bb, quads, (offset - 1) * 4, offset * 4)) {
+ offset++;
+ }
+
+ if (offset != count) {
+ throw new IllegalStateException(String.format(
+ "Read quads failed to match declared size read: %d declared %d",
+ offset, count));
+ }
+
+ bufferPool.returnBuffer(bb);
+
+ return quads;
+ }
+
+ private boolean read(ByteBuffer bb, long[] quads, int previous, int offset) {
+ if (!bb.hasRemaining()) {
+ return false;
+ }
+
+ byte headerByte = bb.get();
+ byte[] header = decodeHeader(headerByte);
+
+ // -1 because we already read the header byte.
+ if (bb.remaining() < tupleLength(header) - 1) {
+ return false;
+ }
+
+ long[] delta = readDelta(bb, header);
+
+ if (delta[0] == 0 && delta[1] == 0 && delta[2] == 0 && delta[3] == 0) {
+ throw new IllegalStateException("Overran end of data in block");
+ }
+
+ if (previous < 0) {
+ System.arraycopy(delta, 0, quads, offset, 4);
+ } else {
+ LongArrayOperations.add(4, quads, previous, delta, 0, quads, offset);
+ }
+
+ return true;
+ }
+
+ private byte[] decodeHeader(byte headerByte) {
+ byte[] header = new byte[4];
+ header[0] = (byte)((headerByte >> 0) & 0x03);
+ header[1] = (byte)((headerByte >> 2) & 0x03);
+ header[2] = (byte)((headerByte >> 4) & 0x03);
+ header[3] = (byte)((headerByte >> 6) & 0x03);
+
+ return header;
+ }
+
+ private long[] readDelta(ByteBuffer bb, byte[] header) {
+ long[] delta = new long[4];
+ delta[0] = readField(bb, header[0]);
+ delta[1] = readField(bb, header[1]);
+ delta[2] = readField(bb, header[2]);
+ delta[3] = readField(bb, header[3]);
+
+ return delta;
+ }
+
+ private long readField(ByteBuffer bb, byte field) {
+ switch (field) {
+ case BYTE: return (long)bb.get();
+ case SHORT: return (long)bb.getShort();
+ case INT: return (long)bb.getInt();
+ case LONG: return (long)bb.getLong();
+ default: throw new IllegalArgumentException("Invalid field: " + field);
+ }
+ }
+}
Added: projects/xa2/src/CompressedBlockSeq.java
===================================================================
--- projects/xa2/src/CompressedBlockSeq.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/CompressedBlockSeq.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,149 @@
+/*
+ * CompressedBlockSeq.java - Manages a sequence of compressed blocks to/from
+ * disk.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * Manages a sequence of compressed blocks to/from disk.
+ *
+ * @created 2007-02-06
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.io.FileOutputStream;
+import java.nio.channels.FileChannel;
+
+public class CompressedBlockSeq {
+ private File file;
+ private ByteBufferPool bufferPool;
+ private RandomAccessFile raf;
+ private FileChannel fc;
+
+ public CompressedBlockSeq(File file, ByteBufferPool bufferPool) {
+ this.file = file;
+ this.bufferPool = bufferPool;
+ }
+
+ public void delete() throws IOException {
+ try {
+ close();
+ } finally {
+ file.delete();
+ file = null;
+ }
+ }
+
+ // FIXME:This shares alot of structure with write(long[],int) refactor.
+ // FIXME:Will need to return a list of CompressedBlock's for indexing to work
+ public void write(QuadStream quads) throws IOException {
+ FileOutputStream fos = new FileOutputStream(file);
+ FileChannel fc = fos.getChannel();
+ try {
+ CompressedBlock block = new CompressedBlock(bufferPool);
+ if (quads.next()) {
+ while (block.write(quads, fc)) ; // No loop body required
+ } else {
+ // FIXME: This is an empty seq' what to do here?
+ }
+ } finally {
+ fc.force(true);
+ fos.close();
+ }
+ }
+
+ // Watch out here this isn't threadsafe yet - will need to make it so.
+ public void write(long[] quads, int length) throws IOException {
+ if (length > quads.length) {
+ throw new IllegalArgumentException("Attempt to write more bytes than data");
+ }
+
+ if (length == 0) {
+ return;
+ }
+
+ QuadQSort.sort(quads, 0, length/4);
+
+ FileOutputStream fos = new FileOutputStream(file);
+ FileChannel fc = fos.getChannel();
+ try {
+ CompressedBlock block = new CompressedBlock(bufferPool);
+
+ int offset = 0;
+ while (offset < length) {
+ offset = block.write(quads, offset, length - offset, fc);
+ }
+ } finally {
+ fc.force(true);
+ fos.close();
+ }
+ }
+
+ // FIXME: Opening file anew for every block!?
+ public long[] readBlock(int i) throws IOException {
+ if (file == null) {
+ throw new IllegalStateException("Attempt to read deleted sequence");
+ } else if (raf == null) {
+ raf = new RandomAccessFile(file, "r");
+ fc = raf.getChannel();
+ } else if (fc == null) {
+ fc = raf.getChannel();
+ }
+ try {
+ CompressedBlock block = new CompressedBlock(bufferPool);
+
+ fc.position(bufferPool.getBufferSize() * i);
+
+ return block.read(fc);
+ } finally {
+ close();
+ }
+ }
+
+ public void close() throws IOException {
+ try {
+ if (raf != null) {
+ raf.close();
+ }
+ } finally {
+ raf = null;
+ fc = null;
+ }
+ }
+
+ public void finalize() {
+ try {
+ close();
+ } catch (IOException ei) {
+ System.err.println("Error closing file in finalizer.");
+ ei.printStackTrace();
+ }
+ }
+
+ public File getFile() {
+ return file;
+ }
+}
Added: projects/xa2/src/CompressedBlockSeqUnitTest.java
===================================================================
--- projects/xa2/src/CompressedBlockSeqUnitTest.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/CompressedBlockSeqUnitTest.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,156 @@
+/*
+ * CompressedBlockSeqUnitTest.java - Unit Tests for CompressedBlockSeq.java
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Unit Tests for CompressedBlockSeq.
+ *
+ * @created 2007-02-06
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class CompressedBlockSeqUnitTest extends TestCase {
+ public static final String testfile1 = "test/testcseq";
+
+ private File test1;
+ private ByteBufferPool bufferPool;
+
+ public CompressedBlockSeqUnitTest(String name) {
+ super(name);
+ }
+
+ public static Test suite() {
+ TestSuite suite = new TestSuite();
+ suite.addTest(new CompressedBlockSeqUnitTest("testWrite1"));
+ suite.addTest(new CompressedBlockSeqUnitTest("testWriteRead1"));
+ suite.addTest(new CompressedBlockSeqUnitTest("testWriteRead2"));
+ return suite;
+ }
+
+ public void main(String args[]) {
+ junit.textui.TestRunner.run(suite());
+ }
+
+ public void setUp() throws Exception {
+ test1 = setupFile(testfile1);
+ bufferPool = new ByteBufferPool(1*1024, 10);
+ }
+
+ private File setupFile(String filename) throws Exception {
+ File file = new File(filename);
+ if (file.exists()) {
+ file.delete();
+ } else {
+ if (!file.getParentFile().exists()) {
+ file.getParentFile().mkdirs();
+ }
+ }
+
+ return file;
+ }
+
+ public void tearDown() throws Exception {
+// test1.delete();
+ }
+
+ private void assertEquals(String msg, long[] expected, int offset, long[] received)
+ throws Exception {
+ assertTrue(msg + " : Lengths", expected.length - offset >= received.length);
+ for (int i = 0; i < received.length; i++) {
+ assertEquals(String.format(msg + " : expected[%d]==received[%d]", i + offset, i),
+ expected[i + offset], received[i]);
+ }
+ }
+
+ public long[] getTestset1() {
+ long[] data = new long[10000];
+ for (int i = 0; i < data.length; i++) {
+ data[i] = i + 1;
+ }
+
+ return data;
+ }
+
+ private long[] getTestset2() {
+ long[] data = new long[10000]; // 2500 quads of non-uniform non-regular data.
+ data[0] = 2; data[1] = 3; data[2] = 4; data[3] = 5;
+ for (int i = 4; i < 10000; i+=4) {
+ data[i+0] = data[i-4] + (i*11123L)%((long)Integer.MAX_VALUE*16);
+ data[i+1] = data[i-3] + (i*11123L)%((long)Integer.MAX_VALUE*8);
+ data[i+2] = data[i-2] + (i*11123L)%((long)Integer.MAX_VALUE*4);
+ data[i+3] = data[i-1] + (i*11123L)%((long)Integer.MAX_VALUE*2);
+ }
+
+ return data;
+ }
+
+ public void testWrite1() throws Exception {
+ CompressedBlockSeq seq = new CompressedBlockSeq(test1, bufferPool);
+ long[] data = getTestset1();
+ seq.write(data, data.length);
+ }
+
+ public void testWriteRead1() throws Exception {
+ CompressedBlockSeq seq = new CompressedBlockSeq(test1, bufferPool);
+ long[] testdata = getTestset1();
+ seq.write(testdata, testdata.length);
+
+ int offset = 0;
+ for (int blocknumber = 0;;blocknumber++) {
+ long[] data = seq.readBlock(blocknumber);
+ if (data == null) {
+ break;
+ }
+ assertEquals("testWriteRead1", testdata, offset, data);
+ offset += data.length;
+ }
+
+ assertEquals("Not enough data read", testdata.length, offset);
+ }
+
+ public void testWriteRead2() throws Exception {
+ CompressedBlockSeq seq = new CompressedBlockSeq(test1, bufferPool);
+ long[] testdata = getTestset2();
+ seq.write(testdata, testdata.length);
+
+ int offset = 0;
+ for (int blocknumber = 0;;blocknumber++) {
+ long[] data = seq.readBlock(blocknumber);
+ if (data == null) {
+ break;
+ }
+ assertEquals("testWriteRead1", testdata, offset, data);
+ offset += data.length;
+ }
+
+ assertEquals("Not enough data read", testdata.length, offset);
+ }
+}
Added: projects/xa2/src/CompressedBlockUnitTest.java
===================================================================
--- projects/xa2/src/CompressedBlockUnitTest.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/CompressedBlockUnitTest.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,221 @@
+/*
+ * CompressedBlockUnitTest.java - Unit Tests for CompressedBlock.java
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Unit Tests for CompressedBlock.
+ *
+ * @created 2007-02-06
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class CompressedBlockUnitTest extends TestCase {
+ public static final String testfile1 = "test/testfile1";
+
+ private File test1;
+ private ByteBufferPool bufferPool;
+
+ public CompressedBlockUnitTest(String name) {
+ super(name);
+ }
+
+ public static Test suite() {
+ TestSuite suite = new TestSuite();
+ suite.addTest(new CompressedBlockUnitTest("testWrite1"));
+ suite.addTest(new CompressedBlockUnitTest("testWrite2"));
+ suite.addTest(new CompressedBlockUnitTest("testWrite3"));
+ suite.addTest(new CompressedBlockUnitTest("testWriteRead1"));
+ suite.addTest(new CompressedBlockUnitTest("testWriteRead2"));
+ suite.addTest(new CompressedBlockUnitTest("testWriteRead3"));
+ suite.addTest(new CompressedBlockUnitTest("testWriteRead4"));
+ return suite;
+ }
+
+ public void main(String args[]) {
+ junit.textui.TestRunner.run(suite());
+ }
+
+ public void setUp() throws Exception {
+ test1 = setupFile(testfile1);
+ bufferPool = new ByteBufferPool(8*1024, 10);
+ }
+
+ private File setupFile(String filename) throws Exception {
+ File file = new File(filename);
+ if (file.exists()) {
+ file.delete();
+ } else {
+ if (!file.getParentFile().exists()) {
+ file.getParentFile().mkdirs();
+ }
+ }
+
+ return file;
+ }
+
+ public void tearDown() throws Exception {
+// test1.delete();
+ }
+
+ private void assertEquals(String msg, long[] expected, long[] received)
+ throws Exception {
+ assertEquals(msg + " : Lengths", expected.length, received.length);
+ for (int i = 0; i < expected.length; i++) {
+ assertEquals(String.format(msg + " : expected[%d]==received[%d]", i, i),
+ expected[i], received[i]);
+ }
+ }
+
+ private long[] getTestSet1() {
+ long[] data = new long[1000]; // 250 quads of uniform regular data.
+ for (int i = 1; i < 1000; i++) {
+ data[i-1] = i;
+ }
+
+ return data;
+ }
+
+ private long[] getTestSet2() {
+ long[] data = new long[1000]; // 250 quads of non-uniform regular data.
+ data[0] = 2; data[1] = 3; data[2] = 4; data[3] = 5;
+ for (int i = 4; i < 1000; i++) {
+ data[i] = data[i-4] + i;
+ }
+
+ return data;
+ }
+
+ private long[] getTestSet3() {
+ long[] data = new long[1000]; // 250 quads of non-uniform non-regular data.
+ data[0] = 2; data[1] = 3; data[2] = 4; data[3] = 5;
+ for (int i = 4; i < 1000; i+=4) {
+ data[i+0] = data[i-4] + (i*11123L)%((long)Integer.MAX_VALUE*16);
+ data[i+1] = data[i-3] + (i*11123L)%((long)Integer.MAX_VALUE*8);
+ data[i+2] = data[i-2] + (i*11123L)%((long)Integer.MAX_VALUE*4);
+ data[i+3] = data[i-1] + (i*11123L)%((long)Integer.MAX_VALUE*2);
+ }
+
+ return data;
+ }
+
+ private long[] getTestSet4() {
+ long[] data = new long[1000]; // 250 quads of non-uniform non-regular data.
+ data[0] = 2; data[1] = 3; data[2] = 4; data[3] = 5;
+ for (int i = 4; i < 1000; i+=4) {
+ data[i+0] = data[i-4] + i*16%65000;
+ data[i+1] = data[i-3] + i*160%65000;
+ data[i+2] = data[i-2] + i*1600%65000;
+ data[i+3] = data[i-1] + i*16000%65000;
+ }
+
+ return data;
+ }
+
+ public void testWrite1() throws Exception {
+ RandomAccessFile raf = new RandomAccessFile(test1, "rw");
+ try {
+ FileChannel fc = raf.getChannel();
+ CompressedBlock block = new CompressedBlock(bufferPool);
+
+ long[] data = getTestSet1();
+
+ assertEquals(1000, block.write(data, 0, data.length, fc));
+ } finally {
+ raf.close();
+ }
+ }
+
+ public void testWrite2() throws Exception {
+ RandomAccessFile raf = new RandomAccessFile(test1, "rw");
+ try {
+ FileChannel fc = raf.getChannel();
+ CompressedBlock block = new CompressedBlock(bufferPool);
+
+ long[] data = getTestSet2();
+
+ assertEquals(1000, block.write(data, 0, data.length, fc));
+ } finally {
+ raf.close();
+ }
+ }
+
+ public void testWrite3() throws Exception {
+ RandomAccessFile raf = new RandomAccessFile(test1, "rw");
+ try {
+ FileChannel fc = raf.getChannel();
+ CompressedBlock block = new CompressedBlock(bufferPool);
+
+ long[] data = getTestSet2();
+
+ assertEquals(1000, block.write(data, 0, data.length, fc));
+ } finally {
+ raf.close();
+ }
+ }
+
+ public void testWriteRead1() throws Exception {
+ testWriteRead("testWriteRead1", getTestSet1());
+ }
+
+ public void testWriteRead2() throws Exception {
+ testWriteRead("testWriteRead2", getTestSet2());
+ }
+
+ public void testWriteRead3() throws Exception {
+ testWriteRead("testWriteRead3", getTestSet3());
+ }
+
+ public void testWriteRead4() throws Exception {
+ testWriteRead("testWriteRead4", getTestSet4());
+ }
+
+ public void testWriteRead(String testname, long[] testset) throws Exception {
+// System.out.println("\nRunning " + testname + "\n");
+ RandomAccessFile raf = new RandomAccessFile(test1, "rw");
+ try {
+ FileChannel fc = raf.getChannel();
+ CompressedBlock block = new CompressedBlock(bufferPool);
+
+ assertEquals("Not all quads written to disk",
+ testset.length, block.write(testset, 0, testset.length, fc));
+
+ raf.close();
+ raf = new RandomAccessFile(test1, "r");
+ fc = raf.getChannel();
+
+ long[] result = block.read(fc);
+
+ assertEquals("Read result != Test data", testset, result);
+ } finally {
+ raf.close();
+ }
+ }
+}
Added: projects/xa2/src/DigitMerger.java
===================================================================
--- projects/xa2/src/DigitMerger.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/DigitMerger.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,77 @@
+/*
+ * DigitMerger.java - A merger of PhaseDigits
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * A merger for PhaseDigits.
+ *
+ * @created 2007-02-13
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+
+public class DigitMerger {
+ private PhaseDigit digit;
+ private File dir;
+ private String prefix;
+ private int exponent;
+ private int base;
+ private ByteBufferPool bufferPool;
+ private List<? super PhaseDigit> digits;
+
+ public DigitMerger(File dir, String prefix, int exponent, int base,
+ ByteBufferPool bufferPool, List<? super PhaseDigit> digits) {
+ this.digit = null;
+ this.dir = dir;
+ this.prefix = prefix;
+ this.exponent = exponent;
+ this.base = base;
+ this.bufferPool = bufferPool;
+ this.digits = digits;
+ }
+
+ public void merge(List<PhaseElement> elems) throws IOException {
+ if (digit == null) {
+ digit = new PhaseDigit(dir, prefix, elems, exponent, base, bufferPool,
+ new DigitMerger(dir, prefix, exponent + 1, base, bufferPool, digits));
+ digits.add(digit);
+ } else {
+ digit.increment(elems);
+ }
+ IOException ioe = null;
+ for (PhaseElement elem : elems) {
+ try {
+ elem.delete();
+ } catch (IOException ei) {
+ ioe = ei;
+ }
+ }
+ if (ioe != null) {
+ throw ioe;
+ }
+ }
+}
Added: projects/xa2/src/FileQuadStream.java
===================================================================
--- projects/xa2/src/FileQuadStream.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/FileQuadStream.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,99 @@
+/*
+ * FileQuadStream.java - A stream of quads streamed from disk.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+
+/**
+ * A stream of quads streamed from disk.
+ *
+ * @created 2007-02-08
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class FileQuadStream implements QuadStream {
+ FileInputStream file;
+ FileChannel fc;
+ ByteBuffer bb;
+ long[] quad;
+
+ public FileQuadStream(File file) throws IOException {
+ this.file = new FileInputStream(file);
+ this.fc = this.file.getChannel();
+ this.bb = ByteBuffer.allocate(32 * 1024);
+ }
+
+ public void beforeFirst() {
+ try {
+ this.quad = new long[4];
+ fc.position(0);
+ readNextBuffer();
+ } catch (IOException ei) {
+ throw new IllegalStateException("IOException in beforeFirst", ei);
+ }
+ }
+
+ private boolean readNextBuffer() throws IOException {
+ bb.clear();
+ if (fc.read(bb) == -1) {
+ return false;
+ }
+ while (bb.hasRemaining()) {
+ if (fc.read(bb) == -1) {
+ break;
+ }
+ }
+ bb.flip();
+
+ return true;
+ }
+
+ public boolean next() {
+ try {
+ if (quad != null && bb.remaining() >= 4 * 8) {
+ quad[0] = bb.getLong();
+ quad[1] = bb.getLong();
+ quad[2] = bb.getLong();
+ quad[3] = bb.getLong();
+ return true;
+ } else {
+ if (readNextBuffer()) {
+ return next();
+ } else {
+ quad = null;
+ return false;
+ }
+ }
+ } catch (IOException ei) {
+ throw new IllegalStateException("IOException in next()", ei);
+ }
+ }
+
+ public long[] getQuad() {
+ return quad;
+ }
+}
Added: projects/xa2/src/GenerateFileQuads.java
===================================================================
--- projects/xa2/src/GenerateFileQuads.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/GenerateFileQuads.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,127 @@
+/*
+ * GenerateFileQuads.java - Writes quads to disk raw.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.LongBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Comparator;
+import java.util.Random;
+import java.util.TreeSet;
+
+/**
+ * Writes quads to disk raw.
+ *
+ * @created 2007-02-08
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class GenerateFileQuads {
+ public static void writeSequential(int n, String filename) throws Exception {
+ FileOutputStream file = new FileOutputStream(filename);
+ long[] quads = new long[n];
+ for (int i = 0; i < n; i++) {
+ quads[i] = i + 1;
+ }
+ ByteBuffer bb = ByteBuffer.allocate(n*8);
+ LongBuffer lb = bb.asLongBuffer();
+ lb.put(quads);
+ FileChannel fc = file.getChannel();
+ fc.write(bb);
+ file.close();
+ }
+
+ public static void writeRandom(int n, String filename) throws Exception {
+ FileOutputStream file = new FileOutputStream(filename);
+ Random random = new Random();
+ long[] quads = new long[n];
+ TreeSet<long[]> previous = new TreeSet<long[]>(new Comparator<long[]>() {
+ public int compare(long[] lhs, long[] rhs) {
+ for (int i = 0; i < 4; i++) {
+ if (lhs[i] < rhs[i]) {
+ return -1;
+ } else if (lhs[i] > rhs[i]) {
+ return +1;
+ }
+ }
+ return 0;
+ }
+ });
+
+ int i = 0;
+ while (i < n / 4) {
+ long[] quad = new long[4];
+ for (int j = 0; j < 4; j++) {
+ quad[j] = Math.abs(random.nextLong() % (100L*1000L*1000L*1000L));
+ }
+ if (!previous.contains(quad)) {
+ System.arraycopy(quad, 0, quads, i*4, 4);
+ previous.add(quad);
+ i++;
+ }
+ }
+ ByteBuffer bb = ByteBuffer.allocate(n*8);
+ LongBuffer lb = bb.asLongBuffer();
+ lb.put(quads);
+ FileChannel fc = file.getChannel();
+ fc.write(bb);
+ file.close();
+ }
+
+ public static void writeSorted(int n, String random, String sorted) throws Exception {
+ FileInputStream rFile = new FileInputStream(random);
+ FileChannel input = rFile.getChannel();
+ ByteBuffer bb = ByteBuffer.allocateDirect(n*8);
+ bb.clear();
+ input.read(bb);
+ input.close();
+ rFile.close();
+ bb.flip();
+ LongBuffer lb = bb.asLongBuffer();
+ long[] quads = new long[n];
+ lb.get(quads);
+ QuadQSort.sort(quads, 0, n/4);
+ lb.clear();
+ lb.put(quads);
+ FileOutputStream sFile = new FileOutputStream(sorted);
+ FileChannel output = sFile.getChannel();
+ output.write(bb);
+ output.close();
+ sFile.close();
+ }
+
+ public static void main(String[] args) throws Exception {
+ writeSequential(10000, "testdata/10000sequential.quads");
+ writeRandom(10000, "testdata/10000random.quads");
+ writeSorted(10000, "testdata/10000random.quads", "testdata/10000sorted.quads");
+
+ writeRandom(100000, "testdata/100000random.quads");
+ writeSorted(100000, "testdata/100000random.quads", "testdata/100000sorted.quads");
+
+ writeRandom(10000000, "testdata/10000000random.quads");
+// writeSorted(10000000, "testdata/10000000random.quads", "testdata/10000000sorted.quads");
+ }
+}
Added: projects/xa2/src/GenerateLargeRandomQuads.java
===================================================================
--- projects/xa2/src/GenerateLargeRandomQuads.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/GenerateLargeRandomQuads.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,84 @@
+/*
+ * GenerateLargeRandomQuads.java - generates large random quads.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.RandomAccessFile;
+import java.io.IOException;
+import java.nio.MappedByteBuffer;
+import java.nio.LongBuffer;
+import java.nio.channels.FileChannel;
+import java.util.Comparator;
+import java.util.Random;
+import java.util.TreeSet;
+
+/**
+ * Generates large random quads.
+ *
+ * Each statement is assumed to add 1 new node unseen before.
+ * The location of the new node is distributed 2% 30% 20% 48%.
+ * The rest of the nodes are assumed to be evenly distributed from a set of
+ * nodes that starts at 10.
+ *
+ * Due to the allocation with each quad duplicates cannot occur.
+ *
+ * @created 2007-02-16
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class GenerateLargeRandomQuads {
+ public static void writeRandom(int n, String filename) throws Exception {
+ RandomAccessFile file = new RandomAccessFile(filename, "rw");
+ FileChannel fc = file.getChannel();
+ Random random = new Random();
+
+ MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_WRITE, 0, 8*n);
+ LongBuffer lb = bb.asLongBuffer();
+
+ long[] quad = new long[4];
+ long max = 9;
+ for (int i = 0; i < n / 4; i++) {
+ double ppos = random.nextDouble();
+ int pos;
+ max++;
+ if (ppos < 0.02) {
+ pos = 0;
+ } else if (ppos < 0.32) {
+ pos = 1;
+ } else if (ppos < 0.52) {
+ pos = 2;
+ } else {
+ pos = 3;
+ }
+ for (int j = 0; j < 4; j++) {
+ quad[j] = j == pos ? max : Math.abs(random.nextLong() % max);
+ }
+ lb.put(quad);
+ }
+ bb.force();
+ file.close();
+ }
+
+ public static void main(String[] args) throws Exception {
+ writeRandom(4*10000000, "testdata/40000000random.quads");
+ }
+}
Added: projects/xa2/src/IndexListener.java
===================================================================
--- projects/xa2/src/IndexListener.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/IndexListener.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,35 @@
+/*
+ * IndexListener.java - Provides the callback to notify a PhaseElement of index
+ * entries.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * Allows a PhaseElement to be notified of index entries.
+ *
+ * @created 2007-02-26
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public interface IndexListener {
+ public void index(long[] quad);
+}
Added: projects/xa2/src/LicensedTemplate.java
===================================================================
--- projects/xa2/src/LicensedTemplate.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/LicensedTemplate.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,35 @@
+/*
+ * LicensedTemplate.java - Template for classes
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * @created 2007-02-06
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class LicensedTemplate {
+ public String toString() {
+ return "Copright (c) 2006/7 Andrae Muys <andrae at netymon.com, All rights reserved\n" +
+ "Distributed the GNU General Public License version 2 (GPL)";
+ }
+}
Added: projects/xa2/src/LongArrayOperations.java
===================================================================
--- projects/xa2/src/LongArrayOperations.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/LongArrayOperations.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,71 @@
+/*
+ * QuadOperations.java - Various operations on arrays of longs.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.util.Comparator;
+import java.util.PriorityQueue;
+
+/**
+ * Various operations on arrays of longs.
+ *
+ * @created 2007-02-06
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006/7 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class LongArrayOperations {
+ /**
+ * diff - difference
+ * sub - subtrahend
+ * min - minuend
+ */
+ public static long[] subtract(int stride,
+ long[]sub, int subOff, long[] min, int minOff, long[] diff, int diffOff) {
+ for (int i = 0; i < stride; i++) {
+ diff[diffOff + i] = sub[subOff + i] - min[minOff + i];
+ }
+
+ return diff;
+ }
+
+ public static long[] add(int stride,
+ long[] lhs, int lhsOff, long[] rhs, int rhsOff, long[] sum, int sumOff) {
+ for (int i = 0; i < stride; i++) {
+ sum[sumOff + i] = lhs[lhsOff + i] + rhs[rhsOff + i];
+ }
+
+ return sum;
+ }
+
+ public static int compare(int stride, long[] lhs, int lhsOff, long[] rhs, int rhsOff) {
+ for (int i = 0; i < stride; i++) {
+ if (lhs[lhsOff + i] < rhs[rhsOff + i]) {
+ return -1;
+ } else if (lhs[lhsOff + i] > rhs[rhsOff + i]) {
+ return +1;
+ }
+ }
+
+ return 0;
+ }
+
+}
Added: projects/xa2/src/MergedQuadStreams.java
===================================================================
--- projects/xa2/src/MergedQuadStreams.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/MergedQuadStreams.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,108 @@
+/*
+ * MergedQuadStreams.java - Represents the merger of a number of QuadStreams.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.PriorityQueue;
+
+/**
+ * Represents the merger of a number of QuadStreams.
+ *
+ * @created 2007-02-13
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class MergedQuadStreams implements ComparableQuadStream {
+ private List<ComparableQuadStream> elements;
+ private PriorityQueue<ComparableQuadStream> queue;
+ private long[] quad;
+
+ public MergedQuadStreams(List<? extends ComparableQuadStream> elements) {
+ this.elements = new ArrayList<ComparableQuadStream>(elements);
+ this.queue = new PriorityQueue<ComparableQuadStream>();
+ }
+
+ public void add(ComparableQuadStream element) {
+ if (quad != null || !queue.isEmpty()) {
+ throw new IllegalArgumentException("Attempt to add element after beforeFirst()");
+ }
+ this.elements.add(element);
+ }
+
+ public void beforeFirst() {
+ queue.clear();
+ for (ComparableQuadStream elem : elements) {
+ elem.beforeFirst();
+ if (elem.next()) {
+ queue.offer(elem);
+ }
+ }
+ }
+
+ public boolean next() {
+ if (queue.isEmpty()) {
+ quad = null;
+ return false;
+ }
+ ComparableQuadStream elem = queue.poll();
+ quad = elem.getQuad();
+ if (elem.next()) {
+ queue.offer(elem);
+ }
+ return true;
+ }
+
+ public long[] getQuad() {
+ if (quad == null && !queue.isEmpty()) {
+ throw new IllegalArgumentException(
+ "Attempt to call getQuad() before calling beforeFirst()");
+ }
+ if (quad == null) {
+ return null;
+ }
+ long[] temp = new long[4];
+ System.arraycopy(quad, 0, temp, 0, 4);
+ return temp;
+ }
+
+ public int compareTo(ComparableQuadStream rhs) {
+ if (quad != null && rhs.getQuad() != null) {
+ return LongArrayOperations.compare(4, quad, 0, rhs.getQuad(), 0);
+ } else if (quad == null && rhs.getQuad() == null) {
+ return 0;
+ } else if (quad != null && rhs.getQuad() == null) {
+ return +1;
+ } else if (quad == null && rhs.getQuad() != null) {
+ return -1;
+ } else {
+ throw new IllegalStateException("Can't reach here, supresses compile error");
+ }
+ }
+
+ public String toString() {
+ return String.format("MergedQuadStreams<%d>:[%s : %s]",
+ System.identityHashCode(this), elements, queue);
+ }
+}
Added: projects/xa2/src/PhaseDigit.java
===================================================================
--- projects/xa2/src/PhaseDigit.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/PhaseDigit.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,229 @@
+/*
+ * PhaseDigit.java - Serialises a stream of quads into a phase element.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.PriorityQueue;
+
+/**
+ * Seralises a stream of quads into a phase element.
+ *
+ * For the moment implements Quads to allow reading the data back.
+ * Eventually it will provide a find() method that returns a slice.
+ *
+ * @created 2007-02-07
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class PhaseDigit implements ComparableQuadStream {
+ private ByteBufferPool bufferPool;
+ private ArrayList<PhaseElement> elements;
+ private PriorityQueue<PhaseElement> queue;
+ private long[] quad;
+ private int exponent;
+ private int base;
+ private DigitMerger merger;
+ private File dir;
+ private String prefix;
+
+ /**
+ * This constructor is used to create a phase-digit accessing an existing
+ * phase-element fileset.
+ */
+ public PhaseDigit(File dir, final String prefix, final int exponent,
+ ByteBufferPool bufferPool) throws IOException, PhaseDigitEmptyEscape {
+ if (!dir.exists()) {
+ throw new IllegalArgumentException("Nominated phase directory does not exist: "
+ + dir);
+ }
+ this.bufferPool = bufferPool;
+ elements = new ArrayList<PhaseElement>();
+ for (File file : getPhaseFiles(dir, prefix, exponent)) {
+ elements.add(new PhaseElement(file, bufferPool));
+ }
+
+ if (elements.size() == 0) {
+ throw new PhaseDigitEmptyEscape();
+ }
+
+ this.queue = new PriorityQueue<PhaseElement>();
+ }
+
+ /**
+ * This constructor is used to create a new phase-element corresponding to a
+ * quad-stream. Another constructor is going to be needed for opening an
+ * existing phase-element.
+ *
+ * Need to add merging here - migrate List{PhaseElements} to List{list{PhaseElements}} -
+ * create a new PhaseElement constructor (PhaseElement(long[])) and a new
+ * CBSeq.write(PhaseElement[]) method that performs the merge using a PriorityQueue.
+ *
+ * Note: size is measured in quads, _not_ longs.
+ */
+ public PhaseDigit(File dir, String prefix, QuadStream quads, int size, int base,
+ ByteBufferPool bufferPool, DigitMerger merger) throws IOException {
+ if (!dir.exists()) {
+ dir.mkdir();
+ } else if (!dir.isDirectory()) {
+ throw new IllegalArgumentException("PhaseDigit dir is not a directory");
+ }
+
+ if (getPhaseFiles(dir, prefix, 0).length > 0) {
+ throw new IllegalArgumentException("Stale phase-digit files found");
+ }
+
+ this.prefix = prefix;
+ this.exponent = 0;
+ this.base = base;
+ this.bufferPool = bufferPool;
+
+ quads.beforeFirst();
+ elements = new ArrayList<PhaseElement>();
+ for (;;) {
+ File file = File.createTempFile(prefix + "-" + exponent + "-", ".pd", dir);
+ PhaseElement pe = new PhaseElement(file, bufferPool);
+ if (pe.partialUnorderedWrite(quads, size)) {
+ elements.add(pe);
+ if (elements.size() == base) {
+ merger.merge(elements);
+ elements.clear();
+ }
+ } else {
+ pe.delete();
+ break;
+ }
+ }
+
+ this.queue = new PriorityQueue<PhaseElement>();
+ }
+
+ /**
+ * This constructor is used to create a new PhaseDigit from the merger of a
+ * less-significant PhaseDigit.
+ */
+ public PhaseDigit(File dir, String prefix, List<? extends PhaseElement> elements,
+ int exponent, int base, ByteBufferPool bufferPool, DigitMerger merger)
+ throws IOException {
+ if (!dir.exists()) {
+ throw new IllegalArgumentException("PhaseDigit dir does not exist");
+ } else if (!dir.isDirectory()) {
+ throw new IllegalArgumentException("PhaseDigit dir is not a directory");
+ }
+
+ if (getPhaseFiles(dir, prefix, exponent).length > 0) {
+ throw new IllegalArgumentException("Stale phase-digit files found");
+ }
+
+ this.exponent = exponent;
+ this.base = base;
+ this.merger = merger;
+ this.dir = dir;
+ this.prefix = prefix;
+
+ this.bufferPool = bufferPool;
+ this.elements = new ArrayList<PhaseElement>();
+
+ increment(elements);
+
+ this.queue = new PriorityQueue<PhaseElement>();
+ }
+
+ public void increment(List<? extends PhaseElement> elements) throws IOException {
+ MergedQuadStreams elems = new MergedQuadStreams(elements);
+ elems.beforeFirst();
+ File file = File.createTempFile(prefix + "-" + exponent + "-", ".pd", dir);
+ PhaseElement pe = new PhaseElement(file, bufferPool);
+ pe.fullOrderedWrite(elems);
+
+ this.elements.add(pe);
+ if (this.elements.size() == base) {
+ merger.merge(this.elements);
+ this.elements.clear();
+ }
+ }
+
+ private File[] getPhaseFiles(final File dir, final String prefix, final int exponent) {
+ return dir.listFiles(new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.startsWith(prefix + "-" + exponent);
+ }
+ });
+ }
+
+ public void beforeFirst() {
+ queue.clear();
+ for (PhaseElement element : elements) {
+ element.beforeFirst();
+ if (element.next()) {
+ queue.offer(element);
+ }
+ }
+ }
+
+
+ public boolean next() {
+ if (queue.isEmpty()) {
+ quad = null;
+ return false;
+ }
+ PhaseElement elem = queue.poll();
+ quad = elem.getQuad();
+ if (elem.next()) {
+ queue.offer(elem);
+ }
+ return true;
+ }
+
+ public long[] getQuad() {
+ if (quad == null && !queue.isEmpty()) {
+ throw new IllegalStateException(
+ "Attempt to call getQuad() before calling beforeFirst()");
+ }
+ long[] temp = new long[4];
+ System.arraycopy(this.quad, 0, temp, 0, 4);
+ return temp;
+ }
+
+ public int compareTo(ComparableQuadStream rhs) {
+ if (quad != null && rhs.getQuad() != null) {
+ return LongArrayOperations.compare(4, quad, 0, rhs.getQuad(), 0);
+ } else if (quad == null && rhs.getQuad() == null) {
+ return 0;
+ } else if (quad != null && rhs.getQuad() == null) {
+ return +1;
+ } else if (quad == null && rhs.getQuad() != null) {
+ return -1;
+ } else {
+ throw new IllegalStateException("Can't reach here, supresses compile error");
+ }
+ }
+
+ public String toString() {
+ return String.format("PhaseDigit<%d>:[%s : %d]",
+ System.identityHashCode(this), prefix, exponent);
+ }
+}
Added: projects/xa2/src/PhaseDigitEmptyEscape.java
===================================================================
--- projects/xa2/src/PhaseDigitEmptyEscape.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/PhaseDigitEmptyEscape.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,37 @@
+/*
+ * PhaseDigitEmptyEscape.java - Used as an escape continuation to indicate a
+ * PhaseDigit is empty.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * Used as an escape continuation to indicate a PhaseDigit is empty.
+ *
+ * @created 2007-02-06
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class PhaseDigitEmptyEscape extends Exception {
+ public PhaseDigitEmptyEscape() {
+ super("Escape Continuation");
+ }
+}
Added: projects/xa2/src/PhaseElement.java
===================================================================
--- projects/xa2/src/PhaseElement.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/PhaseElement.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,159 @@
+/*
+ * PhaseElement.java - Represents a PhaseElement backed by a CSeq.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Represents a PhaseElement backed by a CSeq.
+ *
+ * @created 2007-02-07
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class PhaseElement implements ComparableQuadStream {
+ private CompressedBlockSeq seq;
+ private long[] data;
+ private int blockNr;
+ private int index;
+
+ public PhaseElement(File file, ByteBufferPool bufferPool) throws IOException {
+ this.seq = new CompressedBlockSeq(file, bufferPool);
+ this.blockNr = -1;
+ }
+
+ public boolean partialUnorderedWrite(QuadStream quads, int size) throws IOException {
+ long[] data = new long[size * 4];
+ int offset = 0;
+ // At the start of every iteration 'offset' points to the first unused long.
+ while (offset < data.length) {
+ if (!quads.next()) {
+ break;
+ }
+ System.arraycopy(quads.getQuad(), 0, data, offset, 4);
+ offset +=4;
+ }
+
+ // offset == 0 iff quads.next() returned false on first iteration of loop.
+ if (offset == 0) {
+ return false;
+ }
+
+ seq.write(data, data.length);
+ return true;
+ }
+
+ public void fullOrderedWrite(QuadStream quads) throws IOException {
+ quads.beforeFirst();
+ this.seq.write(quads);
+ }
+
+ public void delete() throws IOException {
+ try {
+ seq.delete();
+ } finally {
+ seq = null;
+ data = null;
+ }
+ }
+
+ public void close() throws IOException {
+ seq.close();
+ seq = null;
+ data = null;
+ }
+
+ public void beforeFirst() {
+ if (seq == null) {
+ throw new IllegalStateException("Attempt to call beforeFirst on deleted element");
+ }
+ try {
+ this.blockNr = 0;
+ data = seq.readBlock(blockNr++);
+ index = 0;
+ } catch (IOException ei) {
+ throw new IllegalStateException("attempt to read first block from seq failed", ei);
+ }
+ }
+
+ public boolean next() {
+ if (data == null) {
+ return false;
+ }
+
+ if (index >= data.length) {
+ try {
+ data = seq.readBlock(blockNr++);
+ } catch (IOException ei) {
+ throw new IllegalStateException(
+ "Attempt to read block " + blockNr + " from seq failed", ei);
+ }
+ index = 0;
+ if (data == null) {
+ return false;
+ }
+ }
+
+ index += 4;
+
+ return true;
+ }
+
+ /**
+ * Result is reallocated on each call, safe to retain without clone/copy.
+ */
+ public long[] getQuad() {
+ if (blockNr == 0 && index == 0) {
+ throw new IllegalStateException(
+ "Attempt to call getQuad() before calling beforeFirst()");
+ }
+ if (data == null) {
+ return null;
+ }
+
+ long[] temp = new long[4];
+ System.arraycopy(data, index - 4, temp, 0, 4);
+
+ return temp;
+ }
+
+ public int compareTo(ComparableQuadStream rhs) {
+ if (getQuad() != null && rhs.getQuad() != null) {
+ return LongArrayOperations.compare(4, getQuad(), 0, rhs.getQuad(), 0);
+ } else if (getQuad() == null && rhs.getQuad() == null) {
+ return 0;
+ } else if (getQuad() != null && rhs.getQuad() == null) {
+ return +1;
+ } else if (getQuad() == null && rhs.getQuad() != null) {
+ return -1;
+ } else {
+ throw new IllegalStateException("Can't reach here, supresses compile error");
+ }
+ }
+
+ public String toString() {
+ return String.format("PhaseElement<%d>:[%s]",
+ System.identityHashCode(this), seq.getFile());
+ }
+}
Added: projects/xa2/src/PhaseElementUnitTest.java
===================================================================
--- projects/xa2/src/PhaseElementUnitTest.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/PhaseElementUnitTest.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,168 @@
+/*
+ * PhaseElementUnitTest.java - Unit Tests for PhaseElement.java
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Unit Tests for PhaseElement.
+ *
+ * @created 2007-02-14
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class PhaseElementUnitTest extends TestCase {
+ public static final String testfile1 = "test/testpe1";
+ public static final String testfile2 = "test/testpe2";
+
+ private File test1;
+ private File test2;
+ private ByteBufferPool bufferPool;
+
+ public PhaseElementUnitTest(String name) {
+ super(name);
+ }
+
+ public static Test suite() {
+ TestSuite suite = new TestSuite();
+ suite.addTest(new PhaseElementUnitTest("testWrite1"));
+ suite.addTest(new PhaseElementUnitTest("testWriteRead1"));
+ suite.addTest(new PhaseElementUnitTest("testRead1"));
+ suite.addTest(new PhaseElementUnitTest("testWrite2"));
+ suite.addTest(new PhaseElementUnitTest("testWriteRead2"));
+ suite.addTest(new PhaseElementUnitTest("testRead2"));
+ return suite;
+ }
+
+ public void main(String args[]) {
+ junit.textui.TestRunner.run(suite());
+ }
+
+ public void setUp() throws Exception {
+ test1 = setupFile(testfile1);
+ test2 = setupFile(testfile2);
+ bufferPool = new ByteBufferPool(1*1024, 10);
+ }
+
+ private File setupFile(String filename) throws Exception {
+ File file = new File(filename);
+ if (file.exists()) {
+ file.delete();
+ } else {
+ if (!file.getParentFile().exists()) {
+ file.getParentFile().mkdirs();
+ }
+ }
+
+ return file;
+ }
+
+ public void tearDown() throws Exception {
+// test1.delete();
+ }
+
+ private void assertEquals(String msg, QuadStream expected, QuadStream received)
+ throws Exception {
+ expected.beforeFirst();
+ received.beforeFirst();
+ int i = 0;
+ while (expected.next()) {
+ assertTrue("Expected longer quads", received.next());
+ long[] equad = expected.getQuad();
+ long[] rquad = received.getQuad();
+ for (int j = 0; j < 4; j++) {
+ if (equad[j] != rquad[j]) {
+ assertEquals(String.format(
+ "Quad[%d] mismatch - expected [ %d %d %d %d ] received [ %d %d %d %d ]\n", i,
+ equad[0], equad[1], equad[2], equad[3],
+ rquad[0], rquad[1], rquad[2], rquad[3]),
+ equad[j], rquad[j]);
+ }
+ }
+ i++;
+ }
+
+ assertFalse("Received more quads than expected", received.next());
+ }
+
+ public void testWrite1() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000sequential.quads"));
+ PhaseElement pe = new PhaseElement(test1, bufferPool);
+ pe.fullOrderedWrite(quads);
+ }
+
+ public void testWriteRead1() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000sequential.quads"));
+ PhaseElement pe = new PhaseElement(test1, bufferPool);
+ pe.fullOrderedWrite(quads);
+
+ assertEquals("PhaseElement does match quads", quads, pe);
+ }
+
+ public void testRead1() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000sequential.quads"));
+ PhaseElement pe1 = new PhaseElement(test1, bufferPool);
+ pe1.fullOrderedWrite(quads);
+ assertEquals("PhaseElement does match quads", quads, pe1);
+ pe1.close();
+
+ PhaseElement pe2 = new PhaseElement(test1, bufferPool);
+ assertEquals("PhaseElement does match quads", quads, pe2);
+ }
+
+ public void testWrite2() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000random.quads"));
+ PhaseElement pe = new PhaseElement(test2, bufferPool);
+ quads.beforeFirst();
+ pe.partialUnorderedWrite(quads, 2500);
+ }
+
+ public void testWriteRead2() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000random.quads"));
+ FileQuadStream expected = new FileQuadStream(new File("testdata/10000sorted.quads"));
+ PhaseElement pe = new PhaseElement(test2, bufferPool);
+ quads.beforeFirst();
+ pe.partialUnorderedWrite(quads, 2500);
+
+ assertEquals("PhaseElement does match quads", expected, pe);
+ }
+
+ public void testRead2() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000random.quads"));
+ FileQuadStream expected = new FileQuadStream(new File("testdata/10000sorted.quads"));
+ PhaseElement pe1 = new PhaseElement(test2, bufferPool);
+ quads.beforeFirst();
+ pe1.partialUnorderedWrite(quads, 2500);
+ assertEquals("PhaseElement does match quads", expected, pe1);
+ pe1.close();
+
+ PhaseElement pe2 = new PhaseElement(test2, bufferPool);
+ assertEquals("PhaseElement does match quads", expected, pe2);
+ }
+}
Added: projects/xa2/src/PhaseSet.java
===================================================================
--- projects/xa2/src/PhaseSet.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/PhaseSet.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,113 @@
+/*
+ * PhaseSet.java - Represents a PhaseSet
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Represents a PhaseSet
+ *
+ * @created 2007-02-13
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class PhaseSet implements ComparableQuadStream {
+ private static int nextPhaseNumber = 1;
+
+ private ComparableQuadStream phaseQuads;
+ private int phaseNumber;
+
+ /**
+ * Create a PhaseSet corresponding to a QuadStream.
+ */
+ public PhaseSet(final File dir, QuadStream quads, final int baseSize, final int base,
+ final ByteBufferPool bufferPool) throws IOException {
+ synchronized (PhaseSet.class) {
+ this.phaseNumber = nextPhaseNumber++;
+ }
+
+ List<PhaseDigit> digits = new ArrayList<PhaseDigit>();
+ PhaseDigit digit = new PhaseDigit(dir, "pe" + phaseNumber, quads, baseSize, base,
+ bufferPool, new DigitMerger(dir, "pe" + phaseNumber, 1, base, bufferPool, digits));
+
+ digits.add(digit);
+ phaseQuads = new MergedQuadStreams(digits);
+ }
+
+ /**
+ * Create a PhaseSet corresponding to a previously created phase.
+ */
+ public PhaseSet(final File dir, final int phaseNumber, final ByteBufferPool bufferPool)
+ throws IOException {
+ if (!dir.exists()) {
+ throw new IllegalArgumentException("PhaseSet directory does not exist: " + dir);
+ }
+
+ this.phaseNumber = phaseNumber;
+
+ List<PhaseDigit> digits = new ArrayList<PhaseDigit>();
+ File[] phaseFiles = dir.listFiles(new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.startsWith("pe" + phaseNumber);
+ }
+ });
+
+ for (int i = 0; i < phaseFiles.length; i++) {
+ try {
+ digits.add(new PhaseDigit(dir, "pe" + phaseNumber, i, bufferPool));
+ } catch (PhaseDigitEmptyEscape ep) {
+ // Escape continuation.
+ }
+ }
+
+ phaseQuads = new MergedQuadStreams(digits);
+ }
+
+ public int getPhaseNumber() {
+ return phaseNumber;
+ }
+
+ public void beforeFirst() {
+ phaseQuads.beforeFirst();
+ }
+
+ public boolean next() {
+ return phaseQuads.next();
+ }
+
+ public long[] getQuad() {
+ return phaseQuads.getQuad();
+ }
+
+ public int compareTo(ComparableQuadStream rhs) {
+ return phaseQuads.compareTo(rhs);
+ }
+
+ public String toString() {
+ return String.format("PhaseSet<%d>:[%d]", System.identityHashCode(this), phaseNumber);
+ }
+}
Added: projects/xa2/src/PhaseSetUnitTest.java
===================================================================
--- projects/xa2/src/PhaseSetUnitTest.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/PhaseSetUnitTest.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,213 @@
+/*
+ * PhaseSetUnitTest.java - Unit Tests for PhaseSet.java
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Unit Tests for PhaseSet.
+ *
+ * @created 2007-02-08
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class PhaseSetUnitTest extends TestCase {
+ private ByteBufferPool bufferPool;
+
+ public PhaseSetUnitTest(String name) {
+ super(name);
+ }
+
+ public static Test suite() {
+ TestSuite suite = new TestSuite();
+ suite.addTest(new PhaseSetUnitTest("testWrite1"));
+ suite.addTest(new PhaseSetUnitTest("testWriteRead1"));
+ suite.addTest(new PhaseSetUnitTest("testRead1"));
+ suite.addTest(new PhaseSetUnitTest("testWrite2"));
+ suite.addTest(new PhaseSetUnitTest("testWriteRead2"));
+ suite.addTest(new PhaseSetUnitTest("testRead2"));
+// suite.addTest(new PhaseSetUnitTest("testWrite3"));
+// suite.addTest(new PhaseSetUnitTest("testWriteRead3"));
+// suite.addTest(new PhaseSetUnitTest("testRead3"));
+// suite.addTest(new PhaseSetUnitTest("testWrite4"));
+// suite.addTest(new PhaseSetUnitTest("testWriteRead4"));
+// suite.addTest(new PhaseSetUnitTest("testRead4"));
+ return suite;
+ }
+
+ public void main(String args[]) {
+ junit.textui.TestRunner.run(suite());
+ }
+
+ public void setUp() throws Exception {
+ bufferPool = new ByteBufferPool(1*1024, 10);
+ }
+
+ public void tearDown() throws Exception {
+ }
+
+ private void assertEquals(String msg, QuadStream expected, QuadStream received)
+ throws Exception {
+ expected.beforeFirst();
+ received.beforeFirst();
+ int i = 0;
+ while (expected.next()) {
+ assertTrue("Expected longer quads", received.next());
+ long[] equad = expected.getQuad();
+ long[] rquad = received.getQuad();
+ for (int j = 0; j < 4; j++) {
+ if (equad[j] != rquad[j]) {
+ assertEquals(String.format(
+ "Quad[%d] mismatch - expected [ %d %d %d %d ] received [ %d %d %d %d ]\n", i,
+ equad[0], equad[1], equad[2], equad[3],
+ rquad[0], rquad[1], rquad[2], rquad[3]),
+ equad[j], rquad[j]);
+ }
+ }
+ i++;
+ }
+
+ assertFalse("Received more quads than expected", received.next());
+ }
+
+ public void testWrite1() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000sequential.quads"));
+ File dir = new File("test/petest1");
+ if (dir.exists()) {
+ dir.delete();
+ }
+ dir.mkdirs();
+
+ PhaseSet pe = new PhaseSet(dir, quads, 500, 4, bufferPool);
+ assertEquals("Should have phase number 1", 1, pe.getPhaseNumber());
+ }
+
+ public void testWriteRead1() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000sequential.quads"));
+ File dir = new File("test/petest1");
+
+ PhaseSet pe = new PhaseSet(dir, quads, 500, 4, bufferPool);
+
+ assertEquals("PhaseSet not equal to source quads", quads, pe);
+ }
+
+ public void testRead1() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000sequential.quads"));
+ PhaseSet pe = new PhaseSet(new File("test/petest1"), 1, bufferPool);
+
+ assertEquals("PhaseSet not equal to source quads", quads, pe);
+ }
+
+ public void testWrite2() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000random.quads"));
+ File dir = new File("test/petest1");
+ if (dir.exists()) {
+ dir.delete();
+ }
+ dir.mkdirs();
+
+ PhaseSet pe = new PhaseSet(dir, quads, 500, 4, bufferPool);
+ assertEquals("Should have phase number 3", 3, pe.getPhaseNumber());
+ }
+
+ public void testWriteRead2() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/10000random.quads"));
+ FileQuadStream expected = new FileQuadStream(new File("testdata/10000sorted.quads"));
+ File dir = new File("test/petest1");
+
+ PhaseSet pe = new PhaseSet(dir, quads, 500, 4, bufferPool);
+
+ assertEquals("PhaseSet not equal to source quads", expected, pe);
+ }
+
+ public void testRead2() throws Exception {
+ FileQuadStream expected = new FileQuadStream(new File("testdata/10000sorted.quads"));
+ PhaseSet pe = new PhaseSet(new File("test/petest1"), 3, bufferPool);
+
+ assertEquals("PhaseSet not equal to source quads", expected, pe);
+ }
+
+ public void testWrite3() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/100000random.quads"));
+ File dir = new File("test/petest1");
+ if (dir.exists()) {
+ dir.delete();
+ }
+ dir.mkdirs();
+
+ PhaseSet pe = new PhaseSet(dir, quads, 500, 4, bufferPool);
+ assertEquals("Should have phase number 5", 5, pe.getPhaseNumber());
+ }
+
+ public void testWriteRead3() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/100000random.quads"));
+ FileQuadStream expected = new FileQuadStream(new File("testdata/100000sorted.quads"));
+ File dir = new File("test/petest1");
+
+ PhaseSet pe = new PhaseSet(dir, quads, 500, 4, bufferPool);
+
+ assertEquals("PhaseSet not equal to source quads", expected, pe);
+ }
+
+ public void testRead3() throws Exception {
+ FileQuadStream expected = new FileQuadStream(new File("testdata/100000sorted.quads"));
+ PhaseSet pe = new PhaseSet(new File("test/petest1"), 5, bufferPool);
+
+ assertEquals("PhaseSet not equal to source quads", expected, pe);
+ }
+
+ public void testWrite4() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/100000random.quads"));
+ File dir = new File("test/petest1");
+ if (dir.exists()) {
+ dir.delete();
+ }
+ dir.mkdirs();
+
+ PhaseSet pe = new PhaseSet(dir, quads, 10*1024, 4, bufferPool);
+ assertEquals("Should have phase number 7", 7, pe.getPhaseNumber());
+ }
+
+ public void testWriteRead4() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/100000random.quads"));
+ FileQuadStream expected = new FileQuadStream(new File("testdata/100000sorted.quads"));
+ File dir = new File("test/petest1");
+
+ PhaseSet pe = new PhaseSet(dir, quads, 10*1024, 4, bufferPool);
+
+ assertEquals("PhaseSet not equal to source quads", expected, pe);
+ }
+
+ public void testRead4() throws Exception {
+ FileQuadStream expected = new FileQuadStream(new File("testdata/100000sorted.quads"));
+ PhaseSet pe = new PhaseSet(new File("test/petest1"), 7, bufferPool);
+
+ assertEquals("PhaseSet not equal to source quads", expected, pe);
+ }
+}
Added: projects/xa2/src/QuadQSort.java
===================================================================
--- projects/xa2/src/QuadQSort.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/QuadQSort.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,84 @@
+/**
+ * Derived from the HybridTuples implementation in mulgara - not particularly
+ * cache efficient, need to replace later. For now it sorts quads fast enough
+ * to allow testing to proceed.
+ */
+public class QuadQSort {
+ public static void sort(long[] buffer, int start, int end) {
+ if (end - start < 2) {
+ return;
+ }
+
+ int pivot = partition(buffer, start, end);
+
+ if (pivot > (start + 1)) {
+ sort(buffer, start, pivot);
+ }
+ if (pivot < (end - 2)) {
+ sort(buffer, pivot + 1, end);
+ }
+ }
+
+ private static int partition(long[] buffer, int start, int end) {
+ int pivot = ((end - start) / 2) + start;
+ int lhs = start;
+ int rhs = end - 1;
+
+ for (;;) {
+ while ((lhs < pivot) && (comparela(buffer, pivot*4, lhs*4) > 0)) {
+ lhs++;
+ }
+ while ((rhs > pivot) && (comparela(buffer, pivot*4, rhs*4) < 0)) {
+ rhs--;
+ }
+
+ if (lhs >= rhs) {
+ return pivot;
+ }
+
+ swap(buffer, 4, lhs*4, rhs*4);
+
+ if (lhs == pivot) {
+ lhs++;
+ pivot = rhs;
+ } else if (rhs == pivot) {
+ pivot = lhs;
+ rhs--;
+ } else {
+ lhs++;
+ rhs--;
+ }
+ }
+ }
+
+
+ private static int comparela(long[] buffer, int left, int right) {
+ if (buffer[left] < buffer[right]) {
+ return -1;
+ } else if (buffer[left] > buffer[right]) {
+ return +1;
+ } else if (buffer[left+1] > buffer[right+1]) {
+ return -1;
+ } else if (buffer[left+1] < buffer[right+1]) {
+ return +1;
+ } else if (buffer[left+2] > buffer[right+2]) {
+ return -1;
+ } else if (buffer[left+2] < buffer[right+2]) {
+ return +1;
+ } else if (buffer[left+3] > buffer[right+3]) {
+ return -1;
+ } else if (buffer[left+3] < buffer[right+3]) {
+ return +1;
+ } else {
+ return 0;
+ }
+ }
+
+ private static void swap(long[] buffer, int stride, int left, int right) {
+ for (int i = 0; i < stride; i++) {
+ long tmp = buffer[left + i];
+ buffer[left + i] = buffer[right + i];
+ buffer[right + i] = tmp;
+ }
+ }
+}
Added: projects/xa2/src/QuadStream.java
===================================================================
--- projects/xa2/src/QuadStream.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/QuadStream.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,42 @@
+/*
+ * QuadStream.java - A stream of quads.
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * A stream of quads.
+ *
+ * @created 2007-02-07
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+
+public interface QuadStream {
+ public void beforeFirst();
+ public boolean next();
+
+ /**
+ * Result may be aliased and reused; unless class specifies otherwise should
+ * be cloned/copied on return.
+ */
+ public long[] getQuad();
+}
Added: projects/xa2/src/ScaleTest1.java
===================================================================
--- projects/xa2/src/ScaleTest1.java 2007-08-28 07:05:45 UTC (rev 386)
+++ projects/xa2/src/ScaleTest1.java 2007-08-28 07:39:57 UTC (rev 387)
@@ -0,0 +1,94 @@
+/*
+ * ScaleTest1.java - Initial Simple Scale Test
+ *
+ * Copyright (C) 2006/7 Andrae Muys <andrae at netymon.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * First scalability test
+ *
+ * @created 2007-02-16
+ *
+ * @author <a href="mailto:andrae.muys at netymon.com">Andrae Muys</a>
+ *
+ * @copyright ©2006 <a href="http://www.netymon.com/">Netymon Pty Ltd</a>
+ *
+ * @licence GPLv2
+ */
+public class ScaleTest1 extends TestCase {
+ private ByteBufferPool bufferPool;
+
+ public ScaleTest1(String name) {
+ super(name);
+ }
+
+ public static Test suite() {
+ TestSuite suite = new TestSuite();
+ suite.addTest(new ScaleTest1("scaleTest1"));
+ return suite;
+ }
+
+ public void main(String args[]) {
+ junit.textui.TestRunner.run(suite());
+ }
+
+ public void setUp() throws Exception {
+ bufferPool = new ByteBufferPool(1*1024, 10);
+ }
+
+ public void tearDown() throws Exception {
+ }
+
+ private void assertEquals(String msg, QuadStream expected, QuadStream received)
+ throws Exception {
+ expected.beforeFirst();
+ received.beforeFirst();
+ int i = 0;
+ while (expected.next()) {
+ assertTrue("Expected longer quads", received.next());
+ long[] equad = expected.getQuad();
+ long[] rquad = received.getQuad();
+ for (int j = 0; j < 4; j++) {
+ if (equad[j] != rquad[j]) {
+ assertEquals(String.format(
+ "Quad[%d] mismatch - expected [ %d %d %d %d ] received [ %d %d %d %d ]\n", i,
+ equad[0], equad[1], equad[2], equad[3],
+ rquad[0], rquad[1], rquad[2], rquad[3]),
+ equad[j], rquad[j]);
+ }
+ }
+ i++;
+ }
+
+ assertFalse("Received more quads than expected", received.next());
+ }
+
+ public void scaleTest1() throws Exception {
+ FileQuadStream quads = new FileQuadStream(new File("testdata/40000000random.quads"));
+ File dir = new File("test/scaletest1");
+
+ PhaseSet pe = new PhaseSet(dir, quads, 1024*1024, 4, bufferPool);
+ }
+}
Added: projects/xa2/testdata/10000random.quads
===================================================================
(Binary files differ)
Property changes on: projects/xa2/testdata/10000random.quads
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: projects/xa2/testdata/10000sequential.quads
===================================================================
(Binary files differ)
Property changes on: projects/xa2/testdata/10000sequential.quads
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
Added: projects/xa2/testdata/10000sorted.quads
===================================================================
(Binary files differ)
Property changes on: projects/xa2/testdata/10000sorted.quads
___________________________________________________________________
Name: svn:mime-type
+ application/octet-stream
More information about the Mulgara-svn
mailing list