refactor map databases

src/org/oscim/database/MapDatabaseFactory.java

@@ -27,7 +27,7 @@ public final class MapDatabaseFactory {
 		String mapDatabaseName = attributeSet.getAttributeValue(null,
 				MAP_DATABASE_ATTRIBUTE_NAME);
 		if (mapDatabaseName == null) {
-			return MapDatabases.OSCIMAP_READER;
+			return MapDatabases.OPENSCIENCEMAP2;
 		}
 
 		return MapDatabases.valueOf(mapDatabaseName);
@@ -40,13 +40,13 @@ public final class MapDatabaseFactory {
 	 */
 	public static IMapDatabase createMapDatabase(MapDatabases mapDatabase) {
 		switch (mapDatabase) {
-			case MAPSFORGE_FILE:
+			case MAPSFORGE:
 				return new org.oscim.database.mapfile.MapDatabase();
-			case TEST_READER:
+			case TEST:
 				return new org.oscim.database.test.MapDatabase();
-			case PBMAP_READER:
+			case OPENSCIENCEMAP1:
 				return new org.oscim.database.oscimap.MapDatabase();
-			case OSCIMAP_READER:
+			case OPENSCIENCEMAP2:
 				return new org.oscim.database.oscimap2.MapDatabase();
 			case OPENSCIENCEMAP4:
 				return new org.oscim.database.oscimap4.MapDatabase();

src/org/oscim/database/MapDatabases.java

@@ -21,26 +21,26 @@ public enum MapDatabases {
 	/**
 	 * ...
 	 */
-	MAPSFORGE_FILE,
+	TEST,
 
 	/**
 	 * ...
 	 */
-	TEST_READER,
+	MAPSFORGE,
 
 	/**
 	 * ...
 	 */
-	POSTGIS_READER,
+	POSTGIS,
 
 	/**
 	 * ...
 	 */
-	PBMAP_READER,
+	OPENSCIENCEMAP1,
 	/**
 	 * ...
 	 */
-	OSCIMAP_READER,
+	OPENSCIENCEMAP2,
 
 	/**
 	 * ...

src/org/oscim/database/common/LwHttp.java

@@ -30,11 +30,13 @@ import org.oscim.core.Tile;
 import android.os.SystemClock;
 import android.util.Log;
 
-public class LwHttp extends InputStream{
+public class LwHttp {
 	private static final String TAG = LwHttp.class.getName();
 	//private static final boolean DEBUG = false;
 
-	private final static byte[] RESPONSE_HTTP_OK = "200 OK".getBytes();
+	private final static byte[] HEADER_HTTP_OK = "200 OK".getBytes();
+	private final static byte[] HEADER_CONTENT_TYPE = "Content-Type".getBytes();
+	private final static byte[] HEADER_CONTENT_LENGTH = "Content-Length".getBytes();
 	private final static int RESPONSE_EXPECTED_LIVES = 100;
 	private final static int RESPONSE_TIMEOUT = 10000;
 
@@ -55,17 +57,43 @@ public class LwHttp extends InputStream{
 	private byte[] REQUEST_GET_END;
 	private byte[] mRequestBuffer;
 
-	private boolean mInflateContent;
+	private final boolean mInflateContent;
+	private final byte[] mContentType;
+	private final String mExtension;
+
+	private int mContentLength = -1;
+
+	public LwHttp(String contentType, String extension, boolean deflate) {
+		mExtension = extension;
+		mContentType = contentType.getBytes();
+		mInflateContent = deflate;
+
+	}
+
+	static class Buffer extends BufferedInputStream {
+		public Buffer(InputStream is) {
+			super(is, 4096);
+		}
+
+		@Override
+		public synchronized int read() throws IOException {
+			return super.read();
+		}
+
+		@Override
+		public synchronized int read(byte[] buffer, int offset, int byteCount) throws IOException {
+			return super.read(buffer, offset, byteCount);
+		}
+	}
+
+	public boolean setServer(String urlString) {
 
-	public boolean setServer(String urlString, String extension, boolean zlibDeflate) {
 		URL url;
 		try {
 			url = new URL(urlString);
 		} catch (MalformedURLException e) {
-
 			e.printStackTrace();
 			return false;
-			//return new OpenResult("invalid url: " + options.get("url"));
 		}
 
 		int port = url.getPort();
@@ -74,13 +102,14 @@ public class LwHttp extends InputStream{
 
 		String host = url.getHost();
 		String path = url.getPath();
-		Log.d(TAG, "open oscim database: " + host + " " + port + " " + path);
+		Log.d(TAG, "open database: " + host + " " + port + " " + path);
 
 		REQUEST_GET_START = ("GET " + path).getBytes();
-		REQUEST_GET_END = (extension + " HTTP/1.1\n" +
+
-				"Host: " + host + "\n" +
+		REQUEST_GET_END = ("." + mExtension + " HTTP/1.1" +
-				"Connection: Keep-Alive\n\n").getBytes();
+				"\nHost: " + host +
-		mInflateContent = zlibDeflate;
+				"\nConnection: Keep-Alive" +
+				"\n\n").getBytes();
 
 		mHost = host;
 		mPort = port;
@@ -91,7 +120,6 @@ public class LwHttp extends InputStream{
 		return true;
 	}
 
-	@Override
 	public void close() {
 		if (mSocket != null) {
 			try {
@@ -117,6 +145,8 @@ public class LwHttp extends InputStream{
 		int end = 0;
 		int len = 0;
 
+		mContentLength = -1;
+
 		// header cannot be larger than BUFFER_SIZE for this to work
 		for (; pos < read || (len = is.read(buf, read, BUFFER_SIZE - read)) >= 0; len = 0) {
 			read += len;
@@ -129,7 +159,7 @@ public class LwHttp extends InputStream{
 				if (first) {
 					// check only for OK
 					first = false;
-					if (!compareBytes(buf, pos + 9, end, RESPONSE_HTTP_OK, 6)) {
+					if (!check(HEADER_HTTP_OK, 6, buf, pos + 9, end)) {
 						String line = new String(buf, pos, end - pos - 1);
 						Log.d(TAG, ">" + line + "< ");
 						return null;
@@ -138,10 +168,16 @@ public class LwHttp extends InputStream{
 					// check empty line (header end)
 					end += 1;
 					break;
+				} else if (check(HEADER_CONTENT_TYPE, 12, buf, pos, end)) {
+					if (!check(mContentType, mContentType.length, buf, pos + 14, end))
+						return null;
+				} else if (check(HEADER_CONTENT_LENGTH, 14, buf, pos, end)) {
+					mContentLength =  parseInt(pos + 16, end-1, buf);
+
 				}
 
 				//String line = new String(buf, pos, end - pos - 1);
-				//Log.d(TAG, ">" + line + "< ");
+				//Log.d(TAG, ">" + line + "<  " + mContentLength);
 
 				pos += (end - pos) + 1;
 				end = pos;
@@ -234,12 +270,12 @@ public class LwHttp extends InputStream{
 		mSocket.setTcpNoDelay(true);
 
 		mCommandStream = mSocket.getOutputStream();
-		mResponseStream = new BufferedInputStream(mSocket.getInputStream(), 4096);
+		mResponseStream = new BufferedInputStream(mSocket.getInputStream());
 
 		return true;
 	}
 
-	// write (positive) integer as char sequence to buffer
+	// write (positive) integer to byte array
 	protected static int writeInt(int val, int pos, byte[] buf) {
 		if (val == 0) {
 			buf[pos] = '0';
@@ -259,9 +295,17 @@ public class LwHttp extends InputStream{
 
 		return pos + i;
 	}
+	// parse (positive) integer from byte array
+	protected static int parseInt(int pos, int end, byte[] buf) {
+		int val = 0;
+		for (; pos < end; pos++)
+			val = val * 10 + (buf[pos]) - '0';
 
-	private static boolean compareBytes(byte[] buffer, int position, int available,
+		return val;
-			byte[] string, int length) {
+	}
+
+	private static boolean check(byte[] string, int length, byte[] buffer,
+			int position, int available) {
 
 		if (available - position < length)
 			return false;
@@ -273,16 +317,13 @@ public class LwHttp extends InputStream{
 		return true;
 	}
 
-	static int decodeInt(byte[] buffer, int offset) {
-		return buffer[offset] << 24 | (buffer[offset + 1] & 0xff) << 16
-				| (buffer[offset + 2] & 0xff) << 8
-				| (buffer[offset + 3] & 0xff);
-	}
-
 	public void requestCompleted() {
 		mLastRequest = SystemClock.elapsedRealtime();
 	}
 
+	public int getContentLength(){
+		return mContentLength;
+	}
 	/**
 	 * Write custom tile url
 	 *
@@ -295,10 +336,5 @@ public class LwHttp extends InputStream{
 		return 0;
 	}
 
-	@Override
-	public int read() throws IOException {
-		return mResponseStream.read();
-	}
-
 
 }

src/org/oscim/database/common/ProtobufDecoder.java

@@ -17,15 +17,33 @@ package org.oscim.database.common;
 import java.io.IOException;
 import java.io.InputStream;
 
+import org.oscim.core.Tile;
+import org.oscim.database.IMapDataSink;
 import org.oscim.utils.UTF8Decoder;
 
 import android.util.Log;
 
-public class ProtobufDecoder {
+public abstract class ProtobufDecoder {
 	private final static String TAG = ProtobufDecoder.class.getName();
 
-	private final static int VARINT_LIMIT = 6;
+	protected static final boolean debug = false;
-	private final static int VARINT_MAX = 10;
+
+	static class ProtobufException extends IOException {
+		private static final long serialVersionUID = 1L;
+
+		public ProtobufException(String detailMessage) {
+			super(detailMessage);
+		}
+	}
+
+	final static ProtobufException TRUNCATED_MSG = new ProtobufException("truncated msg");
+	protected final static ProtobufException INVALID_VARINT = new ProtobufException("invalid varint");
+	protected final static ProtobufException INVALID_PACKED_SIZE = new ProtobufException(
+			"invalid message size");
+
+	protected void error(String msg) throws IOException {
+		throw new ProtobufException(msg);
+	}
 
 	private final static int BUFFER_SIZE = 1 << 15; // 32kb
 	protected byte[] buffer = new byte[BUFFER_SIZE];
@@ -34,7 +52,7 @@ public class ProtobufDecoder {
 	protected int bufferPos;
 
 	// bytes available in buffer
-	int bufferFill;
+	protected int bufferFill;
 
 	// offset of buffer in message
 	private int mBufferOffset;
@@ -53,25 +71,8 @@ public class ProtobufDecoder {
 		mStringDecoder = new UTF8Decoder();
 	}
 
-	protected static int readUnsignedInt(InputStream is, byte[] buf) throws IOException {
+	public abstract boolean decode(Tile tile, IMapDataSink sink,
-		// check 4 bytes available..
+			InputStream is, int contentLength) throws IOException;
-		int read = 0;
-		int len = 0;
-
-		while (read < 4 && (len = is.read(buf, read, 4 - read)) >= 0)
-			read += len;
-
-		if (read < 4)
-			return read < 0 ? (read * 10) : read;
-
-		return decodeInt(buf, 0);
-	}
-
-	static int decodeInt(byte[] buffer, int offset) {
-		return buffer[offset] << 24 | (buffer[offset + 1] & 0xff) << 16
-				| (buffer[offset + 2] & 0xff) << 8
-				| (buffer[offset + 3] & 0xff);
-	}
 
 	public void setInputStream(InputStream is, int contentLength) {
 		mInputStream = is;
@@ -84,16 +85,105 @@ public class ProtobufDecoder {
 		mMsgEnd = contentLength;
 	}
 
-//	public void skipAvailable() throws IOException {
+	protected int decodeVarint32() throws IOException {
-//		int bytes = decodeVarint32();
-//		bufferPos += bytes;
-//	}
 
-	protected int decodeInterleavedPoints(float[] coords, int numPoints, float scale)
+		int bytesLeft = 0;
+		int val = 0;
+
+		for (int shift = 0; shift < 32; shift += 7) {
+			if (bytesLeft == 0)
+				bytesLeft = fillBuffer(1);
+
+			byte b = buffer[bufferPos++];
+			val |= (b & 0x7f) << shift;
+
+			if (b >= 0)
+				return val;
+
+			bytesLeft--;
+		}
+
+		throw INVALID_VARINT;
+	}
+
+	protected long decodeVarint64() throws IOException {
+
+		int bytesLeft = 0;
+		long val = 0;
+
+		for (int shift = 0; shift < 64; shift += 7) {
+			if (bytesLeft == 0)
+				bytesLeft = fillBuffer(1);
+
+			byte b = buffer[bufferPos++];
+			val |= (long) (b & 0x7f) << shift;
+
+			if (b >= 0)
+				return val;
+
+			bytesLeft--;
+		}
+
+		throw INVALID_VARINT;
+	}
+
+	protected String decodeString() throws IOException {
+		String result;
+
+		final int size = decodeVarint32();
+		fillBuffer(size);
+
+		if (mStringDecoder == null)
+			result = new String(buffer, bufferPos, size, "UTF-8");
+		else
+			result = mStringDecoder.decode(buffer, bufferPos, size);
+
+		bufferPos += size;
+
+		return result;
+
+	}
+
+	protected float decodeFloat() throws IOException {
+		if (bufferPos + 4 > bufferFill)
+			fillBuffer(4);
+
+		int val = (buffer[bufferPos++] & 0xFF
+				| (buffer[bufferPos++] & 0xFF) << 8
+				| (buffer[bufferPos++] & 0xFF) << 16
+				| (buffer[bufferPos++] & 0xFF) << 24);
+
+		return Float.intBitsToFloat(val);
+	}
+
+	protected double decodeDouble() throws IOException {
+		if (bufferPos + 8 > bufferFill)
+			fillBuffer(8);
+
+		long val = (buffer[bufferPos++] & 0xFF
+				| (buffer[bufferPos++] & 0xFF) << 8
+				| (buffer[bufferPos++] & 0xFF) << 16
+				| (buffer[bufferPos++] & 0xFF) << 24
+				| (buffer[bufferPos++] & 0xFF) << 32
+				| (buffer[bufferPos++] & 0xFF) << 40
+				| (buffer[bufferPos++] & 0xFF) << 48
+				| (buffer[bufferPos++] & 0xFF) << 56);
+
+		return Double.longBitsToDouble(val);
+	}
+
+	protected boolean decodeBool() throws IOException {
+		if (bufferPos + 1 > bufferFill)
+			fillBuffer(1);
+
+		return buffer[bufferPos++] != 0;
+	}
+
+	protected int decodeInterleavedPoints(float[] coords, float scale)
 			throws IOException {
-		int bytes = decodeVarint32();
 
-		readBuffer(bytes);
+		int bytes = decodeVarint32();
+		fillBuffer(bytes);
 
 		int cnt = 0;
 		int lastX = 0;
@@ -131,10 +221,8 @@ public class ProtobufDecoder {
 						| (buf[pos++] & 0x7f) << 21
 						| (buf[pos]) << 28;
 
-				int max = pos + VARINT_LIMIT;
+				if (buf[pos++] < 0)
-				while (buf[pos++] < 0)
+					throw INVALID_VARINT;
-					if (pos == max)
-						throw new IOException("malformed VarInt32");
 			}
 
 			// zigzag decoding
@@ -150,29 +238,34 @@ public class ProtobufDecoder {
 				even = true;
 			}
 		}
+
 		if (pos != bufferPos + bytes)
-			throw new IOException("invalid array " + numPoints);
+			throw INVALID_PACKED_SIZE;
 
 		bufferPos = pos;
 
-		return cnt;
+		// return number of points read
+		return (cnt >> 1);
+	}
+
+	protected static int deZigZag(int val) {
+		return ((val >>> 1) ^ -(val & 1));
 	}
 
 	public void decodeVarintArray(int num, short[] array) throws IOException {
 		int bytes = decodeVarint32();
-
+		fillBuffer(bytes);
-		readBuffer(bytes);
-
-		int cnt = 0;
 
 		byte[] buf = buffer;
 		int pos = bufferPos;
 		int end = pos + bytes;
 		int val;
 
+		int cnt = 0;
+
 		while (pos < end) {
 			if (cnt == num)
-				throw new IOException("invalid array size " + num);
+				throw new ProtobufException("invalid array size " + num);
 
 			if (buf[pos] >= 0) {
 				val = buf[pos++];
@@ -188,20 +281,20 @@ public class ProtobufDecoder {
 						| (buf[pos++] & 0x7f) << 7
 						| (buf[pos++] & 0x7f) << 14
 						| (buf[pos++]) << 21;
-			} else if (buf[pos + 4] >= 0) {
+			} else {
 				val = (buf[pos++] & 0x7f)
 						| (buf[pos++] & 0x7f) << 7
 						| (buf[pos++] & 0x7f) << 14
 						| (buf[pos++] & 0x7f) << 21
-						| (buf[pos++]) << 28;
+						| (buf[pos]) << 28;
-			} else
+				if (buf[pos++] < 0)
-				throw new IOException("malformed VarInt32");
+					throw INVALID_VARINT;
-
+			}
 			array[cnt++] = (short) val;
 		}
 
 		if (pos != bufferPos + bytes)
-			throw new IOException("invalid array " + num);
+			throw INVALID_PACKED_SIZE;
 
 		bufferPos = pos;
 	}
@@ -220,7 +313,7 @@ public class ProtobufDecoder {
 			array = new short[32];
 		}
 
-		readBuffer(bytes);
+		fillBuffer(bytes);
 		int cnt = 0;
 
 		byte[] buf = buffer;
@@ -250,13 +343,8 @@ public class ProtobufDecoder {
 						| (buf[pos++] & 0x7f) << 21
 						| (buf[pos]) << 28;
 
-				int max = pos + VARINT_LIMIT;
+				if (buf[pos++] < 0)
-				while (pos < max)
+					throw INVALID_VARINT;
-					if (buf[pos++] >= 0)
-						break;
-
-				if (pos > max)
-					throw new IOException("malformed VarInt32");
 			}
 
 			if (arrayLength <= cnt) {
@@ -269,6 +357,9 @@ public class ProtobufDecoder {
 			array[cnt++] = (short) val;
 		}
 
+		if (pos != bufferPos + bytes)
+			throw INVALID_PACKED_SIZE;
+
 		bufferPos = pos;
 
 		if (arrayLength > cnt)
@@ -277,16 +368,8 @@ public class ProtobufDecoder {
 		return array;
 	}
 
-	protected int decodeVarint32() throws IOException {
+	// for use int packed varint decoders
-		if (bufferPos + VARINT_MAX > bufferFill)
-			readBuffer(4096);
-
-		return decodeVarint32Filled();
-	}
-
 	protected int decodeVarint32Filled() throws IOException {
-		if (bufferPos + VARINT_MAX > bufferFill)
-			readBuffer(4096);
 
 		byte[] buf = buffer;
 		int pos = bufferPos;
@@ -295,7 +378,6 @@ public class ProtobufDecoder {
 		if (buf[pos] >= 0) {
 			val = buf[pos++];
 		} else {
-
 			if (buf[pos + 1] >= 0) {
 				val = (buf[pos++] & 0x7f)
 						| (buf[pos++]) << 7;
@@ -317,14 +399,8 @@ public class ProtobufDecoder {
 						| (buf[pos++] & 0x7f) << 21
 						| (buf[pos]) << 28;
 
-				// 'Discard upper 32 bits'
+				if (buf[pos++] < 0)
-				int max = pos + VARINT_LIMIT;
+					throw INVALID_VARINT;
-				while (pos < max)
-					if (buf[pos++] >= 0)
-						break;
-
-				if (pos == max)
-					throw new IOException("malformed VarInt32");
 			}
 		}
 
@@ -332,173 +408,86 @@ public class ProtobufDecoder {
 
 		return val;
 	}
-	// FIXME this also accept uin64 atm.
-//	protected int decodeVarint32Filled() throws IOException {
-//
-//		if (buffer[bufferPos] >= 0)
-//			return buffer[bufferPos++];
-//
-//		byte[] buf = buffer;
-//		int pos = bufferPos;
-//		int val = 0;
-//
-//		if (buf[pos + 1] >= 0) {
-//			val = (buf[pos++] & 0x7f)
-//					| (buf[pos++]) << 7;
-//
-//		} else if (buf[pos + 2] >= 0) {
-//			val = (buf[pos++] & 0x7f)
-//					| (buf[pos++] & 0x7f) << 7
-//					| (buf[pos++]) << 14;
-//
-//		} else if (buf[pos + 3] >= 0) {
-//			val = (buf[pos++] & 0x7f)
-//					| (buf[pos++] & 0x7f) << 7
-//					| (buf[pos++] & 0x7f) << 14
-//					| (buf[pos++]) << 21;
-//		} else {
-//			val = (buf[pos++] & 0x7f)
-//					| (buf[pos++] & 0x7f) << 7
-//					| (buf[pos++] & 0x7f) << 14
-//					| (buf[pos++] & 0x7f) << 21
-//					| (buf[pos]) << 28;
-//
-//
-//			// 'Discard upper 32 bits'
-//			int max = pos + VARINT_LIMIT;
-//			while (pos < max)
-//				if (buf[pos++] >= 0)
-//					break;
-//			if (pos == max)
-//				throw new IOException("malformed VarInt32");
-//		}
-//		bufferPos = pos;
-//
-//		return val;
-//	}
-
-	public String decodeString() throws IOException {
-		final int size = decodeVarint32();
-		readBuffer(size);
-
-		String result;
-
-		if (mStringDecoder == null)
-			result = new String(buffer, bufferPos, size, "UTF-8");
-		else
-			result = mStringDecoder.decode(buffer, bufferPos, size);
-
-		bufferPos += size;
-
-		return result;
-
-	}
-
-	public float decodeFloat() throws IOException {
-		if (bufferPos + 4 > bufferFill)
-			readBuffer(4096);
-
-		byte[] buf = buffer;
-		int pos = bufferPos;
-
-		int val = (buf[pos++] & 0xFF
-				| (buf[pos++] & 0xFF) << 8
-				| (buf[pos++] & 0xFF) << 16
-				| (buf[pos++] & 0xFF) << 24);
-
-		bufferPos += 4;
-		return Float.intBitsToFloat(val);
-	}
-
-	public double decodeDouble() throws IOException {
-		if (bufferPos + 8 > bufferFill)
-			readBuffer(4096);
-
-		byte[] buf = buffer;
-		int pos = bufferPos;
-
-		long val = (buf[pos++] & 0xFF
-				| (buf[pos++] & 0xFF) << 8
-				| (buf[pos++] & 0xFF) << 16
-				| (buf[pos++] & 0xFF) << 24
-				| (buf[pos++] & 0xFF) << 32
-				| (buf[pos++] & 0xFF) << 40
-				| (buf[pos++] & 0xFF) << 48
-				| (buf[pos++] & 0xFF) << 56);
-
-		bufferPos += 8;
-		return Double.longBitsToDouble(val);
-	}
-
-	public boolean decodeBool() throws IOException {
-		if (bufferPos + 1 > bufferFill)
-			readBuffer(4096);
-
-		return buffer[bufferPos++] != 0;
-	}
 
 	public boolean hasData() throws IOException {
 		if (mBufferOffset + bufferPos >= mMsgEnd)
 			return false;
 
-		return readBuffer(1);
+		return fillBuffer(1) > 0;
 	}
 
 	public int position() {
 		return mBufferOffset + bufferPos;
 	}
 
-	public boolean readBuffer(int size) throws IOException {
+	public int fillBuffer(int size) throws IOException {
+		int bytesLeft = bufferFill - bufferPos;
+
 		// check if buffer already contains the request bytes
-		if (bufferPos + size < bufferFill)
+		if (bytesLeft >= size)
-			return true;
+			return bytesLeft;
 
 		// check if inputstream is read to the end
 		if (mMsgPos >= mMsgEnd)
-			return false;
+			return bytesLeft;
 
 		int maxSize = buffer.length;
 
 		if (size > maxSize) {
+
+			if (debug)
 				Log.d(TAG, "increase read buffer to " + size + " bytes");
+
 			maxSize = size;
-			bufferFill -= bufferPos;
 
 			byte[] tmp = buffer;
 			buffer = new byte[maxSize];
-			System.arraycopy(tmp, bufferPos, buffer, 0, bufferFill);
+			System.arraycopy(tmp, bufferPos, buffer, 0, bytesLeft);
 
 			mBufferOffset += bufferPos;
 			bufferPos = 0;
-		}
+			bufferFill = bytesLeft;
 
-		if (bufferFill == bufferPos) {
+		} else if (bytesLeft == 0) {
+			// just advance buffer offset and reset buffer
 			mBufferOffset += bufferPos;
 			bufferPos = 0;
 			bufferFill = 0;
+
 		} else if (bufferPos + size > maxSize) {
 			// copy bytes left to the beginning of buffer
-			bufferFill -= bufferPos;
+			if (debug)
-			System.arraycopy(buffer, bufferPos, buffer, 0, bufferFill);
+				Log.d(TAG, "shift " + bufferFill + " " + bufferPos + " " + size);
+
+			System.arraycopy(buffer, bufferPos, buffer, 0, bytesLeft);
+
 			mBufferOffset += bufferPos;
 			bufferPos = 0;
+			bufferFill = bytesLeft;
 		}
 
+		while ((bufferFill - bufferPos) < size) {
+
 			int max = maxSize - bufferFill;
-
-		while ((bufferFill - bufferPos) < size && max > 0) {
-
-			max = maxSize - bufferFill;
 			if (max > mMsgEnd - mMsgPos)
 				max = mMsgEnd - mMsgPos;
 
+			if (max <= 0) {
+				// should not be possible
+				throw new IOException("burp");
+			}
+
 			// read until requested size is available in buffer
 			int len = mInputStream.read(buffer, bufferFill, max);
 
 			if (len < 0) {
+				mMsgEnd = mMsgPos;
+				if (debug)
+					Log.d(TAG, " finished reading " + mMsgPos);
+
 				// finished reading, mark end
 				buffer[bufferFill] = 0;
-				return false;
+				return bufferFill - bufferPos;
 			}
 
 			mMsgPos += len;
@@ -508,6 +497,26 @@ public class ProtobufDecoder {
 				break;
 
 		}
-		return true;
+		return bufferFill - bufferPos;
+	}
+
+	protected static int readUnsignedInt(InputStream is, byte[] buf) throws IOException {
+		// check 4 bytes available..
+		int read = 0;
+		int len = 0;
+
+		while (read < 4 && (len = is.read(buf, read, 4 - read)) >= 0)
+			read += len;
+
+		if (read < 4)
+			return read < 0 ? (read * 10) : read;
+
+		return decodeInt(buf, 0);
+	}
+
+	static int decodeInt(byte[] buffer, int offset) {
+		return buffer[offset] << 24 | (buffer[offset + 1] & 0xff) << 16
+				| (buffer[offset + 2] & 0xff) << 8
+				| (buffer[offset + 3] & 0xff);
 	}
 }

src/org/oscim/database/common/ProtobufMapDatabase.java

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2012 Hannes Janetzek
+ *
+ * This program is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Lesser General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.oscim.database.common;
+
+import java.io.InputStream;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+
+import org.oscim.core.BoundingBox;
+import org.oscim.core.GeoPoint;
+import org.oscim.database.IMapDataSink;
+import org.oscim.database.IMapDatabase;
+import org.oscim.database.MapInfo;
+import org.oscim.database.MapOptions;
+import org.oscim.layers.tile.MapTile;
+
+import android.util.Log;
+
+/**
+ *
+ *
+ */
+public abstract class ProtobufMapDatabase implements IMapDatabase {
+	private static final String TAG = ProtobufMapDatabase.class.getName();
+
+	protected LwHttp mConn;
+	protected final ProtobufDecoder mTileDecoder;
+
+	// 'open' state
+	private boolean mOpen = false;
+
+	public ProtobufMapDatabase(ProtobufDecoder tileDecoder) {
+		mTileDecoder = tileDecoder;
+	}
+
+	private static final MapInfo mMapInfo =
+			new MapInfo(new BoundingBox(-180, -90, 180, 90),
+					new Byte((byte) 4), new GeoPoint(53.11, 8.85),
+					null, 0, 0, 0, "de", "comment", "author", null);
+
+	@Override
+	public QueryResult executeQuery(MapTile tile, IMapDataSink sink) {
+		QueryResult result = QueryResult.SUCCESS;
+
+		try {
+			InputStream is;
+			if (!mConn.sendRequest(tile)) {
+				Log.d(TAG, tile + " Request Failed");
+				result = QueryResult.FAILED;
+			} else if ((is = mConn.readHeader()) != null) {
+				boolean win = mTileDecoder.decode(tile, sink, is, mConn.getContentLength());
+				if (!win)
+					Log.d(TAG, tile + " failed");
+			} else {
+				Log.d(TAG, tile + " Network Error");
+				result = QueryResult.FAILED;
+			}
+		} catch (SocketException e) {
+			Log.d(TAG, tile + " Socket exception: " + e.getMessage());
+			result = QueryResult.FAILED;
+		} catch (SocketTimeoutException e) {
+			Log.d(TAG, tile + " Socket Timeout");
+			result = QueryResult.FAILED;
+		} catch (UnknownHostException e) {
+			Log.d(TAG, tile + " No Network");
+			result = QueryResult.FAILED;
+		} catch (Exception e) {
+			e.printStackTrace();
+			result = QueryResult.FAILED;
+		}
+
+		mConn.requestCompleted();
+
+		if (result != QueryResult.SUCCESS)
+			mConn.close();
+
+		return result;
+	}
+
+	@Override
+	public String getMapProjection() {
+		return null;
+	}
+
+	@Override
+	public MapInfo getMapInfo() {
+		return mMapInfo;
+	}
+
+	@Override
+	public boolean isOpen() {
+		return mOpen;
+	}
+
+	@Override
+	public OpenResult open(MapOptions options) {
+
+		if (mOpen)
+			return OpenResult.SUCCESS;
+
+		if (options == null || !options.containsKey("url"))
+			return new OpenResult("No URL in MapOptions");
+
+
+		if (!mConn.setServer(options.get("url"))) {
+			return new OpenResult("invalid url: " + options.get("url"));
+		}
+
+		mOpen = true;
+
+		return OpenResult.SUCCESS;
+	}
+
+	@Override
+	public void close() {
+		mOpen = false;
+		mConn.close();
+	}
+
+	@Override
+	public void cancel() {
+	}
+}

src/org/oscim/database/mapnik/MapDatabase.java

@@ -14,95 +14,17 @@
  */
 package org.oscim.database.mapnik;
 
-import java.io.InputStream;
-import java.net.SocketException;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
-
-import org.oscim.core.BoundingBox;
-import org.oscim.core.GeoPoint;
 import org.oscim.core.Tile;
-import org.oscim.database.IMapDataSink;
-import org.oscim.database.IMapDatabase;
-import org.oscim.database.MapInfo;
-import org.oscim.database.MapOptions;
 import org.oscim.database.common.LwHttp;
-import org.oscim.layers.tile.MapTile;
+import org.oscim.database.common.ProtobufMapDatabase;
 
-import android.util.Log;
+public class MapDatabase extends ProtobufMapDatabase {
+	//private static final String TAG = MapDatabase.class.getName();
 
-public class MapDatabase implements IMapDatabase {
+	public MapDatabase() {
-	private static final String TAG = MapDatabase.class.getName();
+		super(new TileDecoder());
-
-	private static final MapInfo mMapInfo =
-			new MapInfo(new BoundingBox(-180, -90, 180, 90),
-					new Byte((byte) 4), new GeoPoint(53.11, 8.85),
-					null, 0, 0, 0, "de", "comment", "author",
-					new int[] { 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }
-			);
-
-	// 'open' state
-	private boolean mOpen = false;
-
-	private LwHttp conn;
-	private TileDecoder mTileDecoder;
-
-	@Override
-	public QueryResult executeQuery(MapTile tile, IMapDataSink mapDataSink) {
-		QueryResult result = QueryResult.SUCCESS;
-
-		try {
-			InputStream is;
-			if (conn.sendRequest(tile) && (is = conn.readHeader()) != null) {
-				mTileDecoder.decode(is, tile, mapDataSink);
-			} else {
-				Log.d(TAG, tile + " Network Error");
-				result = QueryResult.FAILED;
-			}
-		} catch (SocketException ex) {
-			Log.d(TAG, tile + " Socket exception: " + ex.getMessage());
-			result = QueryResult.FAILED;
-		} catch (SocketTimeoutException ex) {
-			Log.d(TAG, tile + " Socket Timeout exception: " + ex.getMessage());
-			result = QueryResult.FAILED;
-		} catch (UnknownHostException ex) {
-			Log.d(TAG, tile + " no network");
-			result = QueryResult.FAILED;
-		} catch (Exception ex) {
-			ex.printStackTrace();
-			result = QueryResult.FAILED;
-		}
-
-		conn.requestCompleted();
-
-		if (result != QueryResult.SUCCESS) {
-			conn.close();
-		}
-
-		return result;
-	}
-
-	@Override
-	public MapInfo getMapInfo() {
-		return mMapInfo;
-	}
-
-	@Override
-	public boolean isOpen() {
-		return mOpen;
-	}
-
-	@Override
-	public OpenResult open(MapOptions options) {
-		String extension = ".vector.pbf";
-		if (mOpen)
-			return OpenResult.SUCCESS;
-
-		if (options == null || !options.containsKey("url"))
-			return new OpenResult("options missing");
-
-		conn = new LwHttp() {
 
+		mConn = new LwHttp("image/png","vector.pbf", true) {
 			@Override
 			protected int formatTilePath(Tile tile, byte[] path, int pos) {
 				// url formatter for mapbox streets
@@ -122,32 +44,5 @@ public class MapDatabase implements IMapDatabase {
 				return pos;
 			}
 		};
-
-		if (!conn.setServer(options.get("url"), extension, true)) {
-			return new OpenResult("invalid url: " + options.get("url"));
-		}
-
-		mTileDecoder = new TileDecoder();
-
-		mOpen = true;
-		return OpenResult.SUCCESS;
-	}
-
-	@Override
-	public void close() {
-		mOpen = false;
-
-		mTileDecoder = null;
-		conn.close();
-		conn = null;
-	}
-
-	@Override
-	public String getMapProjection() {
-		return null;
-	}
-
-	@Override
-	public void cancel() {
 	}
 }

src/org/oscim/database/mapnik/TileDecoder.java

@@ -70,7 +70,12 @@ public class TileDecoder extends ProtobufDecoder {
 	private final static float REF_TILE_SIZE = 4096.0f;
 	private float mScale;
 
-	boolean decode(InputStream is, Tile tile, IMapDataSink mapDataCallback) throws IOException {
+	@Override
+	public boolean decode(Tile tile, IMapDataSink mapDataCallback, InputStream is, int contentLength)
+			throws IOException {
+		if (debug)
+			Log.d(TAG, tile + " decode");
+
 		setInputStream(is, Integer.MAX_VALUE);
 		mTile = tile;
 		mMapDataCallback = mapDataCallback;
@@ -88,10 +93,15 @@ public class TileDecoder extends ProtobufDecoder {
 					break;
 
 				default:
-					Log.d(TAG, mTile + " invalid type for tile: " + tag);
+					error(mTile + " invalid type for tile: " + tag);
 					return false;
 			}
 		}
+
+		if (hasData()){
+			error(tile + " invalid tile");
+			return false;
+		}
 		return true;
 	}
 
@@ -147,7 +157,7 @@ public class TileDecoder extends ProtobufDecoder {
 					break;
 
 				default:
-					Log.d(TAG, mTile + " invalid type for layer: " + tag);
+					error(mTile + " invalid type for layer: " + tag);
 					break;
 			}
 
@@ -357,7 +367,7 @@ public class TileDecoder extends ProtobufDecoder {
 					break;
 
 				default:
-					Log.d(TAG, mTile + " invalid type for feature: " + tag);
+					error(mTile + " invalid type for feature: " + tag);
 					break;
 			}
 		}
@@ -371,7 +381,7 @@ public class TileDecoder extends ProtobufDecoder {
 
 	private int decodeCoordinates(int type, Feature feature) throws IOException {
 		int bytes = decodeVarint32();
-		readBuffer(bytes);
+		fillBuffer(bytes);
 
 		if (feature == null) {
 			bufferPos += bytes;
@@ -405,7 +415,7 @@ public class TileDecoder extends ProtobufDecoder {
 		int prevY = 0;
 
 		int cmd = 0;
-		int num = 0;
+		int num = 0, cnt = 0;
 
 		boolean first = true;
 		boolean lastClip = false;
@@ -413,7 +423,7 @@ public class TileDecoder extends ProtobufDecoder {
 		// test bbox for outer..
 		boolean isOuter = true;
 		boolean simplify = mTile.zoomLevel < 14;
-		int pixel = simplify ? 7 : 1;
+		int pixel = simplify ? 7 : 3;
 
 		int xmin = Integer.MAX_VALUE, xmax = Integer.MIN_VALUE;
 		int ymin = Integer.MAX_VALUE, ymax = Integer.MIN_VALUE;
@@ -422,7 +432,10 @@ public class TileDecoder extends ProtobufDecoder {
 			val = decodeVarint32Filled();
 
 			if (num == 0) {
+				// number of points
 				num = val >>> 3;
+				cnt = 0;
+				// path command
 				cmd = val & 0x07;
 
 				if (isLine && lastClip) {
@@ -466,17 +479,31 @@ public class TileDecoder extends ProtobufDecoder {
 			int dx = (curX - prevX);
 			int dy = (curY - prevY);
 
+			if (isPoly && num == 0 && cnt > 0){
+				prevX = curX;
+				prevY = curY;
+
+				// only add last point if it is di
+				int ppos = cnt * 2;
+				if (elem.points[elem.pointPos - ppos] != curX
+						|| elem.points[elem.pointPos - ppos + 1] != curY)
+					elem.addPoint(curX / mScale, curY / mScale);
+
+				lastClip = false;
+				continue;
+			}
+
 			if ((isPoint || cmd == MOVE_TO)
 					|| (dx > pixel || dx < -pixel)
 					|| (dy > pixel || dy < -pixel)
-					// dont clip at tile boundaries
+					// hack to not clip at tile boundaries
 					|| (curX <= 0 || curX >= 4095)
 					|| (curY <= 0 || curY >= 4095)) {
 
 				prevX = curX;
 				prevY = curY;
 				elem.addPoint(curX / mScale, curY / mScale);
-				lastClip = false;
+				cnt++;
 
 				if (simplify && isOuter) {
 					if (curX < xmin)
@@ -490,6 +517,7 @@ public class TileDecoder extends ProtobufDecoder {
 						ymax = curY;
 				}
 
+				lastClip = false;
 				continue;
 			}
 			lastClip = true;
@@ -543,11 +571,11 @@ public class TileDecoder extends ProtobufDecoder {
 					break;
 
 				case TAG_VALUE_SINT:
-					value = String.valueOf(decodeVarint32());
+					value = String.valueOf(deZigZag(decodeVarint32()));
 					break;
 
 				case TAG_VALUE_LONG:
-					value = String.valueOf(decodeVarint32());
+					value = String.valueOf(decodeVarint64());
 					break;
 
 				case TAG_VALUE_FLOAT:

src/org/oscim/database/oscimap/TileDecoder.java

@@ -0,0 +1,449 @@
+/*
+ * Copyright 2013
+ *
+ * This program is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Lesser General Public License as published by the Free Software
+ * Foundation, either version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License along with
+ * this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+package org.oscim.database.oscimap;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.oscim.core.GeometryBuffer.GeometryType;
+import org.oscim.core.MapElement;
+import org.oscim.core.Tag;
+import org.oscim.core.Tile;
+import org.oscim.database.IMapDataSink;
+import org.oscim.database.common.ProtobufDecoder;
+
+import android.util.Log;
+
+public class TileDecoder extends ProtobufDecoder {
+	private final static String TAG = TileDecoder.class.getName();
+
+	private final static float REF_TILE_SIZE = 4096.0f;
+
+	private static final int TAG_TILE_TAGS = 1;
+	private static final int TAG_TILE_WAYS = 2;
+	private static final int TAG_TILE_POLY = 3;
+	private static final int TAG_TILE_NODES = 4;
+	private static final int TAG_WAY_TAGS = 11;
+	private static final int TAG_WAY_INDEX = 12;
+	private static final int TAG_WAY_COORDS = 13;
+	private static final int TAG_WAY_LAYER = 21;
+	private static final int TAG_WAY_NUM_TAGS = 1;
+	private static final int TAG_WAY_NUM_INDICES = 2;
+	private static final int TAG_WAY_NUM_COORDS = 3;
+
+	private static final int TAG_NODE_TAGS = 11;
+	private static final int TAG_NODE_COORDS = 12;
+	private static final int TAG_NODE_LAYER = 21;
+	private static final int TAG_NODE_NUM_TAGS = 1;
+	private static final int TAG_NODE_NUM_COORDS = 2;
+
+	private int MAX_TILE_TAGS = 100;
+	private Tag[] curTags = new Tag[MAX_TILE_TAGS];
+	private int mCurTagCnt;
+
+	private IMapDataSink mSink;
+	private float mScale;
+	private Tile mTile;
+	private final MapElement mElem;
+
+	TileDecoder() {
+		mElem = new MapElement();
+	}
+
+	@Override
+	public boolean decode(Tile tile, IMapDataSink sink, InputStream is, int contentLength)
+			throws IOException {
+
+
+		setInputStream(is, contentLength);
+
+		mTile = tile;
+		mSink = sink;
+		mScale = REF_TILE_SIZE / Tile.SIZE;
+		return decode();
+	}
+
+	private static final int MAX_TAGS_CACHE = 100;
+	private static Map<String, Tag> tagHash = Collections
+			.synchronizedMap(new LinkedHashMap<String, Tag>(
+					MAX_TAGS_CACHE, 0.75f, true) {
+
+				private static final long serialVersionUID = 1L;
+
+				@Override
+				protected boolean removeEldestEntry(Entry<String, Tag> e) {
+					if (size() < MAX_TAGS_CACHE)
+						return false;
+					return true;
+				}
+			});
+
+	private boolean decode() throws IOException {
+		int val;
+		mCurTagCnt = 0;
+
+		while (hasData() && (val = decodeVarint32()) > 0) {
+			// read tag and wire type
+			int tag = (val >> 3);
+
+			switch (tag) {
+				case TAG_TILE_TAGS:
+					decodeTileTags();
+					break;
+
+				case TAG_TILE_WAYS:
+					decodeTileWays(false);
+					break;
+
+				case TAG_TILE_POLY:
+					decodeTileWays(true);
+					break;
+
+				case TAG_TILE_NODES:
+					decodeTileNodes();
+					break;
+
+				default:
+					Log.d(TAG, "invalid type for tile: " + tag);
+					return false;
+			}
+		}
+		return true;
+	}
+
+	private boolean decodeTileTags() throws IOException {
+		String tagString = decodeString();
+
+		if (tagString == null || tagString.length() == 0) {
+			curTags[mCurTagCnt++] = new Tag(Tag.TAG_KEY_NAME, "...");
+			return false;
+		}
+
+		Tag tag = tagHash.get(tagString);
+
+		if (tag == null) {
+			if (tagString.startsWith(Tag.TAG_KEY_NAME))
+				tag = new Tag(Tag.TAG_KEY_NAME, tagString.substring(5), false);
+			else
+				tag = new Tag(tagString);
+
+			tagHash.put(tagString, tag);
+		}
+
+		if (mCurTagCnt >= MAX_TILE_TAGS) {
+			MAX_TILE_TAGS = mCurTagCnt + 10;
+			Tag[] tmp = new Tag[MAX_TILE_TAGS];
+			System.arraycopy(curTags, 0, tmp, 0, mCurTagCnt);
+			curTags = tmp;
+		}
+		curTags[mCurTagCnt++] = tag;
+
+		return true;
+	}
+
+	private boolean decodeTileWays(boolean polygon) throws IOException {
+		int bytes = decodeVarint32();
+
+		int end = position() + bytes;
+		int indexCnt = 0;
+		int tagCnt = 0;
+		int coordCnt = 0;
+		int layer = 5;
+		Tag[] tags = null;
+
+		boolean fail = false;
+
+		while (position() < end) {
+			// read tag and wire type
+			int val = decodeVarint32();
+			if (val == 0)
+				break;
+
+			int tag = (val >> 3);
+
+			switch (tag) {
+				case TAG_WAY_TAGS:
+					tags = decodeWayTags(tagCnt);
+					break;
+
+				case TAG_WAY_INDEX:
+					//index =
+					decodeWayIndices(indexCnt);
+					break;
+
+				case TAG_WAY_COORDS:
+					if (coordCnt == 0) {
+						Log.d(TAG, mTile + " no coordinates");
+					}
+
+					mElem.ensurePointSize(coordCnt, false);
+					int cnt = decodeInterleavedPoints(mElem.points, mScale);
+
+					if (cnt != coordCnt) {
+						Log.d(TAG, mTile + " wrong number of coordintes "
+								+ coordCnt + "/" + cnt);
+						fail = true;
+					}
+
+					break;
+
+				case TAG_WAY_LAYER:
+					layer = decodeVarint32();
+					break;
+
+				case TAG_WAY_NUM_TAGS:
+					tagCnt = decodeVarint32();
+					break;
+
+				case TAG_WAY_NUM_INDICES:
+					indexCnt = decodeVarint32();
+					break;
+
+				case TAG_WAY_NUM_COORDS:
+					coordCnt = decodeVarint32();
+					break;
+
+				default:
+					Log.d(TAG, "X invalid type for way: " + tag);
+			}
+		}
+
+		if (fail || tags == null || indexCnt == 0 || tagCnt == 0) {
+			Log.d(TAG, "failed reading way: bytes:" + bytes + " index:"
+					+ (tags != null ? tags.toString() : "...") + " "
+					+ indexCnt + " " + coordCnt + " " + tagCnt);
+			return false;
+		}
+
+		// FIXME, remove all tiles from cache then remove this below
+		//if (layer == 0)
+		//	layer = 5;
+		mElem.type = polygon ? GeometryType.POLY : GeometryType.LINE;
+		mElem.set(tags, layer);
+		mSink.process(mElem);
+		return true;
+	}
+
+	private boolean decodeTileNodes() throws IOException {
+		int bytes = decodeVarint32();
+
+		int end = position() + bytes;
+		int tagCnt = 0;
+		int coordCnt = 0;
+		byte layer = 0;
+		Tag[] tags = null;
+
+		while (position() < end) {
+			// read tag and wire type
+			int val = decodeVarint32();
+			if (val == 0)
+				break;
+
+			int tag = (val >> 3);
+
+			switch (tag) {
+				case TAG_NODE_TAGS:
+					tags = decodeWayTags(tagCnt);
+					break;
+
+				case TAG_NODE_COORDS:
+					int cnt = decodeNodeCoordinates(coordCnt, layer, tags);
+					if (cnt != coordCnt) {
+						Log.d(TAG, "X wrong number of coordintes");
+						return false;
+					}
+					break;
+
+				case TAG_NODE_LAYER:
+					layer = (byte) decodeVarint32();
+					break;
+
+				case TAG_NODE_NUM_TAGS:
+					tagCnt = decodeVarint32();
+					break;
+
+				case TAG_NODE_NUM_COORDS:
+					coordCnt = decodeVarint32();
+					break;
+
+				default:
+					Log.d(TAG, "X invalid type for node: " + tag);
+			}
+		}
+
+		return true;
+	}
+
+	private int decodeNodeCoordinates(int numNodes, byte layer, Tag[] tags)
+			throws IOException {
+		int bytes = decodeVarint32();
+
+		fillBuffer(bytes);
+		int cnt = 0;
+		int end = position() + bytes;
+		// read repeated sint32
+		int lastX = 0;
+		int lastY = 0;
+		float[] coords = mElem.ensurePointSize(numNodes, false);
+
+		while (position() < end && cnt < numNodes) {
+			int lon = deZigZag(decodeVarint32());
+			int lat = deZigZag(decodeVarint32());
+			lastX = lon + lastX;
+			lastY = lat + lastY;
+			coords[cnt++] = lastX / mScale;
+			coords[cnt++] = Tile.SIZE - lastY / mScale;
+		}
+
+		mElem.index[0] = (short) numNodes;
+		mElem.type = GeometryType.POINT;
+		mElem.set(tags, layer);
+		mSink.process(mElem);
+
+		return cnt;
+	}
+
+	private Tag[] decodeWayTags(int tagCnt) throws IOException {
+		int bytes = decodeVarint32();
+
+		Tag[] tags = new Tag[tagCnt];
+
+		int cnt = 0;
+		int end = position() + bytes;
+		int max = mCurTagCnt;
+
+		while (position() < end) {
+			int tagNum = decodeVarint32();
+
+			if (tagNum < 0 || cnt == tagCnt) {
+				Log.d(TAG, "NULL TAG: " + mTile + " invalid tag:" + tagNum
+						+ " " + tagCnt + "/" + cnt);
+			} else {
+				if (tagNum < Tags.MAX)
+					tags[cnt++] = Tags.tags[tagNum];
+				else {
+					tagNum -= Tags.LIMIT;
+
+					if (tagNum >= 0 && tagNum < max) {
+						// Log.d(TAG, "variable tag: " + curTags[tagNum]);
+						tags[cnt++] = curTags[tagNum];
+					} else {
+						Log.d(TAG, "NULL TAG: " + mTile + " could find tag:"
+								+ tagNum + " " + tagCnt + "/" + cnt);
+					}
+				}
+			}
+		}
+
+		if (tagCnt != cnt)
+			Log.d(TAG, "NULL TAG: " + mTile);
+
+		return tags;
+	}
+
+	private int decodeWayIndices(int indexCnt) throws IOException {
+		mElem.ensureIndexSize(indexCnt, false);
+
+		decodeVarintArray(indexCnt, mElem.index);
+
+		short[] index = mElem.index;
+		int coordCnt = 0;
+
+		for (int i = 0; i < indexCnt; i++) {
+			coordCnt += index[i];
+			index[i] *= 2;
+		}
+
+		// set end marker
+		if (indexCnt < index.length)
+			index[indexCnt] = -1;
+
+		return coordCnt;
+	}
+
+	@Override
+	protected int decodeInterleavedPoints(float[] coords, float scale)
+			throws IOException {
+
+		int bytes = decodeVarint32();
+		fillBuffer(bytes);
+
+		int cnt = 0;
+		int lastX = 0;
+		int lastY = 0;
+		boolean even = true;
+
+		byte[] buf = buffer;
+		int pos = bufferPos;
+		int end = pos + bytes;
+		int val;
+
+		while (pos < end) {
+			if (buf[pos] >= 0) {
+				val = buf[pos++];
+
+			} else if (buf[pos + 1] >= 0) {
+				val = (buf[pos++] & 0x7f)
+						| buf[pos++] << 7;
+
+			} else if (buf[pos + 2] >= 0) {
+				val = (buf[pos++] & 0x7f)
+						| (buf[pos++] & 0x7f) << 7
+						| (buf[pos++]) << 14;
+
+			} else if (buf[pos + 3] >= 0) {
+				val = (buf[pos++] & 0x7f)
+						| (buf[pos++] & 0x7f) << 7
+						| (buf[pos++] & 0x7f) << 14
+						| (buf[pos++]) << 21;
+
+			} else {
+				val = (buf[pos++] & 0x7f)
+						| (buf[pos++] & 0x7f) << 7
+						| (buf[pos++] & 0x7f) << 14
+						| (buf[pos++] & 0x7f) << 21
+						| (buf[pos]) << 28;
+
+				if (buf[pos++] < 0)
+					throw INVALID_VARINT;
+			}
+
+			// zigzag decoding
+			int s = ((val >>> 1) ^ -(val & 1));
+
+			if (even) {
+				lastX = lastX + s;
+				coords[cnt++] = lastX / scale;
+				even = false;
+			} else {
+				lastY = lastY + s;
+				coords[cnt++] = Tile.SIZE - lastY /  scale;
+				even = true;
+			}
+		}
+
+		if (pos != bufferPos + bytes)
+			throw INVALID_PACKED_SIZE;
+
+		bufferPos = pos;
+
+		// return number of points read
+		return cnt;
+	}
+
+}

src/org/oscim/database/oscimap2/LwHttp.java

@@ -1,450 +0,0 @@
-/*
- * Copyright 2013 Hannes Janetzek
- *
- * This program is free software: you can redistribute it and/or modify it under the
- * terms of the GNU Lesser General Public License as published by the Free Software
- * Foundation, either version 3 of the License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
- * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License along with
- * this program. If not, see <http://www.gnu.org/licenses/>.
- */
-package org.oscim.database.oscimap2;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
-import java.net.Socket;
-import java.net.SocketAddress;
-import java.net.URL;
-
-import org.oscim.core.Tile;
-
-import android.os.SystemClock;
-import android.util.Log;
-
-public class LwHttp {
-	private static final String TAG = LwHttp.class.getName();
-	private final static int BUFFER_SIZE = 65536;
-
-	//
-	byte[] buffer = new byte[BUFFER_SIZE];
-
-	// position in buffer
-	int bufferPos;
-
-	// bytes available in buffer
-	int bufferFill;
-
-
-	// offset of buffer in message
-	private int mBufferOffset;
-
-	// max bytes to read: message = header + content
-	private long mReadEnd;
-
-	// overall bytes of message read
-	private int mReadPos;
-
-	private String mHost;
-	private int mPort;
-	private InputStream mInputStream;
-
-	private int mMaxReq = 0;
-	private Socket mSocket;
-	private OutputStream mCommandStream;
-	private InputStream mResponseStream;
-	long mLastRequest = 0;
-	private SocketAddress mSockAddr;
-
-	private final static byte[] RESPONSE_HTTP_OK = "HTTP/1.1 200 OK".getBytes();
-	private final static int RESPONSE_EXPECTED_LIVES = 100;
-	private final static int RESPONSE_EXPECTED_TIMEOUT = 10000;
-
-	private byte[] REQUEST_GET_START;
-	private byte[] REQUEST_GET_END;
-
-	private byte[] mRequestBuffer;
-
-	boolean setServer(String urlString) {
-		URL url;
-		try {
-			url = new URL(urlString);
-		} catch (MalformedURLException e) {
-
-			e.printStackTrace();
-			return false;
-			//return new OpenResult("invalid url: " + options.get("url"));
-		}
-
-		int port = url.getPort();
-		if (port < 0)
-			port = 80;
-
-		String host = url.getHost();
-		String path = url.getPath();
-		Log.d(TAG, "open oscim database: " + host + " " + port + " " + path);
-
-		REQUEST_GET_START = ("GET " + path).getBytes();
-		REQUEST_GET_END = (".osmtile HTTP/1.1\n" +
-				"Host: " + host + "\n" +
-				"Connection: Keep-Alive\n\n").getBytes();
-
-		mHost = host;
-		mPort = port;
-
-		mRequestBuffer = new byte[1024];
-		System.arraycopy(REQUEST_GET_START, 0,
-				mRequestBuffer, 0, REQUEST_GET_START.length);
-		return true;
-	}
-
-	void close() {
-		if (mSocket != null) {
-			try {
-				mSocket.close();
-			} catch (IOException e) {
-				e.printStackTrace();
-			} finally {
-				mSocket = null;
-			}
-		}
-	}
-
-	int readHeader() throws IOException {
-		InputStream is = mResponseStream;
-
-		byte[] buf = buffer;
-		boolean first = true;
-		int read = 0;
-		int pos = 0;
-		int end = 0;
-		int len = 0;
-
-		// header cannot be larger than BUFFER_SIZE for this to work
-		for (; pos < read || (len = is.read(buf, read, BUFFER_SIZE - read)) >= 0; len = 0) {
-			read += len;
-			while (end < read && (buf[end] != '\n'))
-				end++;
-
-			if (buf[end] == '\n') {
-				if (first) {
-					// check only for OK
-					first = false;
-					if (!compareBytes(buf, pos, end, RESPONSE_HTTP_OK, 15))
-						return -1;
-
-				} else if (end - pos == 1) {
-					// check empty line (header end)
-					end += 1;
-					break;
-				}
-
-				// String line = new String(buf, pos, end - pos - 1);
-				// Log.d(TAG, ">" + line + "< " + resp_len);
-
-				pos += (end - pos) + 1;
-				end = pos;
-			}
-		}
-
-		// check 4 bytes available..
-		while ((read - end) < 4 && (len = is.read(buf, read, BUFFER_SIZE - read)) >= 0)
-			read += len;
-
-		if (read - len < 4)
-			return -1;
-
-		int contentLength = decodeInt(buf, end);
-
-		// start of content
-		bufferPos = end + 4;
-		// buffer fill
-		bufferFill = read;
-		mBufferOffset = 0;
-
-		// overall bytes of already read
-		mReadPos = read;
-		mReadEnd = bufferPos + contentLength;
-
-		mInputStream = mResponseStream;
-
-		return 1;
-	}
-
-	boolean sendRequest(Tile tile) throws IOException {
-
-		bufferFill = 0;
-		bufferPos = 0;
-		mReadPos = 0;
-		mCacheFile = null;
-
-		if (mSocket != null && ((mMaxReq-- <= 0)
-				|| (SystemClock.elapsedRealtime() - mLastRequest
-				> RESPONSE_EXPECTED_TIMEOUT))) {
-			try {
-				mSocket.close();
-			} catch (IOException e) {
-
-			}
-
-			// Log.d(TAG, "not alive  - recreate connection " + mMaxReq);
-			mSocket = null;
-		}
-
-		if (mSocket == null) {
-			lwHttpConnect();
-			// we know our server
-			mMaxReq = RESPONSE_EXPECTED_LIVES;
-			// Log.d(TAG, "create connection");
-		} else {
-			// should not be needed
-			int avail = mResponseStream.available();
-			if (avail > 0) {
-				Log.d(TAG, "Consume left-over bytes: " + avail);
-				mResponseStream.read(buffer, 0, avail);
-			}
-		}
-
-		byte[] request = mRequestBuffer;
-		int pos = REQUEST_GET_START.length;
-
-		pos = writeInt(tile.zoomLevel, pos, request);
-		request[pos++] = '/';
-		pos = writeInt(tile.tileX, pos, request);
-		request[pos++] = '/';
-		pos = writeInt(tile.tileY, pos, request);
-
-		int len = REQUEST_GET_END.length;
-		System.arraycopy(REQUEST_GET_END, 0, request, pos, len);
-		len += pos;
-
-		// this does the same but with a few more allocations:
-		// byte[] request = String.format(REQUEST,
-		// Integer.valueOf(tile.zoomLevel),
-		// Integer.valueOf(tile.tileX), Integer.valueOf(tile.tileY)).getBytes();
-
-		try {
-			mCommandStream.write(request, 0, len);
-			mCommandStream.flush();
-			return true;
-		} catch (IOException e) {
-			Log.d(TAG, "recreate connection");
-		}
-
-		lwHttpConnect();
-
-		mCommandStream.write(request, 0, len);
-		mCommandStream.flush();
-
-		return true;
-	}
-
-	private boolean lwHttpConnect() throws IOException {
-		if (mSockAddr == null)
-			mSockAddr = new InetSocketAddress(mHost, mPort);
-
-		mSocket = new Socket();
-		mSocket.connect(mSockAddr, 30000);
-		mSocket.setTcpNoDelay(true);
-
-		mCommandStream = mSocket.getOutputStream(); //new BufferedOutputStream();
-		mResponseStream = mSocket.getInputStream();
-
-		return true;
-	}
-
-	// write (positive) integer as char sequence to buffer
-	private static int writeInt(int val, int pos, byte[] buf) {
-		if (val == 0) {
-			buf[pos] = '0';
-			return pos + 1;
-		}
-
-		int i = 0;
-		for (int n = val; n > 0; n = n / 10, i++)
-			buf[pos + i] = (byte) ('0' + n % 10);
-
-		// reverse bytes
-		for (int j = pos, end = pos + i - 1, mid = pos + i / 2; j < mid; j++, end--) {
-			byte tmp = buf[j];
-			buf[j] = buf[end];
-			buf[end] = tmp;
-		}
-
-		return pos + i;
-	}
-
-	private static boolean compareBytes(byte[] buffer, int position, int available,
-			byte[] string, int length) {
-
-		if (available - position < length)
-			return false;
-
-		for (int i = 0; i < length; i++)
-			if (buffer[position + i] != string[i])
-				return false;
-
-		return true;
-	}
-
-	static int decodeInt(byte[] buffer, int offset) {
-		return buffer[offset] << 24 | (buffer[offset + 1] & 0xff) << 16
-				| (buffer[offset + 2] & 0xff) << 8
-				| (buffer[offset + 3] & 0xff);
-	}
-
-	public boolean hasData() {
-		return mBufferOffset + bufferPos < mReadEnd;
-	}
-
-	public int position() {
-		return mBufferOffset + bufferPos;
-	}
-
-	public void readBuffer(int size) throws IOException {
-		// check if buffer already contains the request bytes
-		if (bufferPos + size < bufferFill)
-			return;
-
-		// check if inputstream is read to the end
-		if (mReadPos == mReadEnd)
-			return;
-
-		int maxSize = buffer.length;
-
-		if (size > maxSize) {
-			Log.d(TAG, "increase read buffer to " + size + " bytes");
-			maxSize = size;
-			byte[] tmp = new byte[maxSize];
-			bufferFill -= bufferPos;
-			System.arraycopy(buffer, bufferPos, tmp, 0, bufferFill);
-			mBufferOffset += bufferPos;
-			bufferPos = 0;
-			buffer = tmp;
-		}
-
-		if (bufferFill == bufferPos) {
-			mBufferOffset += bufferPos;
-			bufferPos = 0;
-			bufferFill = 0;
-		} else if (bufferPos + size > maxSize) {
-			// copy bytes left to the beginning of buffer
-			bufferFill -= bufferPos;
-			System.arraycopy(buffer, bufferPos, buffer, 0, bufferFill);
-			mBufferOffset += bufferPos;
-			bufferPos = 0;
-		}
-
-		int max = maxSize - bufferFill;
-
-		while ((bufferFill - bufferPos) < size && max > 0) {
-
-			max = maxSize - bufferFill;
-			if (max > mReadEnd - mReadPos)
-				max = (int) (mReadEnd - mReadPos);
-
-			// read until requested size is available in buffer
-			int len = mInputStream.read(buffer, bufferFill, max);
-
-			if (len < 0) {
-				// finished reading, mark end
-				buffer[bufferFill] = 0;
-				break;
-			}
-
-			mReadPos += len;
-
-			//			if (mCacheFile != null)
-			//				mCacheFile.write(mReadBuffer, mBufferFill, len);
-
-			if (mReadPos == mReadEnd)
-				break;
-
-			bufferFill += len;
-		}
-	}
-
-	private FileOutputStream mCacheFile;
-
-	boolean cacheRead(Tile tile, File f) {
-		if (f.exists() && f.length() > 0) {
-			FileInputStream in;
-
-			try {
-				in = new FileInputStream(f);
-
-				mReadEnd = f.length();
-				Log.d(TAG, tile + " - using cache: " + mReadEnd);
-				mInputStream = in;
-
-				//decode();
-				in.close();
-
-				return true;
-
-			} catch (FileNotFoundException e) {
-				e.printStackTrace();
-			} catch (Exception ex) {
-				ex.printStackTrace();
-			}
-
-			f.delete();
-			return false;
-		}
-
-		return false;
-	}
-
-	boolean cacheBegin(Tile tile, File f) {
-		if (MapDatabase.USE_CACHE) {
-			try {
-				Log.d(TAG, tile + " - writing cache");
-				mCacheFile = new FileOutputStream(f);
-
-				if (mReadPos > 0) {
-					try {
-						mCacheFile.write(buffer, bufferPos,
-								bufferFill - bufferPos);
-
-					} catch (IOException e) {
-						e.printStackTrace();
-						mCacheFile = null;
-						return false;
-					}
-				}
-			} catch (FileNotFoundException e) {
-				e.printStackTrace();
-				mCacheFile = null;
-				return false;
-			}
-		}
-		return true;
-	}
-
-	void cacheFinish(Tile tile, File file, boolean success) {
-		if (MapDatabase.USE_CACHE) {
-			if (success) {
-				try {
-					mCacheFile.flush();
-					mCacheFile.close();
-					Log.d(TAG, tile + " - cache written " + file.length());
-				} catch (IOException e) {
-					e.printStackTrace();
-				}
-			} else {
-				file.delete();
-			}
-		}
-		mCacheFile = null;
-	}
-}

src/org/oscim/database/oscimap2/MapDatabase.java

@@ -14,206 +14,32 @@
  */
 package org.oscim.database.oscimap2;
 
-import java.io.File;
 import java.io.IOException;
-import java.net.SocketException;
+import java.io.InputStream;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
 import java.util.Arrays;
 
-import org.oscim.core.BoundingBox;
-import org.oscim.core.GeoPoint;
 import org.oscim.core.GeometryBuffer.GeometryType;
 import org.oscim.core.MapElement;
 import org.oscim.core.Tag;
 import org.oscim.core.Tile;
-import org.oscim.database.IMapDatabase;
 import org.oscim.database.IMapDataSink;
-import org.oscim.database.MapInfo;
+import org.oscim.database.common.LwHttp;
-import org.oscim.database.MapOptions;
+import org.oscim.database.common.ProtobufDecoder;
-import org.oscim.layers.tile.MapTile;
+import org.oscim.database.common.ProtobufMapDatabase;
-import org.oscim.utils.UTF8Decoder;
 
-import android.os.Environment;
-import android.os.SystemClock;
 import android.util.Log;
 
 /**
- *
+ * Current Protocol Implementation
- *
  */
-public class MapDatabase implements IMapDatabase {
+public class MapDatabase extends ProtobufMapDatabase {
-	private static final String TAG = MapDatabase.class.getName();
-
-	static final boolean USE_CACHE = false;
-
-	private static final MapInfo mMapInfo =
-			new MapInfo(new BoundingBox(-180, -90, 180, 90),
-					new Byte((byte) 4), new GeoPoint(53.11, 8.85),
-					null, 0, 0, 0, "de", "comment", "author", null);
-
-	private static final String CACHE_DIRECTORY = "/Android/data/org.oscim.app/cache/";
-	private static final String CACHE_FILE = "%d-%d-%d.tile";
-
-	private final static float REF_TILE_SIZE = 4096.0f;
-
-	// 'open' state
-	private boolean mOpen = false;
-	private static File cacheDir;
-
-	private final int MAX_TILE_TAGS = 100;
-	private Tag[] curTags = new Tag[MAX_TILE_TAGS];
-	private int mCurTagCnt;
-
-	private IMapDataSink mMapGenerator;
-	private float mScaleFactor;
-	private MapTile mTile;
-
-	private final boolean debug = false;
-	private LwHttp lwHttp;
-
-	private final UTF8Decoder mStringDecoder;
-	private final MapElement mElem;
-
 	public MapDatabase() {
-		mStringDecoder = new UTF8Decoder();
+		super(new TileDecoder());
-		mElem = new MapElement();
+		mConn = new LwHttp("application/osmtile", "osmtile", false);
 	}
 
-	@Override
+	static class TileDecoder extends ProtobufDecoder {
-	public QueryResult executeQuery(MapTile tile, IMapDataSink mapDataSink) {
+		private final static String TAG = TileDecoder.class.getName();
-		QueryResult result = QueryResult.SUCCESS;
-
-		mTile = tile;
-
-		mMapGenerator = mapDataSink;
-
-		// scale coordinates to tile size
-		mScaleFactor = REF_TILE_SIZE / Tile.SIZE;
-
-		File f = null;
-
-		if (USE_CACHE) {
-			f = new File(cacheDir, String.format(CACHE_FILE,
-					Integer.valueOf(tile.zoomLevel),
-					Integer.valueOf(tile.tileX),
-					Integer.valueOf(tile.tileY)));
-
-			if (lwHttp.cacheRead(tile, f))
-				return QueryResult.SUCCESS;
-		}
-
-		try {
-
-			if (lwHttp.sendRequest(tile) && lwHttp.readHeader() >= 0) {
-				lwHttp.cacheBegin(tile, f);
-				decode();
-			} else {
-				Log.d(TAG, tile + " Network Error");
-				result = QueryResult.FAILED;
-			}
-		} catch (SocketException ex) {
-			Log.d(TAG, tile + " Socket exception: " + ex.getMessage());
-			result = QueryResult.FAILED;
-		} catch (SocketTimeoutException ex) {
-			Log.d(TAG, tile + " Socket Timeout exception: " + ex.getMessage());
-			result = QueryResult.FAILED;
-		} catch (UnknownHostException ex) {
-			Log.d(TAG, tile + " no network");
-			result = QueryResult.FAILED;
-		} catch (Exception ex) {
-			ex.printStackTrace();
-			result = QueryResult.FAILED;
-		}
-
-		lwHttp.mLastRequest = SystemClock.elapsedRealtime();
-
-		if (result == QueryResult.SUCCESS) {
-
-			lwHttp.cacheFinish(tile, f, true);
-		} else {
-			lwHttp.cacheFinish(tile, f, false);
-			lwHttp.close();
-		}
-		return result;
-	}
-
-	@Override
-	public String getMapProjection() {
-		return null;
-	}
-
-	@Override
-	public MapInfo getMapInfo() {
-		return mMapInfo;
-	}
-
-	@Override
-	public boolean isOpen() {
-		return mOpen;
-	}
-
-	@Override
-	public OpenResult open(MapOptions options) {
-		if (mOpen)
-			return OpenResult.SUCCESS;
-
-		if (options == null || !options.containsKey("url"))
-			return new OpenResult("options missing");
-
-		lwHttp = new LwHttp();
-
-		if (!lwHttp.setServer(options.get("url"))) {
-			return new OpenResult("invalid url: " + options.get("url"));
-		}
-
-		if (USE_CACHE) {
-			if (cacheDir == null) {
-				String externalStorageDirectory = Environment
-						.getExternalStorageDirectory()
-						.getAbsolutePath();
-				String cacheDirectoryPath = externalStorageDirectory + CACHE_DIRECTORY;
-				cacheDir = createDirectory(cacheDirectoryPath);
-			}
-		}
-
-		mOpen = true;
-		initDecorder();
-
-		return OpenResult.SUCCESS;
-	}
-
-	@Override
-	public void close() {
-		mOpen = false;
-
-		lwHttp.close();
-
-		if (USE_CACHE) {
-			cacheDir = null;
-		}
-	}
-
-	@Override
-	public void cancel() {
-	}
-
-	private static File createDirectory(String pathName) {
-		File file = new File(pathName);
-		if (!file.exists() && !file.mkdirs()) {
-			throw new IllegalArgumentException("could not create directory: " + file);
-		} else if (!file.isDirectory()) {
-			throw new IllegalArgumentException("not a directory: " + file);
-		} else if (!file.canRead()) {
-			throw new IllegalArgumentException("cannot read directory: " + file);
-		} else if (!file.canWrite()) {
-			throw new IllegalArgumentException("cannot write directory: " + file);
-		}
-		return file;
-	}
-
-	// /////////////// hand sewed tile protocol buffers decoder ///////////////
-
 		private static final int TAG_TILE_NUM_TAGS = 1;
 		private static final int TAG_TILE_TAG_KEYS = 2;
 		private static final int TAG_TILE_TAG_VALUES = 3;
@@ -233,26 +59,58 @@ public class MapDatabase implements IMapDatabase {
 		private static final int TAG_ELEM_MIN_HEIGHT = 32;
 		private static final int TAG_ELEM_PRIORITY = 41;
 
-	private short[] mTmpKeys = new short[100];
+		private short[] mSArray = new short[100];
 		private final Tag[] mTmpTags = new Tag[20];
-	private Tag[][] mElementTags;
+		private final Tag[][] mElementTags;
+		private final int MAX_TILE_TAGS = 100;
+		private Tag[] curTags = new Tag[MAX_TILE_TAGS];
+		private int mCurTagCnt;
+
+		// scale coordinates to tile size
+		private final static float REF_TILE_SIZE = 4096.0f;
+		private float mScale;
+
+		private Tile mTile;
+
+		private final MapElement mElem;
+
+		private IMapDataSink mMapDataSink;
+
+		TileDecoder() {
+			mElem = new MapElement();
 
-	private void initDecorder() {
 			// reusable tag set
 			Tag[][] tags = new Tag[10][];
 			for (int i = 0; i < 10; i++)
 				tags[i] = new Tag[i + 1];
 			mElementTags = tags;
+
 		}
 
-	private boolean decode() throws IOException {
+		@Override
+		public boolean decode(Tile tile, IMapDataSink sink, InputStream is, int contentLength)
+				throws IOException {
+
+			int byteCount = readUnsignedInt(is, buffer);
+			Log.d(TAG, tile + " contentLength:" + byteCount);
+			if (byteCount < 0) {
+				Log.d(TAG, "invalid contentLength: " + byteCount);
+				return false;
+			}
+
+			setInputStream(is, byteCount);
+
+			mTile = tile;
+			mMapDataSink = sink;
+
+			mScale = REF_TILE_SIZE / Tile.SIZE;
 
 			mCurTagCnt = 0;
 
 			int val;
 			int numTags = 0;
 
-		while (lwHttp.hasData() && (val = decodeVarint32()) > 0) {
+			while (hasData() && (val = decodeVarint32()) > 0) {
 				// read tag and wire type
 				int tag = (val >> 3);
 
@@ -264,7 +122,11 @@ public class MapDatabase implements IMapDatabase {
 						break;
 
 					case TAG_TILE_TAG_KEYS:
-					mTmpKeys = decodeShortArray(numTags, mTmpKeys);
+						int len = numTags;
+						if (mSArray.length < len)
+							mSArray = new short[len];
+
+						decodeVarintArray(numTags, mSArray);
 						break;
 
 					case TAG_TILE_TAG_VALUES:
@@ -289,7 +151,7 @@ public class MapDatabase implements IMapDatabase {
 		private boolean decodeTileTags(int curTag) throws IOException {
 			String tagString = decodeString();
 
-		String key = Tags.keys[mTmpKeys[curTag]];
+			String key = Tags.keys[mSArray[curTag]];
 			Tag tag;
 
 			if (key == Tag.TAG_KEY_NAME)
@@ -304,14 +166,16 @@ public class MapDatabase implements IMapDatabase {
 		}
 
 		private int decodeWayIndices(int indexCnt) throws IOException {
-		mElem.index = decodeShortArray(indexCnt, mElem.index);
+			mElem.ensureIndexSize(indexCnt, false);
+			decodeVarintArray(indexCnt, mElem.index);
 
 			short[] index = mElem.index;
 			int coordCnt = 0;
 
-		for (int i = 0; i < indexCnt; i++)
+			for (int i = 0; i < indexCnt; i++) {
-			coordCnt += index[i] *= 2;
+				coordCnt += index[i];
-
+				index[i] *= 2;
+			}
 			// set end marker
 			if (indexCnt < index.length)
 				index[indexCnt] = -1;
@@ -325,15 +189,14 @@ public class MapDatabase implements IMapDatabase {
 			Tag[] tags = null;
 			short[] index = null;
 
-		int end = lwHttp.position() + bytes;
+			int end = position() + bytes;
 			int indexCnt = 1;
 
-		boolean skip = false;
 			boolean fail = false;
 
 			int coordCnt = 0;
 			if (type == TAG_TILE_POINT) {
-			coordCnt = 2;
+				coordCnt = 1;
 				mElem.index[0] = 2;
 			}
 
@@ -342,7 +205,7 @@ public class MapDatabase implements IMapDatabase {
 			mElem.height = 0;
 			mElem.minHeight = 0;
 
-		while (lwHttp.position() < end) {
+			while (position() < end) {
 				// read tag and wire type
 				int val = decodeVarint32();
 				if (val == 0)
@@ -366,9 +229,10 @@ public class MapDatabase implements IMapDatabase {
 					case TAG_ELEM_COORDS:
 						if (coordCnt == 0) {
 							Log.d(TAG, mTile + " no coordinates");
-						skip = true;
 						}
-					int cnt = decodeWayCoordinates(skip, coordCnt);
+
+						mElem.ensurePointSize(coordCnt, false);
+						int cnt = decodeInterleavedPoints(mElem.points, mScale);
 
 						if (cnt != coordCnt) {
 							Log.d(TAG, mTile + " wrong number of coordintes");
@@ -418,7 +282,7 @@ public class MapDatabase implements IMapDatabase {
 					break;
 			}
 
-		mMapGenerator.process(mElem);
+			mMapDataSink.process(mElem);
 
 			return true;
 		}
@@ -429,10 +293,10 @@ public class MapDatabase implements IMapDatabase {
 			Tag[] tmp = mTmpTags;
 
 			int cnt = 0;
-		int end = lwHttp.position() + bytes;
+			int end = position() + bytes;
 			int max = mCurTagCnt;
 
-		while (lwHttp.position() < end) {
+			while (position() < end) {
 				int tagNum = decodeVarint32();
 
 				if (tagNum < 0) {
@@ -467,199 +331,5 @@ public class MapDatabase implements IMapDatabase {
 
 			return tags;
 		}
-
-	private final static int VARINT_LIMIT = 5;
-	private final static int VARINT_MAX = 10;
-
-	private int decodeWayCoordinates(boolean skip, int nodes) throws IOException {
-		int bytes = decodeVarint32();
-
-		lwHttp.readBuffer(bytes);
-
-		if (skip) {
-			lwHttp.bufferPos += bytes;
-			return nodes;
-		}
-
-		int cnt = 0;
-
-		int lastX = 0;
-		int lastY = 0;
-		boolean even = true;
-
-		float scale = mScaleFactor;
-		float[] coords = mElem.ensurePointSize(nodes, false);
-
-		byte[] buf = lwHttp.buffer;
-		int pos = lwHttp.bufferPos;
-		int end = pos + bytes;
-		int val;
-
-		while (pos < end) {
-			if (buf[pos] >= 0) {
-				val = buf[pos++];
-
-			} else if (buf[pos + 1] >= 0) {
-				val = (buf[pos++] & 0x7f)
-						| buf[pos++] << 7;
-
-			} else if (buf[pos + 2] >= 0) {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++] & 0x7f) << 7
-						| (buf[pos++]) << 14;
-
-			} else if (buf[pos + 3] >= 0) {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++] & 0x7f) << 7
-						| (buf[pos++] & 0x7f) << 14
-						| (buf[pos++]) << 21;
-
-			} else {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++] & 0x7f) << 7
-						| (buf[pos++] & 0x7f) << 14
-						| (buf[pos++] & 0x7f) << 21
-						| (buf[pos]) << 28;
-
-				int max = pos + VARINT_LIMIT;
-				while (pos < max)
-					if (buf[pos++] >= 0)
-						break;
-
-				if (pos == max)
-					throw new IOException("malformed VarInt32 in " + mTile);
-			}
-
-			// zigzag decoding
-			int s = ((val >>> 1) ^ -(val & 1));
-
-			if (even) {
-				lastX = lastX + s;
-				coords[cnt++] = lastX / scale;
-				even = false;
-			} else {
-				lastY = lastY + s;
-				coords[cnt++] = lastY / scale;
-				even = true;
-			}
-		}
-
-		lwHttp.bufferPos = pos;
-
-		return cnt;
-	}
-
-	private short[] decodeShortArray(int num, short[] array) throws IOException {
-		int bytes = decodeVarint32();
-
-		if (array.length < num)
-			array = new short[num];
-
-		lwHttp.readBuffer(bytes);
-
-		int cnt = 0;
-
-		byte[] buf = lwHttp.buffer;
-		int pos = lwHttp.bufferPos;
-		int end = pos + bytes;
-		int val;
-
-		while (pos < end) {
-			if (buf[pos] >= 0) {
-				val = buf[pos++];
-			} else if (buf[pos + 1] >= 0) {
-				val = (buf[pos++] & 0x7f)
-						| buf[pos++] << 7;
-			} else if (buf[pos + 2] >= 0) {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++] & 0x7f) << 7
-						| (buf[pos++]) << 14;
-			} else if (buf[pos + 3] >= 0) {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++] & 0x7f) << 7
-						| (buf[pos++] & 0x7f) << 14
-						| (buf[pos++]) << 21;
-			} else {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++] & 0x7f) << 7
-						| (buf[pos++] & 0x7f) << 14
-						| (buf[pos++] & 0x7f) << 21
-						| (buf[pos]) << 28;
-
-				int max = pos + VARINT_LIMIT;
-				while (pos < max)
-					if (buf[pos++] >= 0)
-						break;
-
-				if (pos == max)
-					throw new IOException("malformed VarInt32 in " + mTile);
-			}
-
-			array[cnt++] = (short) val;
-		}
-
-		lwHttp.bufferPos = pos;
-
-		return array;
-	}
-
-	private int decodeVarint32() throws IOException {
-		if (lwHttp.bufferPos + VARINT_MAX > lwHttp.bufferFill)
-			lwHttp.readBuffer(4096);
-
-		byte[] buf = lwHttp.buffer;
-		int pos = lwHttp.bufferPos;
-		int val;
-
-		if (buf[pos] >= 0) {
-			val = buf[pos++];
-		} else {
-
-			if (buf[pos + 1] >= 0) {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++]) << 7;
-
-			} else if (buf[pos + 2] >= 0) {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++] & 0x7f) << 7
-						| (buf[pos++]) << 14;
-
-			} else if (buf[pos + 3] >= 0) {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++] & 0x7f) << 7
-						| (buf[pos++] & 0x7f) << 14
-						| (buf[pos++]) << 21;
-			} else {
-				val = (buf[pos++] & 0x7f)
-						| (buf[pos++] & 0x7f) << 7
-						| (buf[pos++] & 0x7f) << 14
-						| (buf[pos++] & 0x7f) << 21
-						| (buf[pos]) << 28;
-
-				// 'Discard upper 32 bits'
-				int max = pos + VARINT_LIMIT;
-				while (pos < max)
-					if (buf[pos++] >= 0)
-						break;
-
-				if (pos == max)
-					throw new IOException("malformed VarInt32 in " + mTile);
-			}
-		}
-
-		lwHttp.bufferPos = pos;
-
-		return val;
-	}
-
-	private String decodeString() throws IOException {
-		final int size = decodeVarint32();
-		lwHttp.readBuffer(size);
-		final String result = mStringDecoder.decode(lwHttp.buffer, lwHttp.bufferPos, size);
-
-		lwHttp.bufferPos += size;
-
-		return result;
-
 	}
 }

src/org/oscim/database/oscimap4/MapDatabase.java

@@ -14,127 +14,16 @@
  */
 package org.oscim.database.oscimap4;
 
-import java.io.InputStream;
-import java.net.SocketException;
-import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
-
-import org.oscim.core.BoundingBox;
-import org.oscim.core.GeoPoint;
-import org.oscim.database.IMapDataSink;
-import org.oscim.database.IMapDatabase;
-import org.oscim.database.MapInfo;
-import org.oscim.database.MapOptions;
 import org.oscim.database.common.LwHttp;
-import org.oscim.layers.tile.MapTile;
+import org.oscim.database.common.ProtobufMapDatabase;
-
-import android.util.Log;
 
 /**
- *
+ * Protocol Version in Development
- *
  */
-public class MapDatabase implements IMapDatabase {
+public class MapDatabase extends ProtobufMapDatabase {
-	private static final String TAG = MapDatabase.class.getName();
 
-	static final boolean USE_CACHE = false;
+	public MapDatabase() {
-
+		super(new TileDecoder());
-	private static final MapInfo mMapInfo =
+		mConn = new LwHttp("application/x-protobuf", "vtm", false);
-			new MapInfo(new BoundingBox(-180, -90, 180, 90),
-					new Byte((byte) 4), new GeoPoint(53.11, 8.85),
-					null, 0, 0, 0, "de", "comment", "author", null);
-	// 'open' state
-	private boolean mOpen = false;
-	private LwHttp conn;
-	private TileDecoder mTileDecoder;
-
-	@Override
-	public QueryResult executeQuery(MapTile tile, IMapDataSink sink) {
-		QueryResult result = QueryResult.SUCCESS;
-
-		try {
-			InputStream is;
-			if (!conn.sendRequest(tile)) {
-				Log.d(TAG, tile + " Request Failed");
-				result = QueryResult.FAILED;
-			} else if ((is = conn.readHeader()) != null) {
-				boolean win = mTileDecoder.decode(is, tile, sink);
-				if (!win)
-					Log.d(TAG, tile + " failed");
-			} else {
-				Log.d(TAG, tile + " Network Error");
-				result = QueryResult.FAILED;
-			}
-		} catch (SocketException e) {
-			Log.d(TAG, tile + " Socket exception: " + e.getMessage());
-			result = QueryResult.FAILED;
-		} catch (SocketTimeoutException e) {
-			Log.d(TAG, tile + " Socket Timeout");
-			result = QueryResult.FAILED;
-		} catch (UnknownHostException e) {
-			Log.d(TAG, tile + " No Network");
-			result = QueryResult.FAILED;
-		} catch (Exception e) {
-			e.printStackTrace();
-			result = QueryResult.FAILED;
-		}
-
-		conn.requestCompleted();
-
-		if (result == QueryResult.SUCCESS) {
-
-			//conn.cacheFinish(tile, f, true);
-		} else {
-			//conn.cacheFinish(tile, f, false);
-			conn.close();
-		}
-		return result;
-	}
-
-	@Override
-	public String getMapProjection() {
-		return null;
-	}
-
-	@Override
-	public MapInfo getMapInfo() {
-		return mMapInfo;
-	}
-
-	@Override
-	public boolean isOpen() {
-		return mOpen;
-	}
-
-	@Override
-	public OpenResult open(MapOptions options) {
-		String extension = ".vtm";
-
-		if (mOpen)
-			return OpenResult.SUCCESS;
-
-		if (options == null || !options.containsKey("url"))
-			return new OpenResult("No URL in MapOptions");
-
-		conn = new LwHttp();
-
-		if (!conn.setServer(options.get("url"), extension, false)) {
-			return new OpenResult("invalid url: " + options.get("url"));
-		}
-
-		mOpen = true;
-		mTileDecoder = new TileDecoder();
-
-		return OpenResult.SUCCESS;
-	}
-
-	@Override
-	public void close() {
-		mOpen = false;
-		conn.close();
-	}
-
-	@Override
-	public void cancel() {
 	}
 }

src/org/oscim/database/oscimap4/TileDecoder.java

@@ -31,9 +31,6 @@ import android.util.Log;
 public class TileDecoder extends ProtobufDecoder {
 	private final static String TAG = TileDecoder.class.getName();
 
-	private final MapElement mElem;
-	private Tile mTile;
-
 	private static final int TAG_TILE_VERSION = 1;
 	//private static final int TAG_TILE_TIMESTAMP = 2;
 	//private static final int TAG_TILE_ISWATER = 3;
@@ -62,6 +59,10 @@ public class TileDecoder extends ProtobufDecoder {
 	//private static final int TAG_ELEM_PRIORITY = 41;
 
 	private short[] mSArray = new short[100];
+
+	private Tile mTile;
+
+	private final MapElement mElem;
 	private final Tag[][] mElementTags;
 
 	private final TagSet curTags = new TagSet(100);
@@ -81,7 +82,8 @@ public class TileDecoder extends ProtobufDecoder {
 
 	}
 
-	boolean decode(InputStream is, Tile tile, IMapDataSink sink)
+	@Override
+	public boolean decode(Tile tile, IMapDataSink sink, InputStream is, int contentLength)
 			throws IOException {
 
 		int byteCount = readUnsignedInt(is, buffer);
@@ -229,14 +231,15 @@ public class TileDecoder extends ProtobufDecoder {
 
 	private int decodeWayIndices(int indexCnt) throws IOException {
 		mElem.ensureIndexSize(indexCnt, false);
-		//mElem.index =
 		decodeVarintArray(indexCnt, mElem.index);
 
 		short[] index = mElem.index;
 		int coordCnt = 0;
 
-		for (int i = 0; i < indexCnt; i++)
+		for (int i = 0; i < indexCnt; i++) {
-			coordCnt += index[i] *= 2;
+			coordCnt += index[i];
+			index[i] *= 2;
+		}
 
 		// set end marker
 		if (indexCnt < index.length)
@@ -260,7 +263,7 @@ public class TileDecoder extends ProtobufDecoder {
 
 		int coordCnt = 0;
 		if (type == TAG_TILE_POINT) {
-			coordCnt = 2;
+			coordCnt = 1;
 			mElem.index[0] = 2;
 		}
 
@@ -297,14 +300,14 @@ public class TileDecoder extends ProtobufDecoder {
 				case TAG_ELEM_COORDS:
 					if (coordCnt == 0) {
 						Log.d(TAG, mTile + " no coordinates");
-						//skip = true;
 					}
 
 					mElem.ensurePointSize(coordCnt, false);
-					int cnt = decodeInterleavedPoints(mElem.points, coordCnt, mScaleFactor);
+					int cnt = decodeInterleavedPoints(mElem.points, mScaleFactor);
 
 					if (cnt != coordCnt) {
-						Log.d(TAG, mTile + " wrong number of coordintes");
+						Log.d(TAG, mTile + " wrong number of coordintes "
+								+ coordCnt + "/" + cnt);
 						fail = true;
 					}
 					break;

src/org/oscim/layers/tile/vector/MapTileLayer.java

@@ -73,7 +73,6 @@ public class MapTileLayer extends TileLayer<MapTileLoader> {
 
 		pauseLoaders(true);
 
-		//mJobQueue.clear();
 		mTileManager.clearJobs();
 
 		mMapOptions = options;
@@ -97,12 +96,10 @@ public class MapTileLayer extends TileLayer<MapTileLoader> {
 				mMapDatabase = mapDatabase;
 		}
 
-		if (options.db == MapDatabases.OSCIMAP_READER ||
+		if (options.db == MapDatabases.OPENSCIENCEMAP1)
-				options.db == MapDatabases.MAPSFORGE_FILE ||
-				options.db == MapDatabases.TEST_READER)
-			MapView.enableClosePolygons = true;
-		else
 			MapView.enableClosePolygons = false;
+		else
+			MapView.enableClosePolygons = true;
 
 		mTileManager.setZoomTable(mMapDatabase.getMapInfo().zoomLevel);