diff --git a/src/org/oscim/core/MapElement.java b/src/org/oscim/core/MapElement.java index f47968dc..e441bd14 100644 --- a/src/org/oscim/core/MapElement.java +++ b/src/org/oscim/core/MapElement.java @@ -16,7 +16,7 @@ package org.oscim.core; /** * MapElement is created by MapDatabase(s) and passed to MapTileLoader - * via IMapDatabaseCallback.renderElement() MapTileLoader processes the + * via IMapDataSink.renderElement() MapTileLoader processes the * data into MapTile.layers. * ----- * This is really just a buffer object that belongs to MapDatabase, so diff --git a/src/org/oscim/database/IMapDatabaseCallback.java b/src/org/oscim/database/IMapDataSink.java similarity index 73% rename from src/org/oscim/database/IMapDatabaseCallback.java rename to src/org/oscim/database/IMapDataSink.java index 4f0b879b..d8d66025 100644 --- a/src/org/oscim/database/IMapDatabaseCallback.java +++ b/src/org/oscim/database/IMapDataSink.java @@ -18,12 +18,12 @@ import org.oscim.core.MapElement; /** - * MapDatabase callbacks (implemented by MapTileLoader) - * ____ - * NOTE: All parameters passed belong to the caller! i.e. dont hold - * references to any arrays after callback function returns. + * MapDatabase callback (implemented by MapTileLoader) + * . + * NOTE: MapElement passed belong to the caller! i.e. dont hold + * references to its arrays after callback function returns. */ -public interface IMapDatabaseCallback { +public interface IMapDataSink { - void renderElement(MapElement element); + void process(MapElement element); } diff --git a/src/org/oscim/database/IMapDatabase.java b/src/org/oscim/database/IMapDatabase.java index ee1664ff..71ea05cd 100644 --- a/src/org/oscim/database/IMapDatabase.java +++ b/src/org/oscim/database/IMapDatabase.java @@ -28,12 +28,12 @@ public interface IMapDatabase { * * @param tile * the tile to read. - * @param mapDatabaseCallback + * @param mapDataSink * the callback which handles the extracted map elements. * @return true if successful */ abstract QueryResult executeQuery(MapTile tile, - IMapDatabaseCallback mapDatabaseCallback); + IMapDataSink mapDataSink); /** * @return the metadata for the current map file. diff --git a/src/org/oscim/database/mapnik/LwHttp.java b/src/org/oscim/database/common/LwHttp.java similarity index 54% rename from src/org/oscim/database/mapnik/LwHttp.java rename to src/org/oscim/database/common/LwHttp.java index 03facef8..87c04c40 100644 --- a/src/org/oscim/database/mapnik/LwHttp.java +++ b/src/org/oscim/database/common/LwHttp.java @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Lesser General Public License along with * this program. If not, see . */ -package org.oscim.database.mapnik; +package org.oscim.database.common; import java.io.BufferedInputStream; import java.io.IOException; @@ -30,47 +30,34 @@ import org.oscim.core.Tile; import android.os.SystemClock; import android.util.Log; -public class LwHttp { +public class LwHttp extends InputStream{ private static final String TAG = LwHttp.class.getName(); - private final static int BUFFER_SIZE = 65536; + //private static final boolean DEBUG = false; - // - byte[] buffer = new byte[BUFFER_SIZE]; + private final static byte[] RESPONSE_HTTP_OK = "200 OK".getBytes(); + private final static int RESPONSE_EXPECTED_LIVES = 100; + private final static int RESPONSE_TIMEOUT = 10000; - // position in buffer - int bufferPos; - - // bytes available in buffer - int bufferFill; - - // offset of buffer in message - private int mBufferOffset; + private final static int BUFFER_SIZE = 1024; + private final byte[] buffer = new byte[BUFFER_SIZE]; private String mHost; private int mPort; - private InputStream mInputStream; private int mMaxReq = 0; private Socket mSocket; private OutputStream mCommandStream; - private BufferedInputStream mResponseStream; - long mLastRequest = 0; + private InputStream mResponseStream; + private long mLastRequest = 0; private SocketAddress mSockAddr; - //private final static byte[] RESPONSE_HTTP_OK = "HTTP/1.1 200 OK".getBytes(); - private final static byte[] RESPONSE_HTTP_OK = "200 OK".getBytes(); - private final static byte[] RESPONSE_CONTENT_LEN = "Content-Length: ".getBytes(); - private final static int RESPONSE_EXPECTED_LIVES = 100; - private final static int RESPONSE_EXPECTED_TIMEOUT = 10000; - private byte[] REQUEST_GET_START; private byte[] REQUEST_GET_END; - private byte[] mRequestBuffer; - boolean setServer(String urlString) { - urlString = "http://d1s11ojcu7opje.cloudfront.net/dev/764e0b8d"; + private boolean mInflateContent; + public boolean setServer(String urlString, String extension, boolean zlibDeflate) { URL url; try { url = new URL(urlString); @@ -87,14 +74,13 @@ public class LwHttp { String host = url.getHost(); String path = url.getPath(); - Log.d(TAG, "open database: " + host + " " + port + " " + path); + Log.d(TAG, "open oscim database: " + host + " " + port + " " + path); REQUEST_GET_START = ("GET " + path).getBytes(); - REQUEST_GET_END = (".vector.pbf HTTP/1.1\n" + - "User-Agent: Wget/1.13.4 (linux-gnu)\n" + - "Accept: */*\n" + + REQUEST_GET_END = (extension + " HTTP/1.1\n" + "Host: " + host + "\n" + "Connection: Keep-Alive\n\n").getBytes(); + mInflateContent = zlibDeflate; mHost = host; mPort = port; @@ -102,11 +88,11 @@ public class LwHttp { mRequestBuffer = new byte[1024]; System.arraycopy(REQUEST_GET_START, 0, mRequestBuffer, 0, REQUEST_GET_START.length); - return true; } - void close() { + @Override + public void close() { if (mSocket != null) { try { mSocket.close(); @@ -118,23 +104,24 @@ public class LwHttp { } } - int readHeader() throws IOException { + public InputStream readHeader() throws IOException { InputStream is = mResponseStream; - mResponseStream.mark(1 << 16); + is.mark(4096); byte[] buf = buffer; boolean first = true; + int read = 0; int pos = 0; int end = 0; int len = 0; - int contentLength = 0; - // header cannot be larger than BUFFER_SIZE for this to work for (; pos < read || (len = is.read(buf, read, BUFFER_SIZE - read)) >= 0; len = 0) { read += len; + + // end of header lines while (end < read && (buf[end] != '\n')) end++; @@ -142,30 +129,17 @@ public class LwHttp { if (first) { // check only for OK first = false; - if (!compareBytes(buf, pos + 9, end, RESPONSE_HTTP_OK, 6)){ + if (!compareBytes(buf, pos + 9, end, RESPONSE_HTTP_OK, 6)) { String line = new String(buf, pos, end - pos - 1); Log.d(TAG, ">" + line + "< "); - return -1; + return null; } } else if (end - pos == 1) { // check empty line (header end) end += 1; break; } - else { - // parse Content-Length, TODO just encode this with message - for (int i = 0; pos + i < end - 1; i++) { - if (i < 16) { - if (buf[pos + i] == RESPONSE_CONTENT_LEN[i]) - continue; - break; - } - - // read int value - contentLength = contentLength * 10 + (buf[pos + i]) - '0'; - } - } //String line = new String(buf, pos, end - pos - 1); //Log.d(TAG, ">" + line + "< "); @@ -175,32 +149,21 @@ public class LwHttp { } // back to start of content - mResponseStream.reset(); - mResponseStream.mark(0); - mResponseStream.skip(end); + is.reset(); + is.mark(0); + is.skip(end); - // start of content - bufferPos = 0; - mBufferOffset = 0; + if (mInflateContent) + return new InflaterInputStream(is); - // buffer fill - bufferFill = 0; - - // decode zlib compressed content - mInputStream = new InflaterInputStream(mResponseStream); - - return 1; + return is; } - boolean sendRequest(Tile tile) throws IOException { - - bufferFill = 0; - bufferPos = 0; - //mReadPos = 0; + public boolean sendRequest(Tile tile) throws IOException { if (mSocket != null && ((mMaxReq-- <= 0) - || (SystemClock.elapsedRealtime() - mLastRequest - > RESPONSE_EXPECTED_TIMEOUT))) { + || (SystemClock.elapsedRealtime() - mLastRequest > RESPONSE_TIMEOUT))) { + try { mSocket.close(); } catch (IOException e) { @@ -217,37 +180,35 @@ public class LwHttp { mMaxReq = RESPONSE_EXPECTED_LIVES; // Log.d(TAG, "create connection"); } else { - // should not be needed int avail = mResponseStream.available(); - if (avail > 0) { - Log.d(TAG, "Consume left-over bytes: " + avail); - mResponseStream.read(buffer, 0, avail); + if (avail > 0){ + Log.d(TAG, "Consume left-over bytes: " + avail); + + while ((avail = mResponseStream.available()) > 0) + mResponseStream.read(buffer); + Log.d(TAG, "Consumed bytes"); } } byte[] request = mRequestBuffer; int pos = REQUEST_GET_START.length; + int newPos = 0; - request[pos++] = '/'; - request[pos++] = pos2hex(tile.tileX); - request[pos++] = pos2hex(tile.tileY); - request[pos++] = '/'; - pos = writeInt(tile.zoomLevel, pos, request); - request[pos++] = '/'; - pos = writeInt(tile.tileX, pos, request); - request[pos++] = '/'; - pos = writeInt(tile.tileY, pos, request); + if ((newPos = formatTilePath(tile, request, pos)) == 0) { + request[pos++] = '/'; + pos = writeInt(tile.zoomLevel, pos, request); + request[pos++] = '/'; + pos = writeInt(tile.tileX, pos, request); + request[pos++] = '/'; + pos = writeInt(tile.tileY, pos, request); + } else { + pos = newPos; + } int len = REQUEST_GET_END.length; System.arraycopy(REQUEST_GET_END, 0, request, pos, len); len += pos; - //Log.d(TAG, "request " + new String(request,0,len)); - // this does the same but with a few more allocations: - // byte[] request = String.format(REQUEST, - // Integer.valueOf(tile.zoomLevel), - // Integer.valueOf(tile.tileX), Integer.valueOf(tile.tileY)).getBytes(); - try { mCommandStream.write(request, 0, len); mCommandStream.flush(); @@ -264,14 +225,6 @@ public class LwHttp { return true; } - private final static byte[] hexTable = { - '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; - - private static byte pos2hex(int pos){ - return hexTable[(pos % 16)]; - } - private boolean lwHttpConnect() throws IOException { if (mSockAddr == null) mSockAddr = new InetSocketAddress(mHost, mPort); @@ -281,13 +234,13 @@ public class LwHttp { mSocket.setTcpNoDelay(true); mCommandStream = mSocket.getOutputStream(); - mResponseStream = new BufferedInputStream(mSocket.getInputStream()); + mResponseStream = new BufferedInputStream(mSocket.getInputStream(), 4096); return true; } // write (positive) integer as char sequence to buffer - private static int writeInt(int val, int pos, byte[] buf) { + protected static int writeInt(int val, int pos, byte[] buf) { if (val == 0) { buf[pos] = '0'; return pos + 1; @@ -326,72 +279,26 @@ public class LwHttp { | (buffer[offset + 3] & 0xff); } - public boolean hasData() { - try { - return readBuffer(1); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - return false; + public void requestCompleted() { + mLastRequest = SystemClock.elapsedRealtime(); } - public int position() { - return mBufferOffset + bufferPos; + /** + * Write custom tile url + * + * @param tile Tile + * @param path to write url string + * @param curPos current position + * @return new position + */ + protected int formatTilePath(Tile tile, byte[] path, int curPos) { + return 0; } - public boolean readBuffer(int size) throws IOException { - // check if buffer already contains the request bytes - if (bufferPos + size < bufferFill) - return true; - - // check if inputstream is read to the end - //if (mReadPos == mReadEnd) - // return; - - int maxSize = buffer.length; - - if (size > maxSize) { - Log.d(TAG, "increase read buffer to " + size + " bytes"); - maxSize = size; - byte[] tmp = new byte[maxSize]; - bufferFill -= bufferPos; - System.arraycopy(buffer, bufferPos, tmp, 0, bufferFill); - mBufferOffset += bufferPos; - bufferPos = 0; - buffer = tmp; - } - - if (bufferFill == bufferPos) { - mBufferOffset += bufferPos; - bufferPos = 0; - bufferFill = 0; - } else if (bufferPos + size > maxSize) { - // copy bytes left to the beginning of buffer - bufferFill -= bufferPos; - System.arraycopy(buffer, bufferPos, buffer, 0, bufferFill); - mBufferOffset += bufferPos; - bufferPos = 0; - } - - int max = maxSize - bufferFill; - - while ((bufferFill - bufferPos) < size && max > 0) { - - max = maxSize - bufferFill; - - // read until requested size is available in buffer - int len = mInputStream.read(buffer, bufferFill, max); - - if (len < 0) { - // finished reading, mark end - buffer[bufferFill] = 0; - return false; - } - - bufferFill += len; - } - return true; + @Override + public int read() throws IOException { + return mResponseStream.read(); } + } diff --git a/src/org/oscim/database/oscimap4/ProtobufDecoder.java b/src/org/oscim/database/common/ProtobufDecoder.java similarity index 52% rename from src/org/oscim/database/oscimap4/ProtobufDecoder.java rename to src/org/oscim/database/common/ProtobufDecoder.java index abe6a63b..b419fe9a 100644 --- a/src/org/oscim/database/oscimap4/ProtobufDecoder.java +++ b/src/org/oscim/database/common/ProtobufDecoder.java @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Lesser General Public License along with * this program. If not, see . */ -package org.oscim.database.oscimap4; +package org.oscim.database.common; import java.io.IOException; import java.io.InputStream; @@ -24,14 +24,14 @@ import android.util.Log; public class ProtobufDecoder { private final static String TAG = ProtobufDecoder.class.getName(); - private final static int VARINT_LIMIT = 5; + private final static int VARINT_LIMIT = 6; private final static int VARINT_MAX = 10; private final static int BUFFER_SIZE = 1 << 15; // 32kb - byte[] buffer = new byte[BUFFER_SIZE]; + protected byte[] buffer = new byte[BUFFER_SIZE]; // position in buffer - int bufferPos; + protected int bufferPos; // bytes available in buffer int bufferFill; @@ -40,28 +40,57 @@ public class ProtobufDecoder { private int mBufferOffset; // max bytes to read: message = header + content - private long mReadEnd; + private int mMsgEnd; // overall bytes of message read - private int mReadPos; + private int mMsgPos; private InputStream mInputStream; private final UTF8Decoder mStringDecoder; - public ProtobufDecoder(){ + public ProtobufDecoder() { mStringDecoder = new UTF8Decoder(); } - public void setInputStream(InputStream is){ - mInputStream = is; + protected static int readUnsignedInt(InputStream is, byte[] buf) throws IOException { + // check 4 bytes available.. + int read = 0; + int len = 0; + + while (read < 4 && (len = is.read(buf, read, 4 - read)) >= 0) + read += len; + + if (read < 4) + return read < 0 ? (read * 10) : read; + + return decodeInt(buf, 0); } - public void skip()throws IOException{ - int bytes = decodeVarint32(); - bufferPos += bytes; + static int decodeInt(byte[] buffer, int offset) { + return buffer[offset] << 24 | (buffer[offset + 1] & 0xff) << 16 + | (buffer[offset + 2] & 0xff) << 8 + | (buffer[offset + 3] & 0xff); } - public int readInterleavedPoints(float[] coords, int numPoints, float scale) throws IOException { + + public void setInputStream(InputStream is, int contentLength) { + mInputStream = is; + + bufferFill = 0; + bufferPos = 0; + mBufferOffset = 0; + + mMsgPos = 0; + mMsgEnd = contentLength; + } + +// public void skipAvailable() throws IOException { +// int bytes = decodeVarint32(); +// bufferPos += bytes; +// } + + protected int decodeInterleavedPoints(float[] coords, int numPoints, float scale) + throws IOException { int bytes = decodeVarint32(); readBuffer(bytes); @@ -71,8 +100,6 @@ public class ProtobufDecoder { int lastY = 0; boolean even = true; - //float[] coords = mElem.ensurePointSize(nodes, false); - byte[] buf = buffer; int pos = bufferPos; int end = pos + bytes; @@ -105,12 +132,9 @@ public class ProtobufDecoder { | (buf[pos]) << 28; int max = pos + VARINT_LIMIT; - while (pos < max) - if (buf[pos++] >= 0) - break; - - if (pos == max) - throw new IOException("malformed VarInt32"); + while (buf[pos++] < 0) + if (pos == max) + throw new IOException("malformed VarInt32"); } // zigzag decoding @@ -134,7 +158,7 @@ public class ProtobufDecoder { return cnt; } - public void readVarintArray(int num, short[] array) throws IOException { + public void decodeVarintArray(int num, short[] array) throws IOException { int bytes = decodeVarint32(); readBuffer(bytes); @@ -164,7 +188,7 @@ public class ProtobufDecoder { | (buf[pos++] & 0x7f) << 7 | (buf[pos++] & 0x7f) << 14 | (buf[pos++]) << 21; - } else if (buf[pos + 4] >= 0){ + } else if (buf[pos + 4] >= 0) { val = (buf[pos++] & 0x7f) | (buf[pos++] & 0x7f) << 7 | (buf[pos++] & 0x7f) << 14 @@ -182,7 +206,85 @@ public class ProtobufDecoder { bufferPos = pos; } - private int decodeVarint32() throws IOException { + /** + * fill short array from packed uint32. Array values must be positive + * as the end will be marked by -1 if the resulting array is larger + * than the input! + */ + protected short[] decodeUnsignedVarintArray(short[] array) throws IOException { + int bytes = decodeVarint32(); + + int arrayLength = 0; + if (array == null) { + arrayLength = 32; + array = new short[32]; + } + + readBuffer(bytes); + int cnt = 0; + + byte[] buf = buffer; + int pos = bufferPos; + int end = pos + bytes; + int val; + + while (pos < end) { + if (buf[pos] >= 0) { + val = buf[pos++]; + } else if (buf[pos + 1] >= 0) { + val = (buf[pos++] & 0x7f) + | buf[pos++] << 7; + } else if (buf[pos + 2] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++]) << 14; + } else if (buf[pos + 3] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++]) << 21; + } else { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++] & 0x7f) << 21 + | (buf[pos]) << 28; + + int max = pos + VARINT_LIMIT; + while (pos < max) + if (buf[pos++] >= 0) + break; + + if (pos > max) + throw new IOException("malformed VarInt32"); + } + + if (arrayLength <= cnt) { + arrayLength = cnt + 16; + short[] tmp = array; + array = new short[arrayLength]; + System.arraycopy(tmp, 0, array, 0, cnt); + } + + array[cnt++] = (short) val; + } + + bufferPos = pos; + + if (arrayLength > cnt) + array[cnt] = -1; + + return array; + } + + protected int decodeVarint32() throws IOException { + if (bufferPos + VARINT_MAX > bufferFill) + readBuffer(4096); + + return decodeVarint32Filled(); + } + + protected int decodeVarint32Filled() throws IOException { if (bufferPos + VARINT_MAX > bufferFill) readBuffer(4096); @@ -230,6 +332,50 @@ public class ProtobufDecoder { return val; } + // FIXME this also accept uin64 atm. +// protected int decodeVarint32Filled() throws IOException { +// +// if (buffer[bufferPos] >= 0) +// return buffer[bufferPos++]; +// +// byte[] buf = buffer; +// int pos = bufferPos; +// int val = 0; +// +// if (buf[pos + 1] >= 0) { +// val = (buf[pos++] & 0x7f) +// | (buf[pos++]) << 7; +// +// } else if (buf[pos + 2] >= 0) { +// val = (buf[pos++] & 0x7f) +// | (buf[pos++] & 0x7f) << 7 +// | (buf[pos++]) << 14; +// +// } else if (buf[pos + 3] >= 0) { +// val = (buf[pos++] & 0x7f) +// | (buf[pos++] & 0x7f) << 7 +// | (buf[pos++] & 0x7f) << 14 +// | (buf[pos++]) << 21; +// } else { +// val = (buf[pos++] & 0x7f) +// | (buf[pos++] & 0x7f) << 7 +// | (buf[pos++] & 0x7f) << 14 +// | (buf[pos++] & 0x7f) << 21 +// | (buf[pos]) << 28; +// +// +// // 'Discard upper 32 bits' +// int max = pos + VARINT_LIMIT; +// while (pos < max) +// if (buf[pos++] >= 0) +// break; +// if (pos == max) +// throw new IOException("malformed VarInt32"); +// } +// bufferPos = pos; +// +// return val; +// } public String decodeString() throws IOException { final int size = decodeVarint32(); @@ -238,7 +384,7 @@ public class ProtobufDecoder { String result; if (mStringDecoder == null) - result = new String(buffer,bufferPos, size, "UTF-8"); + result = new String(buffer, bufferPos, size, "UTF-8"); else result = mStringDecoder.decode(buffer, bufferPos, size); @@ -247,34 +393,83 @@ public class ProtobufDecoder { return result; } - public boolean hasData() { - return mBufferOffset + bufferPos < mReadEnd; + + public float decodeFloat() throws IOException { + if (bufferPos + 4 > bufferFill) + readBuffer(4096); + + byte[] buf = buffer; + int pos = bufferPos; + + int val = (buf[pos++] & 0xFF + | (buf[pos++] & 0xFF) << 8 + | (buf[pos++] & 0xFF) << 16 + | (buf[pos++] & 0xFF) << 24); + + bufferPos += 4; + return Float.intBitsToFloat(val); + } + + public double decodeDouble() throws IOException { + if (bufferPos + 8 > bufferFill) + readBuffer(4096); + + byte[] buf = buffer; + int pos = bufferPos; + + long val = (buf[pos++] & 0xFF + | (buf[pos++] & 0xFF) << 8 + | (buf[pos++] & 0xFF) << 16 + | (buf[pos++] & 0xFF) << 24 + | (buf[pos++] & 0xFF) << 32 + | (buf[pos++] & 0xFF) << 40 + | (buf[pos++] & 0xFF) << 48 + | (buf[pos++] & 0xFF) << 56); + + bufferPos += 8; + return Double.longBitsToDouble(val); + } + + public boolean decodeBool() throws IOException { + if (bufferPos + 1 > bufferFill) + readBuffer(4096); + + return buffer[bufferPos++] != 0; + } + + public boolean hasData() throws IOException { + if (mBufferOffset + bufferPos >= mMsgEnd) + return false; + + return readBuffer(1); } public int position() { return mBufferOffset + bufferPos; } - public void readBuffer(int size) throws IOException { + public boolean readBuffer(int size) throws IOException { // check if buffer already contains the request bytes if (bufferPos + size < bufferFill) - return; + return true; // check if inputstream is read to the end - if (mReadPos == mReadEnd) - return; + if (mMsgPos >= mMsgEnd) + return false; int maxSize = buffer.length; if (size > maxSize) { Log.d(TAG, "increase read buffer to " + size + " bytes"); maxSize = size; - byte[] tmp = new byte[maxSize]; bufferFill -= bufferPos; - System.arraycopy(buffer, bufferPos, tmp, 0, bufferFill); + + byte[] tmp = buffer; + buffer = new byte[maxSize]; + System.arraycopy(tmp, bufferPos, buffer, 0, bufferFill); + mBufferOffset += bufferPos; bufferPos = 0; - buffer = tmp; } if (bufferFill == bufferPos) { @@ -294,8 +489,8 @@ public class ProtobufDecoder { while ((bufferFill - bufferPos) < size && max > 0) { max = maxSize - bufferFill; - if (max > mReadEnd - mReadPos) - max = (int) (mReadEnd - mReadPos); + if (max > mMsgEnd - mMsgPos) + max = mMsgEnd - mMsgPos; // read until requested size is available in buffer int len = mInputStream.read(buffer, bufferFill, max); @@ -303,16 +498,16 @@ public class ProtobufDecoder { if (len < 0) { // finished reading, mark end buffer[bufferFill] = 0; - break; + return false; } - mReadPos += len; + mMsgPos += len; + bufferFill += len; - if (mReadPos == mReadEnd) + if (mMsgPos == mMsgEnd) break; - bufferFill += len; } + return true; } - } diff --git a/src/org/oscim/database/mapfile/MapDatabase.java b/src/org/oscim/database/mapfile/MapDatabase.java index e64439e6..36c3d811 100644 --- a/src/org/oscim/database/mapfile/MapDatabase.java +++ b/src/org/oscim/database/mapfile/MapDatabase.java @@ -22,8 +22,8 @@ import org.oscim.core.GeometryBuffer.GeometryType; import org.oscim.core.MapElement; import org.oscim.core.Tag; import org.oscim.core.Tile; +import org.oscim.database.IMapDataSink; import org.oscim.database.IMapDatabase; -import org.oscim.database.IMapDatabaseCallback; import org.oscim.database.MapOptions; import org.oscim.database.mapfile.header.MapFileHeader; import org.oscim.database.mapfile.header.MapFileInfo; @@ -210,7 +210,7 @@ public class MapDatabase implements IMapDatabase { * org.oscim.map.reader.MapDatabaseCallback) */ @Override - public QueryResult executeQuery(MapTile tile, IMapDatabaseCallback mapDatabaseCallback) { + public QueryResult executeQuery(MapTile tile, IMapDataSink mapDataSink) { if (sMapFileHeader == null) return QueryResult.FAILED; @@ -235,7 +235,7 @@ public class MapDatabase implements IMapDatabase { QueryCalculations.calculateBaseTiles(queryParameters, tile, subFileParameter); QueryCalculations.calculateBlocks(queryParameters, subFileParameter); - processBlocks(mapDatabaseCallback, queryParameters, subFileParameter); + processBlocks(mapDataSink, queryParameters, subFileParameter); } catch (IOException e) { Log.e(TAG, e.getMessage()); return QueryResult.FAILED; @@ -383,12 +383,12 @@ public class MapDatabase implements IMapDatabase { * the parameters of the current query. * @param subFileParameter * the parameters of the current map file. - * @param mapDatabaseCallback + * @param mapDataSink * the callback which handles the extracted map elements. */ private void processBlock(QueryParameters queryParameters, SubFileParameter subFileParameter, - IMapDatabaseCallback mapDatabaseCallback) { + IMapDataSink mapDataSink) { if (!processBlockSignature()) { return; } @@ -421,7 +421,7 @@ public class MapDatabase implements IMapDatabase { return; } - if (!processPOIs(mapDatabaseCallback, poisOnQueryZoomLevel)) { + if (!processPOIs(mapDataSink, poisOnQueryZoomLevel)) { return; } @@ -436,15 +436,13 @@ public class MapDatabase implements IMapDatabase { // move the pointer to the first way mReadBuffer.setBufferPosition(firstWayOffset); - if (!processWays(queryParameters, mapDatabaseCallback, waysOnQueryZoomLevel)) { + if (!processWays(queryParameters, mapDataSink, waysOnQueryZoomLevel)) { return; } } - - - private void processBlocks(IMapDatabaseCallback mapDatabaseCallback, + private void processBlocks(IMapDataSink mapDataSink, QueryParameters queryParameters, SubFileParameter subFileParameter) throws IOException { boolean queryIsWater = true; @@ -538,7 +536,7 @@ public class MapDatabase implements IMapDatabase { mTileLongitude = (int) (tileLongitudeDeg * 1000000); //try { - processBlock(queryParameters, subFileParameter, mapDatabaseCallback); + processBlock(queryParameters, subFileParameter, mapDataSink); //} catch (ArrayIndexOutOfBoundsException e) { // Log.e(TAG, e.getMessage()); //} @@ -579,14 +577,14 @@ public class MapDatabase implements IMapDatabase { /** * Processes the given number of POIs. * - * @param mapDatabaseCallback + * @param mapDataSink * the callback which handles the extracted POIs. * @param numberOfPois * how many POIs should be processed. * @return true if the POIs could be processed successfully, false * otherwise. */ - private boolean processPOIs(IMapDatabaseCallback mapDatabaseCallback, int numberOfPois) { + private boolean processPOIs(IMapDataSink mapDataSink, int numberOfPois) { Tag[] poiTags = sMapFileHeader.getMapFileInfo().poiTags; Tag[] tags = null; Tag[] curTags; @@ -671,14 +669,14 @@ public class MapDatabase implements IMapDatabase { mElem.startPoints(); mElem.addPoint(longitude, latitude); - mElem.type = GeometryType.POINT; + mElem.type = GeometryType.POINT; mElem.set(curTags, layer); - mapDatabaseCallback.renderElement(mElem); + mapDataSink.process(mElem); -// mGeom.points[0] = longitude; -// mGeom.points[1] = latitude; -// mGeom.index[0] = 2; -// mapDatabaseCallback.renderPOI(layer, curTags, mGeom); + // mGeom.points[0] = longitude; + // mGeom.points[1] = latitude; + // mGeom.index[0] = 2; + // mapDatabaseCallback.renderPOI(layer, curTags, mGeom); } @@ -821,7 +819,7 @@ public class MapDatabase implements IMapDatabase { * * @param queryParameters * the parameters of the current query. - * @param mapDatabaseCallback + * @param mapDataSink * the callback which handles the extracted ways. * @param numberOfWays * how many ways should be processed. @@ -829,7 +827,7 @@ public class MapDatabase implements IMapDatabase { * otherwise. */ private boolean processWays(QueryParameters queryParameters, - IMapDatabaseCallback mapDatabaseCallback, + IMapDataSink mapDataSink, int numberOfWays) { Tag[] tags = null; @@ -993,7 +991,7 @@ public class MapDatabase implements IMapDatabase { mElem.type = closed ? GeometryType.POLY : GeometryType.LINE; mElem.set(curTags, layer); - mapDatabaseCallback.renderElement(mElem); + mapDataSink.process(mElem); } } diff --git a/src/org/oscim/database/mapnik/MapDatabase.java b/src/org/oscim/database/mapnik/MapDatabase.java index 38284685..a552e5d1 100644 --- a/src/org/oscim/database/mapnik/MapDatabase.java +++ b/src/org/oscim/database/mapnik/MapDatabase.java @@ -14,29 +14,21 @@ */ package org.oscim.database.mapnik; -import java.io.IOException; +import java.io.InputStream; import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.UnknownHostException; -import java.util.ArrayList; import org.oscim.core.BoundingBox; import org.oscim.core.GeoPoint; -import org.oscim.core.GeometryBuffer.GeometryType; -import org.oscim.core.MapElement; -import org.oscim.core.Tag; -import org.oscim.core.TagSet; import org.oscim.core.Tile; +import org.oscim.database.IMapDataSink; import org.oscim.database.IMapDatabase; -import org.oscim.database.IMapDatabaseCallback; import org.oscim.database.MapInfo; import org.oscim.database.MapOptions; +import org.oscim.database.common.LwHttp; import org.oscim.layers.tile.MapTile; -import org.oscim.utils.UTF8Decoder; -import org.oscim.utils.pool.Inlist; -import org.oscim.utils.pool.Pool; -import android.os.SystemClock; import android.util.Log; public class MapDatabase implements IMapDatabase { @@ -49,42 +41,20 @@ public class MapDatabase implements IMapDatabase { new int[] { 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 } ); - private final static float REF_TILE_SIZE = 4096.0f; - // 'open' state private boolean mOpen = false; - private IMapDatabaseCallback mMapGenerator; - private float mScaleFactor; - private MapTile mTile; - - private LwHttp lwHttp; - - private final UTF8Decoder mStringDecoder; - private final String mLocale = "de"; - - //private final MapElement mElem; - - public MapDatabase() { - mStringDecoder = new UTF8Decoder(); - //mElem = new MapElement(); - } + private LwHttp conn; + private TileDecoder mTileDecoder; @Override - public QueryResult executeQuery(MapTile tile, IMapDatabaseCallback mapDatabaseCallback) { + public QueryResult executeQuery(MapTile tile, IMapDataSink mapDataSink) { QueryResult result = QueryResult.SUCCESS; - mTile = tile; - - mMapGenerator = mapDatabaseCallback; - - // scale coordinates to tile size - mScaleFactor = REF_TILE_SIZE / Tile.SIZE; - try { - - if (lwHttp.sendRequest(tile) && lwHttp.readHeader() >= 0) { - decode(); + InputStream is; + if (conn.sendRequest(tile) && (is = conn.readHeader()) != null) { + mTileDecoder.decode(is, tile, mapDataSink); } else { Log.d(TAG, tile + " Network Error"); result = QueryResult.FAILED; @@ -103,12 +73,12 @@ public class MapDatabase implements IMapDatabase { result = QueryResult.FAILED; } - lwHttp.mLastRequest = SystemClock.elapsedRealtime(); + conn.requestCompleted(); if (result != QueryResult.SUCCESS) { - lwHttp.close(); + conn.close(); } - Log.d(TAG, ">>> " + result + " >>> " + mTile); + return result; } @@ -124,28 +94,52 @@ public class MapDatabase implements IMapDatabase { @Override public OpenResult open(MapOptions options) { + String extension = ".vector.pbf"; if (mOpen) return OpenResult.SUCCESS; if (options == null || !options.containsKey("url")) return new OpenResult("options missing"); - lwHttp = new LwHttp(); + conn = new LwHttp() { - if (!lwHttp.setServer(options.get("url"))) { + @Override + protected int formatTilePath(Tile tile, byte[] path, int pos) { + // url formatter for mapbox streets + byte[] hexTable = { + '0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + + path[pos++] = '/'; + path[pos++] = hexTable[(tile.tileX) % 16]; + path[pos++] = hexTable[(tile.tileY) % 16]; + path[pos++] = '/'; + pos = LwHttp.writeInt(tile.zoomLevel, pos, path); + path[pos++] = '/'; + pos = LwHttp.writeInt(tile.tileX, pos, path); + path[pos++] = '/'; + pos = LwHttp.writeInt(tile.tileY, pos, path); + return pos; + } + }; + + if (!conn.setServer(options.get("url"), extension, true)) { return new OpenResult("invalid url: " + options.get("url")); } - mOpen = true; - initDecorder(); + mTileDecoder = new TileDecoder(); + mOpen = true; return OpenResult.SUCCESS; } @Override public void close() { mOpen = false; - lwHttp.close(); + + mTileDecoder = null; + conn.close(); + conn = null; } @Override @@ -156,743 +150,4 @@ public class MapDatabase implements IMapDatabase { @Override public void cancel() { } - - private static final int TAG_TILE_LAYERS = 3; - - private static final int TAG_LAYER_VERSION = 15; - private static final int TAG_LAYER_NAME = 1; - private static final int TAG_LAYER_FEATURES = 2; - private static final int TAG_LAYER_KEYS = 3; - private static final int TAG_LAYER_VALUES = 4; - private static final int TAG_LAYER_EXTENT = 5; - - private static final int TAG_FEATURE_ID = 1; - private static final int TAG_FEATURE_TAGS = 2; - private static final int TAG_FEATURE_TYPE = 3; - private static final int TAG_FEATURE_GEOMETRY = 4; - - private static final int TAG_VALUE_STRING = 1; - private static final int TAG_VALUE_FLOAT = 2; - private static final int TAG_VALUE_DOUBLE = 3; - private static final int TAG_VALUE_LONG = 4; - private static final int TAG_VALUE_UINT = 5; - private static final int TAG_VALUE_SINT = 6; - private static final int TAG_VALUE_BOOL = 7; - - private static final int TAG_GEOM_UNKNOWN = 0; - private static final int TAG_GEOM_POINT = 1; - private static final int TAG_GEOM_LINE = 2; - private static final int TAG_GEOM_POLYGON = 3; - - private short[] mTmpTags = new short[1024]; - - private void initDecorder() { - } - - private boolean decode() throws IOException { - - int val; - - while (lwHttp.hasData() && (val = decodeVarint32()) > 0) { - // read tag and wire type - int tag = (val >> 3); - - switch (tag) { - case TAG_TILE_LAYERS: - decodeLayer(); - break; - - default: - Log.d(TAG, mTile + " invalid type for tile: " + tag); - return false; - } - } - return true; - } - - private boolean decodeLayer() throws IOException { - - int version = 0; - int extent = 4096; - - int bytes = decodeVarint32(); - - ArrayList keys = new ArrayList(); - ArrayList values = new ArrayList(); - - String name = null; - int numFeatures = 0; - ArrayList features = new ArrayList(); - - int end = lwHttp.position() + bytes; - while (lwHttp.position() < end) { - // read tag and wire type - int val = decodeVarint32(); - if (val == 0) - break; - - int tag = (val >> 3); - - switch (tag) { - case TAG_LAYER_KEYS: - keys.add(decodeString()); - break; - - case TAG_LAYER_VALUES: - values.add(decodeValue()); - break; - - case TAG_LAYER_FEATURES: - numFeatures++; - decodeFeature(features); - break; - - case TAG_LAYER_VERSION: - version = decodeVarint32(); - break; - - case TAG_LAYER_NAME: - name = decodeString(); - break; - - case TAG_LAYER_EXTENT: - extent = decodeVarint32(); - break; - - default: - Log.d(TAG, mTile + " invalid type for layer: " + tag); - break; - } - - } - - Tag layerTag = new Tag(name, Tag.VALUE_YES); - - if (numFeatures == 0) - return true; - - int[] ignoreLocal = new int[20]; - int numIgnore = 0; - - int fallBackLocal = -1; - int matchedLocal = -1; - - for (int i = 0; i < keys.size(); i++) { - String key = keys.get(i); - if (!key.startsWith(Tag.TAG_KEY_NAME)) - continue; - int len = key.length(); - if (len == 4) { - fallBackLocal = i; - continue; - } - if (len < 7) { - ignoreLocal[numIgnore++] = i; - continue; - } - - if (mLocale.equals(key.substring(5))) { - //Log.d(TAG, "found local " + key); - matchedLocal = i; - } else - ignoreLocal[numIgnore++] = i; - - } - - for (Feature f : features) { - //Log.d(TAG, "geom: " + f.elem.type + " " + f.elem.pointPos + " tags:" + f.numTags + " " - // + name); - - if (f.elem.type == GeometryType.NONE) - continue; - - mTagSet.clear(); - mTagSet.add(layerTag); - - boolean hasName = false; - String fallbackName = null; - - tagLoop: for (int j = 0; j < (f.numTags << 1); j += 2) { - int keyIdx = f.tags[j]; - for (int i = 0; i < numIgnore; i++) - if (keyIdx == ignoreLocal[i]) - continue tagLoop; - - if (keyIdx == fallBackLocal) { - fallbackName = values.get(f.tags[j + 1]); - continue; - } - - String key; - String val = values.get(f.tags[j + 1]); - - if (keyIdx == matchedLocal) { - hasName = true; - mTagSet.add(new Tag(Tag.TAG_KEY_NAME, val, false)); - - } else { - key = keys.get(keyIdx); - mTagSet.add(new Tag(key, val)); - } - } - - if (!hasName && fallbackName != null) - mTagSet.add(new Tag(Tag.TAG_KEY_NAME, fallbackName, false)); - - // FIXME extract layer tag here - f.elem.set(mTagSet.asArray(), 5); - mMapGenerator.renderElement(f.elem); - mFeaturePool.release(f); - } - - return true; - } - - private final TagSet mTagSet = new TagSet(); - private final Pool mFeaturePool = new Pool() { - int count; - - @Override - protected Feature createItem() { - count++; - return new Feature(); - } - - @Override - protected boolean clearItem(Feature item) { - if (count > 100) { - count--; - return false; - } - - item.elem.tags = null; - item.elem.clear(); - item.tags = null; - item.type = 0; - item.numTags = 0; - - return true; - } - }; - - static class Feature extends Inlist { - short[] tags; - int numTags; - int type; - - final MapElement elem; - - Feature() { - elem = new MapElement(); - } - - boolean match(short otherTags[], int otherNumTags, int otherType) { - if (numTags != otherNumTags) - return false; - - if (type != otherType) - return false; - - for (int i = 0; i < numTags << 1; i++) { - if (tags[i] != otherTags[i]) - return false; - } - return true; - } - - } - - private void decodeFeature(ArrayList features) throws IOException { - int bytes = decodeVarint32(); - int end = lwHttp.position() + bytes; - - int type = 0; - long id; - - lastX = 0; - lastY = 0; - - mTmpTags[0] = -1; - - Feature curFeature = null; - int numTags = 0; - - //Log.d(TAG, "start feature"); - while (lwHttp.position() < end) { - // read tag and wire type - int val = decodeVarint32(); - if (val == 0) - break; - - int tag = (val >>> 3); - - switch (tag) { - case TAG_FEATURE_ID: - id = decodeVarint32(); - break; - - case TAG_FEATURE_TAGS: - mTmpTags = decodeShortArray(mTmpTags); - - for (; numTags < mTmpTags.length && mTmpTags[numTags] >= 0;) - numTags += 2; - - numTags >>= 1; - - break; - - case TAG_FEATURE_TYPE: - type = decodeVarint32(); - - //Log.d(TAG, "got type " + type); - - break; - - case TAG_FEATURE_GEOMETRY: - - for (Feature f : features) { - if (f.match(mTmpTags, numTags, type)) { - curFeature = f; - break; - } - } - - if (curFeature == null) { - curFeature = mFeaturePool.get(); - curFeature.tags = new short[numTags << 1]; - System.arraycopy(mTmpTags, 0, curFeature.tags, 0, numTags << 1); - curFeature.numTags = numTags; - curFeature.type = type; - - features.add(curFeature); - } - - decodeCoordinates(type, curFeature); - break; - - default: - Log.d(TAG, mTile + " invalid type for feature: " + tag); - break; - } - } - } - - private final static int CLOSE_PATH = 0x07; - private final static int MOVE_TO = 0x01; - //private final static int LINE_TO = 0x02; - - private int lastX, lastY; - private final int pixel = 7; - - private int decodeCoordinates(int type, Feature feature) throws IOException { - int bytes = decodeVarint32(); - lwHttp.readBuffer(bytes); - - if (feature == null) { - lwHttp.bufferPos += bytes; - return 0; - } - - MapElement elem = feature.elem; - - boolean isPoint = false; - boolean isPoly = false; - boolean isLine = false; - - if (type == TAG_GEOM_LINE) { - elem.startLine(); - isLine = true; - } - else if (type == TAG_GEOM_POLYGON) { - elem.startPolygon(); - isPoly = true; - } else if (type == TAG_GEOM_POINT) { - isPoint = true; - elem.startPoints(); - } else if (type == TAG_GEOM_UNKNOWN) - elem.startPoints(); - - boolean even = true; - - float scale = mScaleFactor; - - byte[] buf = lwHttp.buffer; - int pos = lwHttp.bufferPos; - lwHttp.bufferPos += bytes; - - int end = pos + bytes; - int val; - - int curX = 0; - int curY = 0; - int prevX = 0; - int prevY = 0; - - int cmd = 0; - int num = 0; - - boolean first = true; - boolean lastClip = false; - - // test bbox for outer.. - boolean isOuter = mTile.zoomLevel < 14; - - int xmin = Integer.MAX_VALUE, xmax = Integer.MIN_VALUE; - int ymin = Integer.MAX_VALUE, ymax = Integer.MIN_VALUE; - - while (pos < end) { - if (buf[pos] >= 0) { - val = buf[pos++]; - - } else if (buf[pos + 1] >= 0) { - val = (buf[pos++] & 0x7f) - | buf[pos++] << 7; - - } else if (buf[pos + 2] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++]) << 14; - - } else if (buf[pos + 3] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++]) << 21; - - } else { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++] & 0x7f) << 21 - | (buf[pos]) << 28; - - int max = pos + VARINT_LIMIT; - while (pos < max) - if (buf[pos++] >= 0) - break; - - if (pos == max) - throw new IOException("malformed VarInt32 in " + mTile); - } - - if (num == 0) { - num = val >>> 3; - cmd = val & 0x07; - - if (isLine && lastClip) { - elem.addPoint(curX / scale, curY / scale); - lastClip = false; - } - - if (cmd == CLOSE_PATH) { - num = 0; - continue; - } - if (first) { - first = false; - continue; - } - if (cmd == MOVE_TO) { - if (type == TAG_GEOM_LINE) - elem.startLine(); - else if (type == TAG_GEOM_POLYGON) { - isOuter = false; - elem.startHole(); - } - } - continue; - } - // zigzag decoding - int s = ((val >>> 1) ^ -(val & 1)); - - if (even) { - // get x coordinate - even = false; - curX = lastX = lastX + s; - continue; - } - // get y coordinate and add point to geometry - num--; - - even = true; - curY = lastY = lastY + s; - - int dx = (curX - prevX); - int dy = (curY - prevY); - - if ((isPoint || cmd == MOVE_TO) - || (dx > pixel || dx < -pixel) - || (dy > pixel || dy < -pixel) - // dont clip at tile boundaries - || (curX <= 0 || curX >= 4095) - || (curY <= 0 || curY >= 4095)) { - - prevX = curX; - prevY = curY; - elem.addPoint(curX / scale, curY / scale); - lastClip = false; - - if (isOuter) { - if (curX < xmin) - xmin = curX; - if (curX > xmax) - xmax = curX; - - if (curY < ymin) - ymin = curY; - if (curY > ymax) - ymax = curY; - } - - continue; - } - lastClip = true; - } - - if (isPoly && isOuter && !testBBox(xmax - xmin, ymax - ymin)) { - //Log.d(TAG, "skip small poly "+ elem.indexPos + " > " - // + (xmax - xmin) * (ymax - ymin)); - elem.pointPos -= elem.index[elem.indexPos]; - if (elem.indexPos > 0) { - elem.indexPos -= 3; - elem.index[elem.indexPos + 1] = -1; - } else { - elem.type = GeometryType.NONE; - } - return 0; - } - - if (isLine && lastClip) - elem.addPoint(curX / scale, curY / scale); - - return 1; - } - - private static boolean testBBox(int dx, int dy) { - return dx * dy > 64 * 64; - } - - private String decodeValue() throws IOException { - int bytes = decodeVarint32(); - - String value = null; - - int end = lwHttp.position() + bytes; - - while (lwHttp.position() < end) { - // read tag and wire type - int val = decodeVarint32(); - if (val == 0) - break; - - int tag = (val >> 3); - - switch (tag) { - case TAG_VALUE_STRING: - value = decodeString(); - break; - - case TAG_VALUE_UINT: - value = String.valueOf(decodeVarint32()); - break; - - case TAG_VALUE_SINT: - value = String.valueOf(decodeVarint32()); - break; - - case TAG_VALUE_LONG: - value = String.valueOf(decodeVarint32()); - break; - - case TAG_VALUE_FLOAT: - value = String.valueOf(decodeFloat()); - break; - - case TAG_VALUE_DOUBLE: - value = String.valueOf(decodeDouble()); - break; - - case TAG_VALUE_BOOL: - value = decodeBool() ? "yes" : "no"; - break; - default: - break; - } - - } - return value; - } - - private final static int VARINT_LIMIT = 7; - private final static int VARINT_MAX = 10; - - private short[] decodeShortArray(short[] array) throws IOException { - int bytes = decodeVarint32(); - int arrayLength = array.length; - - lwHttp.readBuffer(bytes); - int cnt = 0; - - byte[] buf = lwHttp.buffer; - int pos = lwHttp.bufferPos; - int end = pos + bytes; - int val; - - while (pos < end) { - if (buf[pos] >= 0) { - val = buf[pos++]; - } else if (buf[pos + 1] >= 0) { - val = (buf[pos++] & 0x7f) - | buf[pos++] << 7; - } else if (buf[pos + 2] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++]) << 14; - } else if (buf[pos + 3] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++]) << 21; - } else { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++] & 0x7f) << 21 - | (buf[pos]) << 28; - - int max = pos + VARINT_LIMIT; - while (pos < max) - if (buf[pos++] >= 0) - break; - - if (pos == max) - throw new IOException("malformed VarInt32 in " + mTile); - } - - if (arrayLength <= cnt) { - arrayLength = cnt + 16; - short[] tmp = array; - array = new short[arrayLength]; - System.arraycopy(tmp, 0, array, 0, cnt); - } - - array[cnt++] = (short) val; - } - - lwHttp.bufferPos = pos; - - if (arrayLength > cnt) - array[cnt] = -1; - - return array; - } - - private int decodeVarint32() throws IOException { - if (lwHttp.bufferPos + VARINT_MAX > lwHttp.bufferFill) - lwHttp.readBuffer(4096); - - byte[] buf = lwHttp.buffer; - int pos = lwHttp.bufferPos; - int val; - - if (buf[pos] >= 0) { - val = buf[pos++]; - } else { - - if (buf[pos + 1] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++]) << 7; - - } else if (buf[pos + 2] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++]) << 14; - - } else if (buf[pos + 3] >= 0) { - val = (buf[pos++] & 0x7f - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++]) << 21); - } else { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++] & 0x7f) << 21 - | (buf[pos]) << 28; - - // 'Discard upper 32 bits' - int max = pos + VARINT_LIMIT; - while (pos < max) - if (buf[pos++] >= 0) - break; - - if (pos == max) - throw new IOException("malformed VarInt32 in " + mTile); - } - } - - lwHttp.bufferPos = pos; - - return val; - } - - private float decodeFloat() throws IOException { - if (lwHttp.bufferPos + 4 > lwHttp.bufferFill) - lwHttp.readBuffer(4096); - - byte[] buf = lwHttp.buffer; - int pos = lwHttp.bufferPos; - - int val = (buf[pos++] & 0xFF - | (buf[pos++] & 0xFF) << 8 - | (buf[pos++] & 0xFF) << 16 - | (buf[pos++] & 0xFF) << 24); - - lwHttp.bufferPos += 4; - - return Float.intBitsToFloat(val); - } - - private double decodeDouble() throws IOException { - if (lwHttp.bufferPos + 8 > lwHttp.bufferFill) - lwHttp.readBuffer(4096); - - byte[] buf = lwHttp.buffer; - int pos = lwHttp.bufferPos; - - long val = (buf[pos++] & 0xFF - | (buf[pos++] & 0xFF) << 8 - | (buf[pos++] & 0xFF) << 16 - | (buf[pos++] & 0xFF) << 24 - | (buf[pos++] & 0xFF) << 32 - | (buf[pos++] & 0xFF) << 40 - | (buf[pos++] & 0xFF) << 48 - | (buf[pos++] & 0xFF) << 56); - - lwHttp.bufferPos += 8; - - return Double.longBitsToDouble(val); - } - - private boolean decodeBool() throws IOException { - if (lwHttp.bufferPos + 1 > lwHttp.bufferFill) - lwHttp.readBuffer(4096); - - boolean val = lwHttp.buffer[lwHttp.bufferPos++] != 0; - - return val; - } - - private String decodeString() throws IOException { - final int size = decodeVarint32(); - lwHttp.readBuffer(size); - final String result = mStringDecoder.decode(lwHttp.buffer, lwHttp.bufferPos, size); - - //Log.d(TAG, "string: " + result); - - lwHttp.bufferPos += size; - - return result; - - } } diff --git a/src/org/oscim/database/mapnik/TileDecoder.java b/src/org/oscim/database/mapnik/TileDecoder.java new file mode 100644 index 00000000..410f22d7 --- /dev/null +++ b/src/org/oscim/database/mapnik/TileDecoder.java @@ -0,0 +1,571 @@ +/* + * Copyright 2013 + * + * This program is free software: you can redistribute it and/or modify it under the + * terms of the GNU Lesser General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * this program. If not, see . + */ +package org.oscim.database.mapnik; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; + +import org.oscim.core.GeometryBuffer.GeometryType; +import org.oscim.core.MapElement; +import org.oscim.core.Tag; +import org.oscim.core.TagSet; +import org.oscim.core.Tile; +import org.oscim.database.IMapDataSink; +import org.oscim.database.common.ProtobufDecoder; +import org.oscim.utils.pool.Inlist; +import org.oscim.utils.pool.Pool; + +import android.util.Log; + +public class TileDecoder extends ProtobufDecoder { + private final static String TAG = TileDecoder.class.getName(); + + private static final int TAG_TILE_LAYERS = 3; + + private static final int TAG_LAYER_VERSION = 15; + private static final int TAG_LAYER_NAME = 1; + private static final int TAG_LAYER_FEATURES = 2; + private static final int TAG_LAYER_KEYS = 3; + private static final int TAG_LAYER_VALUES = 4; + private static final int TAG_LAYER_EXTENT = 5; + + private static final int TAG_FEATURE_ID = 1; + private static final int TAG_FEATURE_TAGS = 2; + private static final int TAG_FEATURE_TYPE = 3; + private static final int TAG_FEATURE_GEOMETRY = 4; + + private static final int TAG_VALUE_STRING = 1; + private static final int TAG_VALUE_FLOAT = 2; + private static final int TAG_VALUE_DOUBLE = 3; + private static final int TAG_VALUE_LONG = 4; + private static final int TAG_VALUE_UINT = 5; + private static final int TAG_VALUE_SINT = 6; + private static final int TAG_VALUE_BOOL = 7; + + private static final int TAG_GEOM_UNKNOWN = 0; + private static final int TAG_GEOM_POINT = 1; + private static final int TAG_GEOM_LINE = 2; + private static final int TAG_GEOM_POLYGON = 3; + + private short[] mTmpTags = new short[1024]; + + private Tile mTile; + private final String mLocale = "de"; + private IMapDataSink mMapDataCallback; + + private final static float REF_TILE_SIZE = 4096.0f; + private float mScale; + + boolean decode(InputStream is, Tile tile, IMapDataSink mapDataCallback) throws IOException { + setInputStream(is, Integer.MAX_VALUE); + mTile = tile; + mMapDataCallback = mapDataCallback; + mScale = REF_TILE_SIZE / Tile.SIZE; + + int val; + + while (hasData() && (val = decodeVarint32()) > 0) { + // read tag and wire type + int tag = (val >> 3); + + switch (tag) { + case TAG_TILE_LAYERS: + decodeLayer(); + break; + + default: + Log.d(TAG, mTile + " invalid type for tile: " + tag); + return false; + } + } + return true; + } + + private boolean decodeLayer() throws IOException { + + //int version = 0; + //int extent = 4096; + + int bytes = decodeVarint32(); + + ArrayList keys = new ArrayList(); + ArrayList values = new ArrayList(); + + String name = null; + int numFeatures = 0; + ArrayList features = new ArrayList(); + + int end = position() + bytes; + while (position() < end) { + // read tag and wire type + int val = decodeVarint32(); + if (val == 0) + break; + + int tag = (val >> 3); + + switch (tag) { + case TAG_LAYER_KEYS: + keys.add(decodeString()); + break; + + case TAG_LAYER_VALUES: + values.add(decodeValue()); + break; + + case TAG_LAYER_FEATURES: + numFeatures++; + decodeFeature(features); + break; + + case TAG_LAYER_VERSION: + //version = + decodeVarint32(); + break; + + case TAG_LAYER_NAME: + name = decodeString(); + break; + + case TAG_LAYER_EXTENT: + //extent = + decodeVarint32(); + break; + + default: + Log.d(TAG, mTile + " invalid type for layer: " + tag); + break; + } + + } + + Tag layerTag = new Tag(name, Tag.VALUE_YES); + + if (numFeatures == 0) + return true; + + int[] ignoreLocal = new int[20]; + int numIgnore = 0; + + int fallBackLocal = -1; + int matchedLocal = -1; + + for (int i = 0; i < keys.size(); i++) { + String key = keys.get(i); + if (!key.startsWith(Tag.TAG_KEY_NAME)) + continue; + int len = key.length(); + if (len == 4) { + fallBackLocal = i; + continue; + } + if (len < 7) { + ignoreLocal[numIgnore++] = i; + continue; + } + + if (mLocale.equals(key.substring(5))) { + //Log.d(TAG, "found local " + key); + matchedLocal = i; + } else + ignoreLocal[numIgnore++] = i; + + } + + for (Feature f : features) { + + if (f.elem.type == GeometryType.NONE) + continue; + + mTagSet.clear(); + mTagSet.add(layerTag); + + boolean hasName = false; + String fallbackName = null; + + tagLoop: for (int j = 0; j < (f.numTags << 1); j += 2) { + int keyIdx = f.tags[j]; + for (int i = 0; i < numIgnore; i++) + if (keyIdx == ignoreLocal[i]) + continue tagLoop; + + if (keyIdx == fallBackLocal) { + fallbackName = values.get(f.tags[j + 1]); + continue; + } + + String key; + String val = values.get(f.tags[j + 1]); + + if (keyIdx == matchedLocal) { + hasName = true; + mTagSet.add(new Tag(Tag.TAG_KEY_NAME, val, false)); + + } else { + key = keys.get(keyIdx); + mTagSet.add(new Tag(key, val)); + } + } + + if (!hasName && fallbackName != null) + mTagSet.add(new Tag(Tag.TAG_KEY_NAME, fallbackName, false)); + + // FIXME extract layer tag here + f.elem.set(mTagSet.asArray(), 5); + mMapDataCallback.process(f.elem); + mFeaturePool.release(f); + } + + return true; + } + + private final TagSet mTagSet = new TagSet(); + private final Pool mFeaturePool = new Pool() { + int count; + + @Override + protected Feature createItem() { + count++; + return new Feature(); + } + + @Override + protected boolean clearItem(Feature item) { + if (count > 100) { + count--; + return false; + } + + item.elem.tags = null; + item.elem.clear(); + item.tags = null; + item.type = 0; + item.numTags = 0; + + return true; + } + }; + + static class Feature extends Inlist { + short[] tags; + int numTags; + int type; + + final MapElement elem; + + Feature() { + elem = new MapElement(); + } + + boolean match(short otherTags[], int otherNumTags, int otherType) { + if (numTags != otherNumTags) + return false; + + if (type != otherType) + return false; + + for (int i = 0; i < numTags << 1; i++) { + if (tags[i] != otherTags[i]) + return false; + } + return true; + } + + } + + private void decodeFeature(ArrayList features) throws IOException { + int bytes = decodeVarint32(); + int end = position() + bytes; + + int type = 0; + //long id; + + lastX = 0; + lastY = 0; + + mTmpTags[0] = -1; + + Feature curFeature = null; + int numTags = 0; + + //Log.d(TAG, "start feature"); + while (position() < end) { + // read tag and wire type + int val = decodeVarint32(); + if (val == 0) + break; + + int tag = (val >>> 3); + + switch (tag) { + case TAG_FEATURE_ID: + //id = + decodeVarint32(); + break; + + case TAG_FEATURE_TAGS: + mTmpTags = decodeUnsignedVarintArray(mTmpTags); + + for (; numTags < mTmpTags.length && mTmpTags[numTags] >= 0;) + numTags += 2; + + numTags >>= 1; + + break; + + case TAG_FEATURE_TYPE: + type = decodeVarint32(); + + //Log.d(TAG, "got type " + type); + + break; + + case TAG_FEATURE_GEOMETRY: + + for (Feature f : features) { + if (f.match(mTmpTags, numTags, type)) { + curFeature = f; + break; + } + } + + if (curFeature == null) { + curFeature = mFeaturePool.get(); + curFeature.tags = new short[numTags << 1]; + System.arraycopy(mTmpTags, 0, curFeature.tags, 0, numTags << 1); + curFeature.numTags = numTags; + curFeature.type = type; + + features.add(curFeature); + } + + decodeCoordinates(type, curFeature); + break; + + default: + Log.d(TAG, mTile + " invalid type for feature: " + tag); + break; + } + } + } + + private final static int CLOSE_PATH = 0x07; + private final static int MOVE_TO = 0x01; + //private final static int LINE_TO = 0x02; + + private int lastX, lastY; + + private int decodeCoordinates(int type, Feature feature) throws IOException { + int bytes = decodeVarint32(); + readBuffer(bytes); + + if (feature == null) { + bufferPos += bytes; + return 0; + } + + MapElement elem = feature.elem; + boolean isPoint = false; + boolean isPoly = false; + boolean isLine = false; + + if (type == TAG_GEOM_LINE) { + elem.startLine(); + isLine = true; + } + else if (type == TAG_GEOM_POLYGON) { + elem.startPolygon(); + isPoly = true; + } else if (type == TAG_GEOM_POINT) { + isPoint = true; + elem.startPoints(); + } else if (type == TAG_GEOM_UNKNOWN) + elem.startPoints(); + + boolean even = true; + int val; + + int curX = 0; + int curY = 0; + int prevX = 0; + int prevY = 0; + + int cmd = 0; + int num = 0; + + boolean first = true; + boolean lastClip = false; + + // test bbox for outer.. + boolean isOuter = true; + boolean simplify = mTile.zoomLevel < 14; + int pixel = simplify ? 7 : 1; + + int xmin = Integer.MAX_VALUE, xmax = Integer.MIN_VALUE; + int ymin = Integer.MAX_VALUE, ymax = Integer.MIN_VALUE; + + for (int end = bufferPos + bytes; bufferPos < end;) { + val = decodeVarint32Filled(); + + if (num == 0) { + num = val >>> 3; + cmd = val & 0x07; + + if (isLine && lastClip) { + elem.addPoint(curX / mScale, curY / mScale); + lastClip = false; + } + + if (cmd == CLOSE_PATH) { + num = 0; + continue; + } + if (first) { + first = false; + continue; + } + if (cmd == MOVE_TO) { + if (type == TAG_GEOM_LINE) + elem.startLine(); + else if (type == TAG_GEOM_POLYGON) { + isOuter = false; + elem.startHole(); + } + } + continue; + } + // zigzag decoding + int s = ((val >>> 1) ^ -(val & 1)); + + if (even) { + // get x coordinate + even = false; + curX = lastX = lastX + s; + continue; + } + // get y coordinate and add point to geometry + num--; + + even = true; + curY = lastY = lastY + s; + + int dx = (curX - prevX); + int dy = (curY - prevY); + + if ((isPoint || cmd == MOVE_TO) + || (dx > pixel || dx < -pixel) + || (dy > pixel || dy < -pixel) + // dont clip at tile boundaries + || (curX <= 0 || curX >= 4095) + || (curY <= 0 || curY >= 4095)) { + + prevX = curX; + prevY = curY; + elem.addPoint(curX / mScale, curY / mScale); + lastClip = false; + + if (simplify && isOuter) { + if (curX < xmin) + xmin = curX; + if (curX > xmax) + xmax = curX; + + if (curY < ymin) + ymin = curY; + if (curY > ymax) + ymax = curY; + } + + continue; + } + lastClip = true; + } + + if (isPoly && isOuter && simplify && !testBBox(xmax - xmin, ymax - ymin)) { + //Log.d(TAG, "skip small poly "+ elem.indexPos + " > " + // + (xmax - xmin) * (ymax - ymin)); + elem.pointPos -= elem.index[elem.indexPos]; + if (elem.indexPos > 0) { + elem.indexPos -= 3; + elem.index[elem.indexPos + 1] = -1; + } else { + elem.type = GeometryType.NONE; + } + return 0; + } + + if (isLine && lastClip) + elem.addPoint(curX / mScale, curY / mScale); + + return 1; + } + + private static boolean testBBox(int dx, int dy) { + return dx * dy > 64 * 64; + } + + private String decodeValue() throws IOException { + int bytes = decodeVarint32(); + + String value = null; + + int end = position() + bytes; + + while (position() < end) { + // read tag and wire type + int val = decodeVarint32(); + if (val == 0) + break; + + int tag = (val >> 3); + + switch (tag) { + case TAG_VALUE_STRING: + value = decodeString(); + break; + + case TAG_VALUE_UINT: + value = String.valueOf(decodeVarint32()); + break; + + case TAG_VALUE_SINT: + value = String.valueOf(decodeVarint32()); + break; + + case TAG_VALUE_LONG: + value = String.valueOf(decodeVarint32()); + break; + + case TAG_VALUE_FLOAT: + value = String.valueOf(decodeFloat()); + break; + + case TAG_VALUE_DOUBLE: + value = String.valueOf(decodeDouble()); + break; + + case TAG_VALUE_BOOL: + value = decodeBool() ? "yes" : "no"; + break; + default: + break; + } + + } + return value; + } +} diff --git a/src/org/oscim/database/oscimap4/LwHttp.java b/src/org/oscim/database/oscimap4/LwHttp.java deleted file mode 100644 index 9731bfbc..00000000 --- a/src/org/oscim/database/oscimap4/LwHttp.java +++ /dev/null @@ -1,455 +0,0 @@ -/* - * Copyright 2013 Hannes Janetzek - * - * This program is free software: you can redistribute it and/or modify it under the - * terms of the GNU Lesser General Public License as published by the Free Software - * Foundation, either version 3 of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but WITHOUT ANY - * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A - * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License along with - * this program. If not, see . - */ -package org.oscim.database.oscimap4; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.MalformedURLException; -import java.net.Socket; -import java.net.SocketAddress; -import java.net.URL; - -import org.oscim.core.Tile; - -import android.os.SystemClock; -import android.util.Log; - -public class LwHttp { - private static final String TAG = LwHttp.class.getName(); - private final static int BUFFER_SIZE = 1 << 15; - byte[] buffer = new byte[BUFFER_SIZE]; - - // position in buffer - int bufferPos; - - // bytes available in buffer - int bufferFill; - - - // offset of buffer in message - private int mBufferOffset; - - // max bytes to read: message = header + content - private long mReadEnd; - - // overall bytes of message read - private int mReadPos; - - private String mHost; - private int mPort; - private InputStream mInputStream; - - private int mMaxReq = 0; - private Socket mSocket; - private OutputStream mCommandStream; - private InputStream mResponseStream; - long mLastRequest = 0; - private SocketAddress mSockAddr; - - private final static byte[] RESPONSE_HTTP_OK = "HTTP/1.1 200 OK".getBytes(); - private final static int RESPONSE_EXPECTED_LIVES = 100; - private final static int RESPONSE_EXPECTED_TIMEOUT = 10000; - private final static String TILE_EXT = ".vtm"; - - private byte[] REQUEST_GET_START; - private byte[] REQUEST_GET_END; - - private byte[] mRequestBuffer; - - boolean setServer(String urlString) { - URL url; - try { - url = new URL(urlString); - } catch (MalformedURLException e) { - - e.printStackTrace(); - return false; - //return new OpenResult("invalid url: " + options.get("url")); - } - - int port = url.getPort(); - if (port < 0) - port = 80; - - String host = url.getHost(); - String path = url.getPath(); - Log.d(TAG, "open oscim database: " + host + " " + port + " " + path); - - REQUEST_GET_START = ("GET " + path).getBytes(); - REQUEST_GET_END = (TILE_EXT + " HTTP/1.1\n" + - "Host: " + host + "\n" + - "Connection: Keep-Alive\n\n").getBytes(); - - mHost = host; - mPort = port; - - mRequestBuffer = new byte[1024]; - System.arraycopy(REQUEST_GET_START, 0, - mRequestBuffer, 0, REQUEST_GET_START.length); - return true; - } - - void close() { - if (mSocket != null) { - try { - mSocket.close(); - } catch (IOException e) { - e.printStackTrace(); - } finally { - mSocket = null; - } - } - } - - int readHeader() throws IOException { - InputStream is = mResponseStream; - mResponseStream.mark(BUFFER_SIZE); - - byte[] buf = buffer; - boolean first = true; - int read = 0; - int pos = 0; - int end = 0; - int len = 0; - - // header cannot be larger than BUFFER_SIZE for this to work - for (; pos < read || (len = is.read(buf, read, BUFFER_SIZE - read)) >= 0; len = 0) { - read += len; - while (end < read && (buf[end] != '\n')) - end++; - - if (buf[end] == '\n') { - if (first) { - // check only for OK - first = false; - if (!compareBytes(buf, pos, end, RESPONSE_HTTP_OK, 15)) - return -1; - - } else if (end - pos == 1) { - // check empty line (header end) - end += 1; - break; - } - - // String line = new String(buf, pos, end - pos - 1); - // Log.d(TAG, ">" + line + "< " + resp_len); - - pos += (end - pos) + 1; - end = pos; - } - } - - // check 4 bytes available.. - while ((read - end) < 4 && (len = is.read(buf, read, BUFFER_SIZE - read)) >= 0) - read += len; - - if (read - len < 4) - return -1; - - int contentLength = decodeInt(buf, end); - -// // back to start of content -// mResponseStream.reset(); -// mResponseStream.mark(0); -// mResponseStream.skip(end + 4); - - // start of content - bufferPos = end + 4; - // buffer fill - bufferFill = read; - mBufferOffset = 0; - - // overall bytes of already read - mReadPos = read; - mReadEnd = bufferPos + contentLength; - - mInputStream = mResponseStream; - - return 1; - } - - boolean sendRequest(Tile tile) throws IOException { - - bufferFill = 0; - bufferPos = 0; - mReadPos = 0; - mCacheFile = null; - - if (mSocket != null && ((mMaxReq-- <= 0) - || (SystemClock.elapsedRealtime() - mLastRequest - > RESPONSE_EXPECTED_TIMEOUT))) { - try { - mSocket.close(); - } catch (IOException e) { - - } - - // Log.d(TAG, "not alive - recreate connection " + mMaxReq); - mSocket = null; - } - - if (mSocket == null) { - lwHttpConnect(); - // we know our server - mMaxReq = RESPONSE_EXPECTED_LIVES; - // Log.d(TAG, "create connection"); - } else { - // should not be needed - int avail = mResponseStream.available(); - if (avail > 0) { - Log.d(TAG, "Consume left-over bytes: " + avail); - mResponseStream.read(buffer, 0, avail); - } - } - - byte[] request = mRequestBuffer; - int pos = REQUEST_GET_START.length; - - pos = writeInt(tile.zoomLevel, pos, request); - request[pos++] = '/'; - pos = writeInt(tile.tileX, pos, request); - request[pos++] = '/'; - pos = writeInt(tile.tileY, pos, request); - - int len = REQUEST_GET_END.length; - System.arraycopy(REQUEST_GET_END, 0, request, pos, len); - len += pos; - - // this does the same but with a few more allocations: - // byte[] request = String.format(REQUEST, - // Integer.valueOf(tile.zoomLevel), - // Integer.valueOf(tile.tileX), Integer.valueOf(tile.tileY)).getBytes(); - - try { - mCommandStream.write(request, 0, len); - mCommandStream.flush(); - return true; - } catch (IOException e) { - Log.d(TAG, "recreate connection"); - } - - lwHttpConnect(); - - mCommandStream.write(request, 0, len); - mCommandStream.flush(); - - return true; - } - - private boolean lwHttpConnect() throws IOException { - if (mSockAddr == null) - mSockAddr = new InetSocketAddress(mHost, mPort); - - mSocket = new Socket(); - mSocket.connect(mSockAddr, 30000); - mSocket.setTcpNoDelay(true); - - mCommandStream = mSocket.getOutputStream(); //new BufferedOutputStream(); - mResponseStream = mSocket.getInputStream(); - - return true; - } - - // write (positive) integer as char sequence to buffer - private static int writeInt(int val, int pos, byte[] buf) { - if (val == 0) { - buf[pos] = '0'; - return pos + 1; - } - - int i = 0; - for (int n = val; n > 0; n = n / 10, i++) - buf[pos + i] = (byte) ('0' + n % 10); - - // reverse bytes - for (int j = pos, end = pos + i - 1, mid = pos + i / 2; j < mid; j++, end--) { - byte tmp = buf[j]; - buf[j] = buf[end]; - buf[end] = tmp; - } - - return pos + i; - } - - private static boolean compareBytes(byte[] buffer, int position, int available, - byte[] string, int length) { - - if (available - position < length) - return false; - - for (int i = 0; i < length; i++) - if (buffer[position + i] != string[i]) - return false; - - return true; - } - - static int decodeInt(byte[] buffer, int offset) { - return buffer[offset] << 24 | (buffer[offset + 1] & 0xff) << 16 - | (buffer[offset + 2] & 0xff) << 8 - | (buffer[offset + 3] & 0xff); - } - - public boolean hasData() { - return mBufferOffset + bufferPos < mReadEnd; - } - - public int position() { - return mBufferOffset + bufferPos; - } - - public void readBuffer(int size) throws IOException { - // check if buffer already contains the request bytes - if (bufferPos + size < bufferFill) - return; - - // check if inputstream is read to the end - if (mReadPos == mReadEnd) - return; - - int maxSize = buffer.length; - - if (size > maxSize) { - Log.d(TAG, "increase read buffer to " + size + " bytes"); - maxSize = size; - byte[] tmp = new byte[maxSize]; - bufferFill -= bufferPos; - System.arraycopy(buffer, bufferPos, tmp, 0, bufferFill); - mBufferOffset += bufferPos; - bufferPos = 0; - buffer = tmp; - } - - if (bufferFill == bufferPos) { - mBufferOffset += bufferPos; - bufferPos = 0; - bufferFill = 0; - } else if (bufferPos + size > maxSize) { - // copy bytes left to the beginning of buffer - bufferFill -= bufferPos; - System.arraycopy(buffer, bufferPos, buffer, 0, bufferFill); - mBufferOffset += bufferPos; - bufferPos = 0; - } - - int max = maxSize - bufferFill; - - while ((bufferFill - bufferPos) < size && max > 0) { - - max = maxSize - bufferFill; - if (max > mReadEnd - mReadPos) - max = (int) (mReadEnd - mReadPos); - - // read until requested size is available in buffer - int len = mInputStream.read(buffer, bufferFill, max); - - if (len < 0) { - // finished reading, mark end - buffer[bufferFill] = 0; - break; - } - - mReadPos += len; - - // if (mCacheFile != null) - // mCacheFile.write(mReadBuffer, mBufferFill, len); - - if (mReadPos == mReadEnd) - break; - - bufferFill += len; - } - } - - private FileOutputStream mCacheFile; - - boolean cacheRead(Tile tile, File f) { - if (f.exists() && f.length() > 0) { - FileInputStream in; - - try { - in = new FileInputStream(f); - - mReadEnd = f.length(); - Log.d(TAG, tile + " - using cache: " + mReadEnd); - mInputStream = in; - - //decode(); - in.close(); - - return true; - - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (Exception ex) { - ex.printStackTrace(); - } - - f.delete(); - return false; - } - - return false; - } - - boolean cacheBegin(Tile tile, File f) { - if (MapDatabase.USE_CACHE) { - try { - Log.d(TAG, tile + " - writing cache"); - mCacheFile = new FileOutputStream(f); - - if (mReadPos > 0) { - try { - mCacheFile.write(buffer, bufferPos, - bufferFill - bufferPos); - - } catch (IOException e) { - e.printStackTrace(); - mCacheFile = null; - return false; - } - } - } catch (FileNotFoundException e) { - e.printStackTrace(); - mCacheFile = null; - return false; - } - } - return true; - } - - void cacheFinish(Tile tile, File file, boolean success) { - if (MapDatabase.USE_CACHE) { - if (success) { - try { - mCacheFile.flush(); - mCacheFile.close(); - Log.d(TAG, tile + " - cache written " + file.length()); - } catch (IOException e) { - e.printStackTrace(); - } - } else { - file.delete(); - } - } - mCacheFile = null; - } -} diff --git a/src/org/oscim/database/oscimap4/MapDatabase.java b/src/org/oscim/database/oscimap4/MapDatabase.java index 58c5948e..4f3b5cbf 100644 --- a/src/org/oscim/database/oscimap4/MapDatabase.java +++ b/src/org/oscim/database/oscimap4/MapDatabase.java @@ -14,29 +14,20 @@ */ package org.oscim.database.oscimap4; -import java.io.File; -import java.io.IOException; +import java.io.InputStream; import java.net.SocketException; import java.net.SocketTimeoutException; import java.net.UnknownHostException; -import java.util.Arrays; import org.oscim.core.BoundingBox; import org.oscim.core.GeoPoint; -import org.oscim.core.GeometryBuffer.GeometryType; -import org.oscim.core.MapElement; -import org.oscim.core.Tag; -import org.oscim.core.TagSet; -import org.oscim.core.Tile; +import org.oscim.database.IMapDataSink; import org.oscim.database.IMapDatabase; -import org.oscim.database.IMapDatabaseCallback; import org.oscim.database.MapInfo; import org.oscim.database.MapOptions; +import org.oscim.database.common.LwHttp; import org.oscim.layers.tile.MapTile; -import org.oscim.utils.UTF8Decoder; -import android.os.Environment; -import android.os.SystemClock; import android.util.Log; /** @@ -52,85 +43,49 @@ public class MapDatabase implements IMapDatabase { new MapInfo(new BoundingBox(-180, -90, 180, 90), new Byte((byte) 4), new GeoPoint(53.11, 8.85), null, 0, 0, 0, "de", "comment", "author", null); - - private static final String CACHE_DIRECTORY = "/Android/data/org.oscim.app/cache/"; - private static final String CACHE_FILE = "%d-%d-%d.tile"; - - private final static float REF_TILE_SIZE = 4096.0f; - // 'open' state private boolean mOpen = false; - private static File cacheDir; - - - private IMapDatabaseCallback mMapGenerator; - private float mScaleFactor; - private MapTile mTile; - - //private final boolean debug = false; private LwHttp conn; - - private final UTF8Decoder mStringDecoder; - private final MapElement mElem; - - public MapDatabase() { - mStringDecoder = new UTF8Decoder(); - mElem = new MapElement(); - } + private TileDecoder mTileDecoder; @Override - public QueryResult executeQuery(MapTile tile, IMapDatabaseCallback mapDatabaseCallback) { + public QueryResult executeQuery(MapTile tile, IMapDataSink sink) { QueryResult result = QueryResult.SUCCESS; - mTile = tile; - - mMapGenerator = mapDatabaseCallback; - - // scale coordinates to tile size - mScaleFactor = REF_TILE_SIZE / Tile.SIZE; - - File f = null; - - if (USE_CACHE) { - f = new File(cacheDir, String.format(CACHE_FILE, - Integer.valueOf(tile.zoomLevel), - Integer.valueOf(tile.tileX), - Integer.valueOf(tile.tileY))); - - if (conn.cacheRead(tile, f)) - return QueryResult.SUCCESS; - } - try { - - if (conn.sendRequest(tile) && conn.readHeader() >= 0) { - conn.cacheBegin(tile, f); - decode(); + InputStream is; + if (!conn.sendRequest(tile)) { + Log.d(TAG, tile + " Request Failed"); + result = QueryResult.FAILED; + } else if ((is = conn.readHeader()) != null) { + boolean win = mTileDecoder.decode(is, tile, sink); + if (!win) + Log.d(TAG, tile + " failed"); } else { Log.d(TAG, tile + " Network Error"); result = QueryResult.FAILED; } - } catch (SocketException ex) { - Log.d(TAG, tile + " Socket exception: " + ex.getMessage()); + } catch (SocketException e) { + Log.d(TAG, tile + " Socket exception: " + e.getMessage()); result = QueryResult.FAILED; - } catch (SocketTimeoutException ex) { - Log.d(TAG, tile + " Socket Timeout exception: " + ex.getMessage()); + } catch (SocketTimeoutException e) { + Log.d(TAG, tile + " Socket Timeout"); result = QueryResult.FAILED; - } catch (UnknownHostException ex) { - Log.d(TAG, tile + " no network"); + } catch (UnknownHostException e) { + Log.d(TAG, tile + " No Network"); result = QueryResult.FAILED; - } catch (Exception ex) { - ex.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); result = QueryResult.FAILED; } - conn.mLastRequest = SystemClock.elapsedRealtime(); + conn.requestCompleted(); if (result == QueryResult.SUCCESS) { - conn.cacheFinish(tile, f, true); + //conn.cacheFinish(tile, f, true); } else { - conn.cacheFinish(tile, f, false); + //conn.cacheFinish(tile, f, false); conn.close(); } return result; @@ -153,30 +108,22 @@ public class MapDatabase implements IMapDatabase { @Override public OpenResult open(MapOptions options) { + String extension = ".vtm"; + if (mOpen) return OpenResult.SUCCESS; if (options == null || !options.containsKey("url")) - return new OpenResult("options missing"); + return new OpenResult("No URL in MapOptions"); conn = new LwHttp(); - if (!conn.setServer(options.get("url"))) { + if (!conn.setServer(options.get("url"), extension, false)) { return new OpenResult("invalid url: " + options.get("url")); } - if (USE_CACHE) { - if (cacheDir == null) { - String externalStorageDirectory = Environment - .getExternalStorageDirectory() - .getAbsolutePath(); - String cacheDirectoryPath = externalStorageDirectory + CACHE_DIRECTORY; - cacheDir = createDirectory(cacheDirectoryPath); - } - } - mOpen = true; - initDecorder(); + mTileDecoder = new TileDecoder(); return OpenResult.SUCCESS; } @@ -184,541 +131,10 @@ public class MapDatabase implements IMapDatabase { @Override public void close() { mOpen = false; - conn.close(); - - if (USE_CACHE) { - cacheDir = null; - } } @Override public void cancel() { } - - private static File createDirectory(String pathName) { - File file = new File(pathName); - if (!file.exists() && !file.mkdirs()) { - throw new IllegalArgumentException("could not create directory: " + file); - } else if (!file.isDirectory()) { - throw new IllegalArgumentException("not a directory: " + file); - } else if (!file.canRead()) { - throw new IllegalArgumentException("cannot read directory: " + file); - } else if (!file.canWrite()) { - throw new IllegalArgumentException("cannot write directory: " + file); - } - return file; - } - - // /////////////// hand sewed tile protocol buffers decoder /////////////// - - private static final int TAG_TILE_VERSION = 1; - //private static final int TAG_TILE_TIMESTAMP = 2; - //private static final int TAG_TILE_ISWATER = 3; - - - private static final int TAG_TILE_NUM_TAGS = 11; - private static final int TAG_TILE_NUM_KEYS = 12; - private static final int TAG_TILE_NUM_VALUES = 13; - - private static final int TAG_TILE_TAG_KEYS = 14; - private static final int TAG_TILE_TAG_VALUES = 15; - private static final int TAG_TILE_TAGS = 16; - - private static final int TAG_TILE_LINE = 21; - private static final int TAG_TILE_POLY = 22; - private static final int TAG_TILE_POINT = 23; - - - private static final int TAG_ELEM_NUM_INDICES = 1; - private static final int TAG_ELEM_NUM_TAGS = 2; - //private static final int TAG_ELEM_HAS_ELEVATION = 3; - private static final int TAG_ELEM_TAGS = 11; - private static final int TAG_ELEM_INDEX = 12; - private static final int TAG_ELEM_COORDS = 13; - private static final int TAG_ELEM_LAYER = 21; - //private static final int TAG_ELEM_HEIGHT = 31; - //private static final int TAG_ELEM_MIN_HEIGHT = 32; - //private static final int TAG_ELEM_PRIORITY = 41; - - private short[] mTmpShortArray = new short[100]; - private Tag[][] mElementTags; - - private final TagSet curTags = new TagSet(100); - //private int mCurTagCnt; - - private void initDecorder() { - // reusable tag set - Tag[][] tags = new Tag[10][]; - for (int i = 0; i < 10; i++) - tags[i] = new Tag[i + 1]; - mElementTags = tags; - } - - private boolean decode() throws IOException { - - //mCurTagCnt = 0; - curTags.clear(true); - int version = -1; - - int val; - int numTags = 0; - int numKeys= -1; - int numValues = -1; - - int curKey = 0; - int curValue = 0; - - String [] keys = null; - String [] values = null; - - while (conn.hasData() && (val = decodeVarint32()) > 0) { - // read tag and wire type - int tag = (val >> 3); - - switch (tag) { - case TAG_TILE_LINE: - case TAG_TILE_POLY: - case TAG_TILE_POINT: - decodeTileElement(tag); - break; - - case TAG_TILE_TAG_KEYS: - if (keys == null || curKey >= numKeys){ - Log.d(TAG, mTile + " wrong number of keys " + numKeys); - return false; - } - keys[curKey++] = decodeString(); - break; - - case TAG_TILE_TAG_VALUES: - if (values == null || curValue >= numValues){ - Log.d(TAG, mTile + " wrong number of values " + numValues); - return false; - } - values[curValue++] = decodeString(); - break; - - case TAG_TILE_NUM_TAGS: - numTags = decodeVarint32(); - //Log.d(TAG, "num tags " + numTags); - break; - - case TAG_TILE_NUM_KEYS: - numKeys = decodeVarint32(); - //Log.d(TAG, "num keys " + numKeys); - keys = new String[numKeys]; - break; - - case TAG_TILE_NUM_VALUES: - numValues = decodeVarint32(); - //Log.d(TAG, "num values " + numValues); - values = new String[numValues]; - break; - - case TAG_TILE_TAGS: - mTmpShortArray = decodeShortArray(numTags, mTmpShortArray); - if (!decodeTileTags(numTags, mTmpShortArray, keys, values)){ - Log.d(TAG, mTile + " invalid tags"); - return false; - } - break; - - case TAG_TILE_VERSION: - version = decodeVarint32(); - if (version != 4){ - Log.d(TAG, mTile + " invalid version "+ version); - return false; - } - break; - - default: - Log.d(TAG, mTile + " invalid type for tile: " + tag); - return false; - } - } - return true; - } - - private boolean decodeTileTags(int numTags, short[] tagIdx, String[] keys, String[] vals) { - Tag tag; - - for (int i = 0; i < numTags*2; i += 2){ - int k = tagIdx[i]; - int v = tagIdx[i+1]; - String key, val; - - if (k < Tags.ATTRIB_OFFSET){ - if (k > Tags.MAX_KEY) - return false; - key = Tags.keys[k]; - } else { - k -= Tags.ATTRIB_OFFSET; - if (k >= keys.length) - return false; - key = keys[k]; - } - - if (v < Tags.ATTRIB_OFFSET){ - if (v > Tags.MAX_VALUE) - return false; - val = Tags.values[v]; - } else { - v -= Tags.ATTRIB_OFFSET; - if (v >= vals.length) - return false; - val = vals[v]; - } - - // FIXME filter out all variable tags - // might depend on theme though - if (key == Tag.TAG_KEY_NAME || key == Tag.KEY_HEIGHT || key == Tag.KEY_MIN_HEIGHT) - tag = new Tag(key, val, false); - else - tag = new Tag(key, val, true); - - curTags.add(tag); - } - - return true; - } - - private int decodeWayIndices(int indexCnt) throws IOException { - mElem.index = decodeShortArray(indexCnt, mElem.index); - - short[] index = mElem.index; - int coordCnt = 0; - - for (int i = 0; i < indexCnt; i++) - coordCnt += index[i] *= 2; - - // set end marker - if (indexCnt < index.length) - index[indexCnt] = -1; - - return coordCnt; - } - - private boolean decodeTileElement(int type) throws IOException { - - int bytes = decodeVarint32(); - Tag[] tags = null; - short[] index = null; - - int end = conn.position() + bytes; - int numIndices = 1; - int numTags = 1; - - boolean skip = false; - boolean fail = false; - - int coordCnt = 0; - if (type == TAG_TILE_POINT) { - coordCnt = 2; - mElem.index[0] = 2; - } - - mElem.layer = 5; - mElem.priority = 0; - mElem.height = 0; - mElem.minHeight = 0; - - while (conn.position() < end) { - // read tag and wire type - int val = decodeVarint32(); - if (val == 0) - break; - - int tag = (val >> 3); - - switch (tag) { - case TAG_ELEM_TAGS: - tags = decodeElementTags(numTags); - break; - - case TAG_ELEM_NUM_INDICES: - numIndices = decodeVarint32(); - break; - - case TAG_ELEM_NUM_TAGS: - numTags = decodeVarint32(); - break; - - case TAG_ELEM_INDEX: - coordCnt = decodeWayIndices(numIndices); - break; - - case TAG_ELEM_COORDS: - if (coordCnt == 0) { - Log.d(TAG, mTile + " no coordinates"); - skip = true; - } - int cnt = decodeWayCoordinates(skip, coordCnt); - - if (cnt != coordCnt) { - Log.d(TAG, mTile + " wrong number of coordintes"); - fail = true; - } - break; - - case TAG_ELEM_LAYER: - mElem.layer = decodeVarint32(); - break; - -// case TAG_ELEM_HEIGHT: -// mElem.height = decodeVarint32(); -// break; -// -// case TAG_ELEM_MIN_HEIGHT: -// mElem.minHeight = decodeVarint32(); -// break; -// -// case TAG_ELEM_PRIORITY: -// mElem.priority = decodeVarint32(); -// break; - - default: - Log.d(TAG, mTile + " invalid type for way: " + tag); - } - } - - if (fail || tags == null || numIndices == 0) { - Log.d(TAG, mTile + " failed reading way: bytes:" + bytes + " index:" - + (Arrays.toString(index)) + " tag:" - + (tags != null ? Arrays.deepToString(tags) : "null") + " " - + numIndices + " " + coordCnt); - return false; - } - - mElem.tags = tags; - switch (type) { - case TAG_TILE_LINE: - mElem.type = GeometryType.LINE; - break; - case TAG_TILE_POLY: - mElem.type = GeometryType.POLY; - break; - case TAG_TILE_POINT: - mElem.type = GeometryType.POINT; - break; - } - - mMapGenerator.renderElement(mElem); - - return true; - } - - private Tag[] decodeElementTags(int numTags) throws IOException { - short[] tagIds = mTmpShortArray = decodeShortArray(numTags, mTmpShortArray); - - Tag[] tags; - - if (numTags < 11) - tags = mElementTags[numTags - 1]; - else - tags = new Tag[numTags]; - - int max = curTags.numTags; - - for (int i = 0; i < numTags; i++){ - int idx = tagIds[i]; - - if (idx < 0 || idx > max) { - Log.d(TAG, mTile + " invalid tag:" + idx + " " + i); - return null; - } - - tags[i] = curTags.tags[idx]; - } - - return tags; - } - - private final static int VARINT_LIMIT = 5; - private final static int VARINT_MAX = 10; - - private int decodeWayCoordinates(boolean skip, int nodes) throws IOException { - int bytes = decodeVarint32(); - - conn.readBuffer(bytes); - - if (skip) { - conn.bufferPos += bytes; - return nodes; - } - - int cnt = 0; - - int lastX = 0; - int lastY = 0; - boolean even = true; - - float scale = mScaleFactor; - float[] coords = mElem.ensurePointSize(nodes, false); - - byte[] buf = conn.buffer; - int pos = conn.bufferPos; - int end = pos + bytes; - int val; - - while (pos < end) { - if (buf[pos] >= 0) { - val = buf[pos++]; - - } else if (buf[pos + 1] >= 0) { - val = (buf[pos++] & 0x7f) - | buf[pos++] << 7; - - } else if (buf[pos + 2] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++]) << 14; - - } else if (buf[pos + 3] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++]) << 21; - - } else { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++] & 0x7f) << 21 - | (buf[pos]) << 28; - - int max = pos + VARINT_LIMIT; - while (pos < max) - if (buf[pos++] >= 0) - break; - - if (pos == max) - throw new IOException("malformed VarInt32 in " + mTile); - } - - // zigzag decoding - int s = ((val >>> 1) ^ -(val & 1)); - - if (even) { - lastX = lastX + s; - coords[cnt++] = lastX / scale; - even = false; - } else { - lastY = lastY + s; - coords[cnt++] = lastY / scale; - even = true; - } - } - - conn.bufferPos = pos; - - return cnt; - } - - private short[] decodeShortArray(int num, short[] array) throws IOException { - int bytes = decodeVarint32(); - - if (array.length < num) - array = new short[num]; - - conn.readBuffer(bytes); - - int cnt = 0; - - byte[] buf = conn.buffer; - int pos = conn.bufferPos; - int end = pos + bytes; - int val; - - while (pos < end) { - if (buf[pos] >= 0) { - val = buf[pos++]; - } else if (buf[pos + 1] >= 0) { - val = (buf[pos++] & 0x7f) - | buf[pos++] << 7; - } else if (buf[pos + 2] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++]) << 14; - } else if (buf[pos + 3] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++]) << 21; - } else if (buf[pos + 4] >= 0){ - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++] & 0x7f) << 21 - | (buf[pos++]) << 28; - } else - throw new IOException("malformed VarInt32 in " + mTile); - - array[cnt++] = (short) val; - } - - conn.bufferPos = pos; - - return array; - } - - private int decodeVarint32() throws IOException { - if (conn.bufferPos + VARINT_MAX > conn.bufferFill) - conn.readBuffer(4096); - - byte[] buf = conn.buffer; - int pos = conn.bufferPos; - int val; - - if (buf[pos] >= 0) { - val = buf[pos++]; - } else { - - if (buf[pos + 1] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++]) << 7; - - } else if (buf[pos + 2] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++]) << 14; - - } else if (buf[pos + 3] >= 0) { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++]) << 21; - } else { - val = (buf[pos++] & 0x7f) - | (buf[pos++] & 0x7f) << 7 - | (buf[pos++] & 0x7f) << 14 - | (buf[pos++] & 0x7f) << 21 - | (buf[pos]) << 28; - - // 'Discard upper 32 bits' - int max = pos + VARINT_LIMIT; - while (pos < max) - if (buf[pos++] >= 0) - break; - - if (pos == max) - throw new IOException("malformed VarInt32 in " + mTile); - } - } - - conn.bufferPos = pos; - - return val; - } - - private String decodeString() throws IOException { - final int size = decodeVarint32(); - conn.readBuffer(size); - final String result = mStringDecoder.decode(conn.buffer, conn.bufferPos, size); - - conn.bufferPos += size; - - return result; - - } } diff --git a/src/org/oscim/database/oscimap4/TileDecoder.java b/src/org/oscim/database/oscimap4/TileDecoder.java new file mode 100644 index 00000000..b12e9f5b --- /dev/null +++ b/src/org/oscim/database/oscimap4/TileDecoder.java @@ -0,0 +1,388 @@ +/* + * Copyright 2013 + * + * This program is free software: you can redistribute it and/or modify it under the + * terms of the GNU Lesser General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * this program. If not, see . + */ +package org.oscim.database.oscimap4; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; + +import org.oscim.core.GeometryBuffer.GeometryType; +import org.oscim.core.MapElement; +import org.oscim.core.Tag; +import org.oscim.core.TagSet; +import org.oscim.core.Tile; +import org.oscim.database.IMapDataSink; +import org.oscim.database.common.ProtobufDecoder; + +import android.util.Log; + +public class TileDecoder extends ProtobufDecoder { + private final static String TAG = TileDecoder.class.getName(); + + private final MapElement mElem; + private Tile mTile; + + private static final int TAG_TILE_VERSION = 1; + //private static final int TAG_TILE_TIMESTAMP = 2; + //private static final int TAG_TILE_ISWATER = 3; + + private static final int TAG_TILE_NUM_TAGS = 11; + private static final int TAG_TILE_NUM_KEYS = 12; + private static final int TAG_TILE_NUM_VALUES = 13; + + private static final int TAG_TILE_TAG_KEYS = 14; + private static final int TAG_TILE_TAG_VALUES = 15; + private static final int TAG_TILE_TAGS = 16; + + private static final int TAG_TILE_LINE = 21; + private static final int TAG_TILE_POLY = 22; + private static final int TAG_TILE_POINT = 23; + + private static final int TAG_ELEM_NUM_INDICES = 1; + private static final int TAG_ELEM_NUM_TAGS = 2; + //private static final int TAG_ELEM_HAS_ELEVATION = 3; + private static final int TAG_ELEM_TAGS = 11; + private static final int TAG_ELEM_INDEX = 12; + private static final int TAG_ELEM_COORDS = 13; + private static final int TAG_ELEM_LAYER = 21; + //private static final int TAG_ELEM_HEIGHT = 31; + //private static final int TAG_ELEM_MIN_HEIGHT = 32; + //private static final int TAG_ELEM_PRIORITY = 41; + + private short[] mSArray = new short[100]; + private final Tag[][] mElementTags; + + private final TagSet curTags = new TagSet(100); + private IMapDataSink mMapDataSink; + // scale coordinates to tile size + private final static float REF_TILE_SIZE = 4096.0f; + private final float mScaleFactor = REF_TILE_SIZE / Tile.SIZE; + + TileDecoder() { + mElem = new MapElement(); + + // reusable tag set + Tag[][] tags = new Tag[10][]; + for (int i = 0; i < 10; i++) + tags[i] = new Tag[i + 1]; + mElementTags = tags; + + } + + boolean decode(InputStream is, Tile tile, IMapDataSink sink) + throws IOException { + + int byteCount = readUnsignedInt(is, buffer); + Log.d(TAG, tile + " contentLength:"+byteCount); + if (byteCount < 0) { + Log.d(TAG, "invalid contentLength: " + byteCount); + return false; + } + + setInputStream(is, byteCount); + + mTile = tile; + mMapDataSink = sink; + + curTags.clear(true); + int version = -1; + + int val; + int numTags = 0; + int numKeys = -1; + int numValues = -1; + + int curKey = 0; + int curValue = 0; + + String[] keys = null; + String[] values = null; + + while (hasData() && (val = decodeVarint32()) > 0) { + // read tag and wire type + int tag = (val >> 3); + //Log.d(TAG, "tag: " + tag); + + switch (tag) { + case TAG_TILE_LINE: + case TAG_TILE_POLY: + case TAG_TILE_POINT: + decodeTileElement(tag); + break; + + case TAG_TILE_TAG_KEYS: + if (keys == null || curKey >= numKeys) { + Log.d(TAG, mTile + " wrong number of keys " + numKeys); + return false; + } + keys[curKey++] = decodeString(); + break; + + case TAG_TILE_TAG_VALUES: + if (values == null || curValue >= numValues) { + Log.d(TAG, mTile + " wrong number of values " + numValues); + return false; + } + values[curValue++] = decodeString(); + break; + + case TAG_TILE_NUM_TAGS: + numTags = decodeVarint32(); + //Log.d(TAG, "num tags " + numTags); + break; + + case TAG_TILE_NUM_KEYS: + numKeys = decodeVarint32(); + //Log.d(TAG, "num keys " + numKeys); + keys = new String[numKeys]; + break; + + case TAG_TILE_NUM_VALUES: + numValues = decodeVarint32(); + //Log.d(TAG, "num values " + numValues); + values = new String[numValues]; + break; + + case TAG_TILE_TAGS: + int len = numTags * 2; + if (mSArray.length < len) + mSArray = new short[len]; + + decodeVarintArray(len, mSArray); + if (!decodeTileTags(numTags, mSArray, keys, values)) { + Log.d(TAG, mTile + " invalid tags"); + return false; + } + break; + + case TAG_TILE_VERSION: + version = decodeVarint32(); + if (version != 4) { + Log.d(TAG, mTile + " invalid version " + version); + return false; + } + break; + + default: + Log.d(TAG, mTile + " invalid type for tile: " + tag); + return false; + } + } + + return true; + } + + private boolean decodeTileTags(int numTags, short[] tagIdx, String[] keys, String[] vals) { + Tag tag; + + for (int i = 0, n = (numTags << 1); i < n; i += 2) { + int k = tagIdx[i]; + int v = tagIdx[i + 1]; + String key, val; + + if (k < Tags.ATTRIB_OFFSET) { + if (k > Tags.MAX_KEY) + return false; + key = Tags.keys[k]; + } else { + k -= Tags.ATTRIB_OFFSET; + if (k >= keys.length) + return false; + key = keys[k]; + } + + if (v < Tags.ATTRIB_OFFSET) { + if (v > Tags.MAX_VALUE) + return false; + val = Tags.values[v]; + } else { + v -= Tags.ATTRIB_OFFSET; + if (v >= vals.length) + return false; + val = vals[v]; + } + + // FIXME filter out all variable tags + // might depend on theme though + if (key == Tag.TAG_KEY_NAME || key == Tag.KEY_HEIGHT || key == Tag.KEY_MIN_HEIGHT) + tag = new Tag(key, val, false); + else + tag = new Tag(key, val, true); + + curTags.add(tag); + } + + return true; + } + + private int decodeWayIndices(int indexCnt) throws IOException { + mElem.ensureIndexSize(indexCnt, false); + //mElem.index = + decodeVarintArray(indexCnt, mElem.index); + + short[] index = mElem.index; + int coordCnt = 0; + + for (int i = 0; i < indexCnt; i++) + coordCnt += index[i] *= 2; + + // set end marker + if (indexCnt < index.length) + index[indexCnt] = -1; + + return coordCnt; + } + + private boolean decodeTileElement(int type) throws IOException { + + int bytes = decodeVarint32(); + Tag[] tags = null; + short[] index = null; + + int end = position() + bytes; + int numIndices = 1; + int numTags = 1; + + //boolean skip = false; + boolean fail = false; + + int coordCnt = 0; + if (type == TAG_TILE_POINT) { + coordCnt = 2; + mElem.index[0] = 2; + } + + mElem.layer = 5; + mElem.priority = 0; + mElem.height = 0; + mElem.minHeight = 0; + + while (position() < end) { + // read tag and wire type + int val = decodeVarint32(); + if (val == 0) + break; + + int tag = (val >> 3); + + switch (tag) { + case TAG_ELEM_TAGS: + tags = decodeElementTags(numTags); + break; + + case TAG_ELEM_NUM_INDICES: + numIndices = decodeVarint32(); + break; + + case TAG_ELEM_NUM_TAGS: + numTags = decodeVarint32(); + break; + + case TAG_ELEM_INDEX: + coordCnt = decodeWayIndices(numIndices); + break; + + case TAG_ELEM_COORDS: + if (coordCnt == 0) { + Log.d(TAG, mTile + " no coordinates"); + //skip = true; + } + + mElem.ensurePointSize(coordCnt, false); + int cnt = decodeInterleavedPoints(mElem.points, coordCnt, mScaleFactor); + + if (cnt != coordCnt) { + Log.d(TAG, mTile + " wrong number of coordintes"); + fail = true; + } + break; + + case TAG_ELEM_LAYER: + mElem.layer = decodeVarint32(); + break; + + // case TAG_ELEM_HEIGHT: + // mElem.height = decodeVarint32(); + // break; + // + // case TAG_ELEM_MIN_HEIGHT: + // mElem.minHeight = decodeVarint32(); + // break; + // + // case TAG_ELEM_PRIORITY: + // mElem.priority = decodeVarint32(); + // break; + + default: + Log.d(TAG, mTile + " invalid type for way: " + tag); + } + } + + if (fail || tags == null || numIndices == 0) { + Log.d(TAG, mTile + " failed reading way: bytes:" + bytes + " index:" + + (Arrays.toString(index)) + " tag:" + + (tags != null ? Arrays.deepToString(tags) : "null") + " " + + numIndices + " " + coordCnt); + return false; + } + + mElem.tags = tags; + switch (type) { + case TAG_TILE_LINE: + mElem.type = GeometryType.LINE; + break; + case TAG_TILE_POLY: + mElem.type = GeometryType.POLY; + break; + case TAG_TILE_POINT: + mElem.type = GeometryType.POINT; + break; + } + + mMapDataSink.process(mElem); + + return true; + } + + private Tag[] decodeElementTags(int numTags) throws IOException { + if (mSArray.length < numTags) + mSArray = new short[numTags]; + short[] tagIds = mSArray; + + decodeVarintArray(numTags, tagIds); + + Tag[] tags; + + if (numTags < 11) + tags = mElementTags[numTags - 1]; + else + tags = new Tag[numTags]; + + int max = curTags.numTags; + + for (int i = 0; i < numTags; i++) { + int idx = tagIds[i]; + + if (idx < 0 || idx > max) { + Log.d(TAG, mTile + " invalid tag:" + idx + " " + i); + return null; + } + + tags[i] = curTags.tags[idx]; + } + + return tags; + } +} diff --git a/src/org/oscim/database/pbmap/MapDatabase.java b/src/org/oscim/database/pbmap/MapDatabase.java index 497a0c88..10b83915 100644 --- a/src/org/oscim/database/pbmap/MapDatabase.java +++ b/src/org/oscim/database/pbmap/MapDatabase.java @@ -40,8 +40,8 @@ import org.oscim.core.GeometryBuffer.GeometryType; import org.oscim.core.MapElement; import org.oscim.core.Tag; import org.oscim.core.Tile; +import org.oscim.database.IMapDataSink; import org.oscim.database.IMapDatabase; -import org.oscim.database.IMapDatabaseCallback; import org.oscim.database.MapInfo; import org.oscim.database.MapOptions; import org.oscim.layers.tile.MapTile; @@ -51,7 +51,7 @@ import android.os.SystemClock; import android.util.Log; /** - * + * Deprecated * */ public class MapDatabase implements IMapDatabase { @@ -89,7 +89,7 @@ public class MapDatabase implements IMapDatabase { private Tag[] curTags = new Tag[MAX_TILE_TAGS]; private int mCurTagCnt; - private IMapDatabaseCallback mMapGenerator; + private IMapDataSink mMapGenerator; private float mScaleFactor; private MapTile mTile; private FileOutputStream mCacheFile; @@ -118,13 +118,13 @@ public class MapDatabase implements IMapDatabase { }); @Override - public QueryResult executeQuery(MapTile tile, IMapDatabaseCallback mapDatabaseCallback) { + public QueryResult executeQuery(MapTile tile, IMapDataSink mapDataSink) { QueryResult result = QueryResult.SUCCESS; mCacheFile = null; mTile = tile; - mMapGenerator = mapDatabaseCallback; + mMapGenerator = mapDataSink; mCurTagCnt = 0; // scale coordinates to tile size @@ -451,7 +451,7 @@ public class MapDatabase implements IMapDatabase { // layer = 5; mElement.type = polygon ? GeometryType.POLY : GeometryType.LINE; mElement.set(tags, layer); - mMapGenerator.renderElement(mElement); + mMapGenerator.process(mElement); return true; } @@ -531,7 +531,7 @@ public class MapDatabase implements IMapDatabase { mElement.index[0] = (short)numNodes; mElement.type = GeometryType.POINT; mElement.set(tags, layer); - mMapGenerator.renderElement(mElement); + mMapGenerator.process(mElement); return cnt; } diff --git a/src/org/oscim/database/test/MapDatabase.java b/src/org/oscim/database/test/MapDatabase.java index c538786b..778ed356 100644 --- a/src/org/oscim/database/test/MapDatabase.java +++ b/src/org/oscim/database/test/MapDatabase.java @@ -19,7 +19,7 @@ import org.oscim.core.MapElement; import org.oscim.core.Tag; import org.oscim.core.Tile; import org.oscim.database.IMapDatabase; -import org.oscim.database.IMapDatabaseCallback; +import org.oscim.database.IMapDataSink; import org.oscim.database.MapInfo; import org.oscim.database.MapOptions; import org.oscim.layers.tile.MapTile; @@ -65,7 +65,7 @@ public class MapDatabase implements IMapDatabase { @Override public QueryResult executeQuery(MapTile tile, - IMapDatabaseCallback mapDatabaseCallback) { + IMapDataSink mapDataSink) { int size = Tile.SIZE; MapElement e = mElem; @@ -96,7 +96,7 @@ public class MapDatabase implements IMapDatabase { e.addPoint(x1, y2); e.set(mTags, 0); - mapDatabaseCallback.renderElement(e); + mapDataSink.process(e); if (renderWays) { e.clear(); @@ -117,7 +117,7 @@ public class MapDatabase implements IMapDatabase { e.addPoint(size / 2, size / 2 + size); e.set(mTagsWay, 0); - mapDatabaseCallback.renderElement(e); + mapDataSink.process(e); e.clear(); // left-top to center @@ -134,7 +134,7 @@ public class MapDatabase implements IMapDatabase { e.addPoint(10, size); e.set(mTagsWay, 1); - mapDatabaseCallback.renderElement(e); + mapDataSink.process(e); } if (renderBoundary) { @@ -149,7 +149,7 @@ public class MapDatabase implements IMapDatabase { } e.set(mTagsBoundary, 1); - mapDatabaseCallback.renderElement(e); + mapDataSink.process(e); } if (renderPlace) { @@ -159,7 +159,7 @@ public class MapDatabase implements IMapDatabase { mTagsPlace[1] = new Tag("name", tile.toString()); e.set(mTagsPlace, 0); - mapDatabaseCallback.renderElement(e); + mapDataSink.process(e); } return QueryResult.SUCCESS; } diff --git a/src/org/oscim/layers/tile/vector/MapTileLoader.java b/src/org/oscim/layers/tile/vector/MapTileLoader.java index 28b9dae9..37fe9d64 100644 --- a/src/org/oscim/layers/tile/vector/MapTileLoader.java +++ b/src/org/oscim/layers/tile/vector/MapTileLoader.java @@ -23,9 +23,9 @@ import org.oscim.core.MapElement; import org.oscim.core.MercatorProjection; import org.oscim.core.Tag; import org.oscim.core.Tile; +import org.oscim.database.IMapDataSink; import org.oscim.database.IMapDatabase; import org.oscim.database.IMapDatabase.QueryResult; -import org.oscim.database.IMapDatabaseCallback; import org.oscim.layers.tile.MapTile; import org.oscim.layers.tile.TileLoader; import org.oscim.layers.tile.TileManager; @@ -53,13 +53,13 @@ import android.util.Log; * @note * 1. The MapWorkers call MapTileLoader.execute() to load a tile. * 2. The tile data will be loaded from current MapDatabase - * 3. MapDatabase calls the IMapDatabaseCallback functions + * 3. MapDatabase calls the IMapDataSink functions * implemented by MapTileLoader for WAY and POI items. * 4. these callbacks then call RenderTheme to get the matching style. * 5. RenderTheme calls IRenderCallback functions with style information * 6. Styled items become added to MapTile.layers... roughly */ -public class MapTileLoader extends TileLoader implements IRenderCallback, IMapDatabaseCallback { +public class MapTileLoader extends TileLoader implements IRenderCallback, IMapDataSink { private static final String TAG = MapTileLoader.class.getName(); @@ -282,9 +282,8 @@ public class MapTileLoader extends TileLoader implements IRenderCallback, IMapDa return true; } - // ---------------- MapDatabaseCallback ----------------- @Override - public void renderElement(MapElement element) { + public void process(MapElement element) { clearState(); mElement = element;