From 700f3e403e057a500f43fe033a1f7a31635f346b Mon Sep 17 00:00:00 2001 From: Hannes Janetzek Date: Tue, 21 May 2013 03:48:48 +0200 Subject: [PATCH] beginnings of oscimap protocol v4 implementation --- src/org/oscim/database/oscimap4/LwHttp.java | 455 +++++++++++ .../oscim/database/oscimap4/MapDatabase.java | 724 ++++++++++++++++++ .../database/oscimap4/ProtobufDecoder.java | 318 ++++++++ src/org/oscim/database/oscimap4/Tags.java | 354 +++++++++ .../oscim/database/oscimap4/TileData_v4.proto | 91 +++ 5 files changed, 1942 insertions(+) create mode 100644 src/org/oscim/database/oscimap4/LwHttp.java create mode 100644 src/org/oscim/database/oscimap4/MapDatabase.java create mode 100644 src/org/oscim/database/oscimap4/ProtobufDecoder.java create mode 100644 src/org/oscim/database/oscimap4/Tags.java create mode 100644 src/org/oscim/database/oscimap4/TileData_v4.proto diff --git a/src/org/oscim/database/oscimap4/LwHttp.java b/src/org/oscim/database/oscimap4/LwHttp.java new file mode 100644 index 00000000..9731bfbc --- /dev/null +++ b/src/org/oscim/database/oscimap4/LwHttp.java @@ -0,0 +1,455 @@ +/* + * Copyright 2013 Hannes Janetzek + * + * This program is free software: you can redistribute it and/or modify it under the + * terms of the GNU Lesser General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * this program. If not, see . + */ +package org.oscim.database.oscimap4; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.MalformedURLException; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.URL; + +import org.oscim.core.Tile; + +import android.os.SystemClock; +import android.util.Log; + +public class LwHttp { + private static final String TAG = LwHttp.class.getName(); + private final static int BUFFER_SIZE = 1 << 15; + byte[] buffer = new byte[BUFFER_SIZE]; + + // position in buffer + int bufferPos; + + // bytes available in buffer + int bufferFill; + + + // offset of buffer in message + private int mBufferOffset; + + // max bytes to read: message = header + content + private long mReadEnd; + + // overall bytes of message read + private int mReadPos; + + private String mHost; + private int mPort; + private InputStream mInputStream; + + private int mMaxReq = 0; + private Socket mSocket; + private OutputStream mCommandStream; + private InputStream mResponseStream; + long mLastRequest = 0; + private SocketAddress mSockAddr; + + private final static byte[] RESPONSE_HTTP_OK = "HTTP/1.1 200 OK".getBytes(); + private final static int RESPONSE_EXPECTED_LIVES = 100; + private final static int RESPONSE_EXPECTED_TIMEOUT = 10000; + private final static String TILE_EXT = ".vtm"; + + private byte[] REQUEST_GET_START; + private byte[] REQUEST_GET_END; + + private byte[] mRequestBuffer; + + boolean setServer(String urlString) { + URL url; + try { + url = new URL(urlString); + } catch (MalformedURLException e) { + + e.printStackTrace(); + return false; + //return new OpenResult("invalid url: " + options.get("url")); + } + + int port = url.getPort(); + if (port < 0) + port = 80; + + String host = url.getHost(); + String path = url.getPath(); + Log.d(TAG, "open oscim database: " + host + " " + port + " " + path); + + REQUEST_GET_START = ("GET " + path).getBytes(); + REQUEST_GET_END = (TILE_EXT + " HTTP/1.1\n" + + "Host: " + host + "\n" + + "Connection: Keep-Alive\n\n").getBytes(); + + mHost = host; + mPort = port; + + mRequestBuffer = new byte[1024]; + System.arraycopy(REQUEST_GET_START, 0, + mRequestBuffer, 0, REQUEST_GET_START.length); + return true; + } + + void close() { + if (mSocket != null) { + try { + mSocket.close(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + mSocket = null; + } + } + } + + int readHeader() throws IOException { + InputStream is = mResponseStream; + mResponseStream.mark(BUFFER_SIZE); + + byte[] buf = buffer; + boolean first = true; + int read = 0; + int pos = 0; + int end = 0; + int len = 0; + + // header cannot be larger than BUFFER_SIZE for this to work + for (; pos < read || (len = is.read(buf, read, BUFFER_SIZE - read)) >= 0; len = 0) { + read += len; + while (end < read && (buf[end] != '\n')) + end++; + + if (buf[end] == '\n') { + if (first) { + // check only for OK + first = false; + if (!compareBytes(buf, pos, end, RESPONSE_HTTP_OK, 15)) + return -1; + + } else if (end - pos == 1) { + // check empty line (header end) + end += 1; + break; + } + + // String line = new String(buf, pos, end - pos - 1); + // Log.d(TAG, ">" + line + "< " + resp_len); + + pos += (end - pos) + 1; + end = pos; + } + } + + // check 4 bytes available.. + while ((read - end) < 4 && (len = is.read(buf, read, BUFFER_SIZE - read)) >= 0) + read += len; + + if (read - len < 4) + return -1; + + int contentLength = decodeInt(buf, end); + +// // back to start of content +// mResponseStream.reset(); +// mResponseStream.mark(0); +// mResponseStream.skip(end + 4); + + // start of content + bufferPos = end + 4; + // buffer fill + bufferFill = read; + mBufferOffset = 0; + + // overall bytes of already read + mReadPos = read; + mReadEnd = bufferPos + contentLength; + + mInputStream = mResponseStream; + + return 1; + } + + boolean sendRequest(Tile tile) throws IOException { + + bufferFill = 0; + bufferPos = 0; + mReadPos = 0; + mCacheFile = null; + + if (mSocket != null && ((mMaxReq-- <= 0) + || (SystemClock.elapsedRealtime() - mLastRequest + > RESPONSE_EXPECTED_TIMEOUT))) { + try { + mSocket.close(); + } catch (IOException e) { + + } + + // Log.d(TAG, "not alive - recreate connection " + mMaxReq); + mSocket = null; + } + + if (mSocket == null) { + lwHttpConnect(); + // we know our server + mMaxReq = RESPONSE_EXPECTED_LIVES; + // Log.d(TAG, "create connection"); + } else { + // should not be needed + int avail = mResponseStream.available(); + if (avail > 0) { + Log.d(TAG, "Consume left-over bytes: " + avail); + mResponseStream.read(buffer, 0, avail); + } + } + + byte[] request = mRequestBuffer; + int pos = REQUEST_GET_START.length; + + pos = writeInt(tile.zoomLevel, pos, request); + request[pos++] = '/'; + pos = writeInt(tile.tileX, pos, request); + request[pos++] = '/'; + pos = writeInt(tile.tileY, pos, request); + + int len = REQUEST_GET_END.length; + System.arraycopy(REQUEST_GET_END, 0, request, pos, len); + len += pos; + + // this does the same but with a few more allocations: + // byte[] request = String.format(REQUEST, + // Integer.valueOf(tile.zoomLevel), + // Integer.valueOf(tile.tileX), Integer.valueOf(tile.tileY)).getBytes(); + + try { + mCommandStream.write(request, 0, len); + mCommandStream.flush(); + return true; + } catch (IOException e) { + Log.d(TAG, "recreate connection"); + } + + lwHttpConnect(); + + mCommandStream.write(request, 0, len); + mCommandStream.flush(); + + return true; + } + + private boolean lwHttpConnect() throws IOException { + if (mSockAddr == null) + mSockAddr = new InetSocketAddress(mHost, mPort); + + mSocket = new Socket(); + mSocket.connect(mSockAddr, 30000); + mSocket.setTcpNoDelay(true); + + mCommandStream = mSocket.getOutputStream(); //new BufferedOutputStream(); + mResponseStream = mSocket.getInputStream(); + + return true; + } + + // write (positive) integer as char sequence to buffer + private static int writeInt(int val, int pos, byte[] buf) { + if (val == 0) { + buf[pos] = '0'; + return pos + 1; + } + + int i = 0; + for (int n = val; n > 0; n = n / 10, i++) + buf[pos + i] = (byte) ('0' + n % 10); + + // reverse bytes + for (int j = pos, end = pos + i - 1, mid = pos + i / 2; j < mid; j++, end--) { + byte tmp = buf[j]; + buf[j] = buf[end]; + buf[end] = tmp; + } + + return pos + i; + } + + private static boolean compareBytes(byte[] buffer, int position, int available, + byte[] string, int length) { + + if (available - position < length) + return false; + + for (int i = 0; i < length; i++) + if (buffer[position + i] != string[i]) + return false; + + return true; + } + + static int decodeInt(byte[] buffer, int offset) { + return buffer[offset] << 24 | (buffer[offset + 1] & 0xff) << 16 + | (buffer[offset + 2] & 0xff) << 8 + | (buffer[offset + 3] & 0xff); + } + + public boolean hasData() { + return mBufferOffset + bufferPos < mReadEnd; + } + + public int position() { + return mBufferOffset + bufferPos; + } + + public void readBuffer(int size) throws IOException { + // check if buffer already contains the request bytes + if (bufferPos + size < bufferFill) + return; + + // check if inputstream is read to the end + if (mReadPos == mReadEnd) + return; + + int maxSize = buffer.length; + + if (size > maxSize) { + Log.d(TAG, "increase read buffer to " + size + " bytes"); + maxSize = size; + byte[] tmp = new byte[maxSize]; + bufferFill -= bufferPos; + System.arraycopy(buffer, bufferPos, tmp, 0, bufferFill); + mBufferOffset += bufferPos; + bufferPos = 0; + buffer = tmp; + } + + if (bufferFill == bufferPos) { + mBufferOffset += bufferPos; + bufferPos = 0; + bufferFill = 0; + } else if (bufferPos + size > maxSize) { + // copy bytes left to the beginning of buffer + bufferFill -= bufferPos; + System.arraycopy(buffer, bufferPos, buffer, 0, bufferFill); + mBufferOffset += bufferPos; + bufferPos = 0; + } + + int max = maxSize - bufferFill; + + while ((bufferFill - bufferPos) < size && max > 0) { + + max = maxSize - bufferFill; + if (max > mReadEnd - mReadPos) + max = (int) (mReadEnd - mReadPos); + + // read until requested size is available in buffer + int len = mInputStream.read(buffer, bufferFill, max); + + if (len < 0) { + // finished reading, mark end + buffer[bufferFill] = 0; + break; + } + + mReadPos += len; + + // if (mCacheFile != null) + // mCacheFile.write(mReadBuffer, mBufferFill, len); + + if (mReadPos == mReadEnd) + break; + + bufferFill += len; + } + } + + private FileOutputStream mCacheFile; + + boolean cacheRead(Tile tile, File f) { + if (f.exists() && f.length() > 0) { + FileInputStream in; + + try { + in = new FileInputStream(f); + + mReadEnd = f.length(); + Log.d(TAG, tile + " - using cache: " + mReadEnd); + mInputStream = in; + + //decode(); + in.close(); + + return true; + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (Exception ex) { + ex.printStackTrace(); + } + + f.delete(); + return false; + } + + return false; + } + + boolean cacheBegin(Tile tile, File f) { + if (MapDatabase.USE_CACHE) { + try { + Log.d(TAG, tile + " - writing cache"); + mCacheFile = new FileOutputStream(f); + + if (mReadPos > 0) { + try { + mCacheFile.write(buffer, bufferPos, + bufferFill - bufferPos); + + } catch (IOException e) { + e.printStackTrace(); + mCacheFile = null; + return false; + } + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + mCacheFile = null; + return false; + } + } + return true; + } + + void cacheFinish(Tile tile, File file, boolean success) { + if (MapDatabase.USE_CACHE) { + if (success) { + try { + mCacheFile.flush(); + mCacheFile.close(); + Log.d(TAG, tile + " - cache written " + file.length()); + } catch (IOException e) { + e.printStackTrace(); + } + } else { + file.delete(); + } + } + mCacheFile = null; + } +} diff --git a/src/org/oscim/database/oscimap4/MapDatabase.java b/src/org/oscim/database/oscimap4/MapDatabase.java new file mode 100644 index 00000000..58c5948e --- /dev/null +++ b/src/org/oscim/database/oscimap4/MapDatabase.java @@ -0,0 +1,724 @@ +/* + * Copyright 2012 Hannes Janetzek + * + * This program is free software: you can redistribute it and/or modify it under the + * terms of the GNU Lesser General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * this program. If not, see . + */ +package org.oscim.database.oscimap4; + +import java.io.File; +import java.io.IOException; +import java.net.SocketException; +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; +import java.util.Arrays; + +import org.oscim.core.BoundingBox; +import org.oscim.core.GeoPoint; +import org.oscim.core.GeometryBuffer.GeometryType; +import org.oscim.core.MapElement; +import org.oscim.core.Tag; +import org.oscim.core.TagSet; +import org.oscim.core.Tile; +import org.oscim.database.IMapDatabase; +import org.oscim.database.IMapDatabaseCallback; +import org.oscim.database.MapInfo; +import org.oscim.database.MapOptions; +import org.oscim.layers.tile.MapTile; +import org.oscim.utils.UTF8Decoder; + +import android.os.Environment; +import android.os.SystemClock; +import android.util.Log; + +/** + * + * + */ +public class MapDatabase implements IMapDatabase { + private static final String TAG = MapDatabase.class.getName(); + + static final boolean USE_CACHE = false; + + private static final MapInfo mMapInfo = + new MapInfo(new BoundingBox(-180, -90, 180, 90), + new Byte((byte) 4), new GeoPoint(53.11, 8.85), + null, 0, 0, 0, "de", "comment", "author", null); + + private static final String CACHE_DIRECTORY = "/Android/data/org.oscim.app/cache/"; + private static final String CACHE_FILE = "%d-%d-%d.tile"; + + private final static float REF_TILE_SIZE = 4096.0f; + + // 'open' state + private boolean mOpen = false; + private static File cacheDir; + + + private IMapDatabaseCallback mMapGenerator; + private float mScaleFactor; + private MapTile mTile; + + //private final boolean debug = false; + private LwHttp conn; + + private final UTF8Decoder mStringDecoder; + private final MapElement mElem; + + public MapDatabase() { + mStringDecoder = new UTF8Decoder(); + mElem = new MapElement(); + } + + @Override + public QueryResult executeQuery(MapTile tile, IMapDatabaseCallback mapDatabaseCallback) { + QueryResult result = QueryResult.SUCCESS; + + mTile = tile; + + mMapGenerator = mapDatabaseCallback; + + // scale coordinates to tile size + mScaleFactor = REF_TILE_SIZE / Tile.SIZE; + + File f = null; + + if (USE_CACHE) { + f = new File(cacheDir, String.format(CACHE_FILE, + Integer.valueOf(tile.zoomLevel), + Integer.valueOf(tile.tileX), + Integer.valueOf(tile.tileY))); + + if (conn.cacheRead(tile, f)) + return QueryResult.SUCCESS; + } + + try { + + if (conn.sendRequest(tile) && conn.readHeader() >= 0) { + conn.cacheBegin(tile, f); + decode(); + } else { + Log.d(TAG, tile + " Network Error"); + result = QueryResult.FAILED; + } + } catch (SocketException ex) { + Log.d(TAG, tile + " Socket exception: " + ex.getMessage()); + result = QueryResult.FAILED; + } catch (SocketTimeoutException ex) { + Log.d(TAG, tile + " Socket Timeout exception: " + ex.getMessage()); + result = QueryResult.FAILED; + } catch (UnknownHostException ex) { + Log.d(TAG, tile + " no network"); + result = QueryResult.FAILED; + } catch (Exception ex) { + ex.printStackTrace(); + result = QueryResult.FAILED; + } + + conn.mLastRequest = SystemClock.elapsedRealtime(); + + if (result == QueryResult.SUCCESS) { + + conn.cacheFinish(tile, f, true); + } else { + conn.cacheFinish(tile, f, false); + conn.close(); + } + return result; + } + + @Override + public String getMapProjection() { + return null; + } + + @Override + public MapInfo getMapInfo() { + return mMapInfo; + } + + @Override + public boolean isOpen() { + return mOpen; + } + + @Override + public OpenResult open(MapOptions options) { + if (mOpen) + return OpenResult.SUCCESS; + + if (options == null || !options.containsKey("url")) + return new OpenResult("options missing"); + + conn = new LwHttp(); + + if (!conn.setServer(options.get("url"))) { + return new OpenResult("invalid url: " + options.get("url")); + } + + if (USE_CACHE) { + if (cacheDir == null) { + String externalStorageDirectory = Environment + .getExternalStorageDirectory() + .getAbsolutePath(); + String cacheDirectoryPath = externalStorageDirectory + CACHE_DIRECTORY; + cacheDir = createDirectory(cacheDirectoryPath); + } + } + + mOpen = true; + initDecorder(); + + return OpenResult.SUCCESS; + } + + @Override + public void close() { + mOpen = false; + + conn.close(); + + if (USE_CACHE) { + cacheDir = null; + } + } + + @Override + public void cancel() { + } + + private static File createDirectory(String pathName) { + File file = new File(pathName); + if (!file.exists() && !file.mkdirs()) { + throw new IllegalArgumentException("could not create directory: " + file); + } else if (!file.isDirectory()) { + throw new IllegalArgumentException("not a directory: " + file); + } else if (!file.canRead()) { + throw new IllegalArgumentException("cannot read directory: " + file); + } else if (!file.canWrite()) { + throw new IllegalArgumentException("cannot write directory: " + file); + } + return file; + } + + // /////////////// hand sewed tile protocol buffers decoder /////////////// + + private static final int TAG_TILE_VERSION = 1; + //private static final int TAG_TILE_TIMESTAMP = 2; + //private static final int TAG_TILE_ISWATER = 3; + + + private static final int TAG_TILE_NUM_TAGS = 11; + private static final int TAG_TILE_NUM_KEYS = 12; + private static final int TAG_TILE_NUM_VALUES = 13; + + private static final int TAG_TILE_TAG_KEYS = 14; + private static final int TAG_TILE_TAG_VALUES = 15; + private static final int TAG_TILE_TAGS = 16; + + private static final int TAG_TILE_LINE = 21; + private static final int TAG_TILE_POLY = 22; + private static final int TAG_TILE_POINT = 23; + + + private static final int TAG_ELEM_NUM_INDICES = 1; + private static final int TAG_ELEM_NUM_TAGS = 2; + //private static final int TAG_ELEM_HAS_ELEVATION = 3; + private static final int TAG_ELEM_TAGS = 11; + private static final int TAG_ELEM_INDEX = 12; + private static final int TAG_ELEM_COORDS = 13; + private static final int TAG_ELEM_LAYER = 21; + //private static final int TAG_ELEM_HEIGHT = 31; + //private static final int TAG_ELEM_MIN_HEIGHT = 32; + //private static final int TAG_ELEM_PRIORITY = 41; + + private short[] mTmpShortArray = new short[100]; + private Tag[][] mElementTags; + + private final TagSet curTags = new TagSet(100); + //private int mCurTagCnt; + + private void initDecorder() { + // reusable tag set + Tag[][] tags = new Tag[10][]; + for (int i = 0; i < 10; i++) + tags[i] = new Tag[i + 1]; + mElementTags = tags; + } + + private boolean decode() throws IOException { + + //mCurTagCnt = 0; + curTags.clear(true); + int version = -1; + + int val; + int numTags = 0; + int numKeys= -1; + int numValues = -1; + + int curKey = 0; + int curValue = 0; + + String [] keys = null; + String [] values = null; + + while (conn.hasData() && (val = decodeVarint32()) > 0) { + // read tag and wire type + int tag = (val >> 3); + + switch (tag) { + case TAG_TILE_LINE: + case TAG_TILE_POLY: + case TAG_TILE_POINT: + decodeTileElement(tag); + break; + + case TAG_TILE_TAG_KEYS: + if (keys == null || curKey >= numKeys){ + Log.d(TAG, mTile + " wrong number of keys " + numKeys); + return false; + } + keys[curKey++] = decodeString(); + break; + + case TAG_TILE_TAG_VALUES: + if (values == null || curValue >= numValues){ + Log.d(TAG, mTile + " wrong number of values " + numValues); + return false; + } + values[curValue++] = decodeString(); + break; + + case TAG_TILE_NUM_TAGS: + numTags = decodeVarint32(); + //Log.d(TAG, "num tags " + numTags); + break; + + case TAG_TILE_NUM_KEYS: + numKeys = decodeVarint32(); + //Log.d(TAG, "num keys " + numKeys); + keys = new String[numKeys]; + break; + + case TAG_TILE_NUM_VALUES: + numValues = decodeVarint32(); + //Log.d(TAG, "num values " + numValues); + values = new String[numValues]; + break; + + case TAG_TILE_TAGS: + mTmpShortArray = decodeShortArray(numTags, mTmpShortArray); + if (!decodeTileTags(numTags, mTmpShortArray, keys, values)){ + Log.d(TAG, mTile + " invalid tags"); + return false; + } + break; + + case TAG_TILE_VERSION: + version = decodeVarint32(); + if (version != 4){ + Log.d(TAG, mTile + " invalid version "+ version); + return false; + } + break; + + default: + Log.d(TAG, mTile + " invalid type for tile: " + tag); + return false; + } + } + return true; + } + + private boolean decodeTileTags(int numTags, short[] tagIdx, String[] keys, String[] vals) { + Tag tag; + + for (int i = 0; i < numTags*2; i += 2){ + int k = tagIdx[i]; + int v = tagIdx[i+1]; + String key, val; + + if (k < Tags.ATTRIB_OFFSET){ + if (k > Tags.MAX_KEY) + return false; + key = Tags.keys[k]; + } else { + k -= Tags.ATTRIB_OFFSET; + if (k >= keys.length) + return false; + key = keys[k]; + } + + if (v < Tags.ATTRIB_OFFSET){ + if (v > Tags.MAX_VALUE) + return false; + val = Tags.values[v]; + } else { + v -= Tags.ATTRIB_OFFSET; + if (v >= vals.length) + return false; + val = vals[v]; + } + + // FIXME filter out all variable tags + // might depend on theme though + if (key == Tag.TAG_KEY_NAME || key == Tag.KEY_HEIGHT || key == Tag.KEY_MIN_HEIGHT) + tag = new Tag(key, val, false); + else + tag = new Tag(key, val, true); + + curTags.add(tag); + } + + return true; + } + + private int decodeWayIndices(int indexCnt) throws IOException { + mElem.index = decodeShortArray(indexCnt, mElem.index); + + short[] index = mElem.index; + int coordCnt = 0; + + for (int i = 0; i < indexCnt; i++) + coordCnt += index[i] *= 2; + + // set end marker + if (indexCnt < index.length) + index[indexCnt] = -1; + + return coordCnt; + } + + private boolean decodeTileElement(int type) throws IOException { + + int bytes = decodeVarint32(); + Tag[] tags = null; + short[] index = null; + + int end = conn.position() + bytes; + int numIndices = 1; + int numTags = 1; + + boolean skip = false; + boolean fail = false; + + int coordCnt = 0; + if (type == TAG_TILE_POINT) { + coordCnt = 2; + mElem.index[0] = 2; + } + + mElem.layer = 5; + mElem.priority = 0; + mElem.height = 0; + mElem.minHeight = 0; + + while (conn.position() < end) { + // read tag and wire type + int val = decodeVarint32(); + if (val == 0) + break; + + int tag = (val >> 3); + + switch (tag) { + case TAG_ELEM_TAGS: + tags = decodeElementTags(numTags); + break; + + case TAG_ELEM_NUM_INDICES: + numIndices = decodeVarint32(); + break; + + case TAG_ELEM_NUM_TAGS: + numTags = decodeVarint32(); + break; + + case TAG_ELEM_INDEX: + coordCnt = decodeWayIndices(numIndices); + break; + + case TAG_ELEM_COORDS: + if (coordCnt == 0) { + Log.d(TAG, mTile + " no coordinates"); + skip = true; + } + int cnt = decodeWayCoordinates(skip, coordCnt); + + if (cnt != coordCnt) { + Log.d(TAG, mTile + " wrong number of coordintes"); + fail = true; + } + break; + + case TAG_ELEM_LAYER: + mElem.layer = decodeVarint32(); + break; + +// case TAG_ELEM_HEIGHT: +// mElem.height = decodeVarint32(); +// break; +// +// case TAG_ELEM_MIN_HEIGHT: +// mElem.minHeight = decodeVarint32(); +// break; +// +// case TAG_ELEM_PRIORITY: +// mElem.priority = decodeVarint32(); +// break; + + default: + Log.d(TAG, mTile + " invalid type for way: " + tag); + } + } + + if (fail || tags == null || numIndices == 0) { + Log.d(TAG, mTile + " failed reading way: bytes:" + bytes + " index:" + + (Arrays.toString(index)) + " tag:" + + (tags != null ? Arrays.deepToString(tags) : "null") + " " + + numIndices + " " + coordCnt); + return false; + } + + mElem.tags = tags; + switch (type) { + case TAG_TILE_LINE: + mElem.type = GeometryType.LINE; + break; + case TAG_TILE_POLY: + mElem.type = GeometryType.POLY; + break; + case TAG_TILE_POINT: + mElem.type = GeometryType.POINT; + break; + } + + mMapGenerator.renderElement(mElem); + + return true; + } + + private Tag[] decodeElementTags(int numTags) throws IOException { + short[] tagIds = mTmpShortArray = decodeShortArray(numTags, mTmpShortArray); + + Tag[] tags; + + if (numTags < 11) + tags = mElementTags[numTags - 1]; + else + tags = new Tag[numTags]; + + int max = curTags.numTags; + + for (int i = 0; i < numTags; i++){ + int idx = tagIds[i]; + + if (idx < 0 || idx > max) { + Log.d(TAG, mTile + " invalid tag:" + idx + " " + i); + return null; + } + + tags[i] = curTags.tags[idx]; + } + + return tags; + } + + private final static int VARINT_LIMIT = 5; + private final static int VARINT_MAX = 10; + + private int decodeWayCoordinates(boolean skip, int nodes) throws IOException { + int bytes = decodeVarint32(); + + conn.readBuffer(bytes); + + if (skip) { + conn.bufferPos += bytes; + return nodes; + } + + int cnt = 0; + + int lastX = 0; + int lastY = 0; + boolean even = true; + + float scale = mScaleFactor; + float[] coords = mElem.ensurePointSize(nodes, false); + + byte[] buf = conn.buffer; + int pos = conn.bufferPos; + int end = pos + bytes; + int val; + + while (pos < end) { + if (buf[pos] >= 0) { + val = buf[pos++]; + + } else if (buf[pos + 1] >= 0) { + val = (buf[pos++] & 0x7f) + | buf[pos++] << 7; + + } else if (buf[pos + 2] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++]) << 14; + + } else if (buf[pos + 3] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++]) << 21; + + } else { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++] & 0x7f) << 21 + | (buf[pos]) << 28; + + int max = pos + VARINT_LIMIT; + while (pos < max) + if (buf[pos++] >= 0) + break; + + if (pos == max) + throw new IOException("malformed VarInt32 in " + mTile); + } + + // zigzag decoding + int s = ((val >>> 1) ^ -(val & 1)); + + if (even) { + lastX = lastX + s; + coords[cnt++] = lastX / scale; + even = false; + } else { + lastY = lastY + s; + coords[cnt++] = lastY / scale; + even = true; + } + } + + conn.bufferPos = pos; + + return cnt; + } + + private short[] decodeShortArray(int num, short[] array) throws IOException { + int bytes = decodeVarint32(); + + if (array.length < num) + array = new short[num]; + + conn.readBuffer(bytes); + + int cnt = 0; + + byte[] buf = conn.buffer; + int pos = conn.bufferPos; + int end = pos + bytes; + int val; + + while (pos < end) { + if (buf[pos] >= 0) { + val = buf[pos++]; + } else if (buf[pos + 1] >= 0) { + val = (buf[pos++] & 0x7f) + | buf[pos++] << 7; + } else if (buf[pos + 2] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++]) << 14; + } else if (buf[pos + 3] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++]) << 21; + } else if (buf[pos + 4] >= 0){ + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++] & 0x7f) << 21 + | (buf[pos++]) << 28; + } else + throw new IOException("malformed VarInt32 in " + mTile); + + array[cnt++] = (short) val; + } + + conn.bufferPos = pos; + + return array; + } + + private int decodeVarint32() throws IOException { + if (conn.bufferPos + VARINT_MAX > conn.bufferFill) + conn.readBuffer(4096); + + byte[] buf = conn.buffer; + int pos = conn.bufferPos; + int val; + + if (buf[pos] >= 0) { + val = buf[pos++]; + } else { + + if (buf[pos + 1] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++]) << 7; + + } else if (buf[pos + 2] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++]) << 14; + + } else if (buf[pos + 3] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++]) << 21; + } else { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++] & 0x7f) << 21 + | (buf[pos]) << 28; + + // 'Discard upper 32 bits' + int max = pos + VARINT_LIMIT; + while (pos < max) + if (buf[pos++] >= 0) + break; + + if (pos == max) + throw new IOException("malformed VarInt32 in " + mTile); + } + } + + conn.bufferPos = pos; + + return val; + } + + private String decodeString() throws IOException { + final int size = decodeVarint32(); + conn.readBuffer(size); + final String result = mStringDecoder.decode(conn.buffer, conn.bufferPos, size); + + conn.bufferPos += size; + + return result; + + } +} diff --git a/src/org/oscim/database/oscimap4/ProtobufDecoder.java b/src/org/oscim/database/oscimap4/ProtobufDecoder.java new file mode 100644 index 00000000..abe6a63b --- /dev/null +++ b/src/org/oscim/database/oscimap4/ProtobufDecoder.java @@ -0,0 +1,318 @@ +/* + * Copyright 2013 + * + * This program is free software: you can redistribute it and/or modify it under the + * terms of the GNU Lesser General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * this program. If not, see . + */ +package org.oscim.database.oscimap4; + +import java.io.IOException; +import java.io.InputStream; + +import org.oscim.utils.UTF8Decoder; + +import android.util.Log; + +public class ProtobufDecoder { + private final static String TAG = ProtobufDecoder.class.getName(); + + private final static int VARINT_LIMIT = 5; + private final static int VARINT_MAX = 10; + + private final static int BUFFER_SIZE = 1 << 15; // 32kb + byte[] buffer = new byte[BUFFER_SIZE]; + + // position in buffer + int bufferPos; + + // bytes available in buffer + int bufferFill; + + // offset of buffer in message + private int mBufferOffset; + + // max bytes to read: message = header + content + private long mReadEnd; + + // overall bytes of message read + private int mReadPos; + + private InputStream mInputStream; + + private final UTF8Decoder mStringDecoder; + + public ProtobufDecoder(){ + mStringDecoder = new UTF8Decoder(); + } + + public void setInputStream(InputStream is){ + mInputStream = is; + } + + public void skip()throws IOException{ + int bytes = decodeVarint32(); + bufferPos += bytes; + } + public int readInterleavedPoints(float[] coords, int numPoints, float scale) throws IOException { + int bytes = decodeVarint32(); + + readBuffer(bytes); + + int cnt = 0; + int lastX = 0; + int lastY = 0; + boolean even = true; + + //float[] coords = mElem.ensurePointSize(nodes, false); + + byte[] buf = buffer; + int pos = bufferPos; + int end = pos + bytes; + int val; + + while (pos < end) { + if (buf[pos] >= 0) { + val = buf[pos++]; + + } else if (buf[pos + 1] >= 0) { + val = (buf[pos++] & 0x7f) + | buf[pos++] << 7; + + } else if (buf[pos + 2] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++]) << 14; + + } else if (buf[pos + 3] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++]) << 21; + + } else { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++] & 0x7f) << 21 + | (buf[pos]) << 28; + + int max = pos + VARINT_LIMIT; + while (pos < max) + if (buf[pos++] >= 0) + break; + + if (pos == max) + throw new IOException("malformed VarInt32"); + } + + // zigzag decoding + int s = ((val >>> 1) ^ -(val & 1)); + + if (even) { + lastX = lastX + s; + coords[cnt++] = lastX / scale; + even = false; + } else { + lastY = lastY + s; + coords[cnt++] = lastY / scale; + even = true; + } + } + if (pos != bufferPos + bytes) + throw new IOException("invalid array " + numPoints); + + bufferPos = pos; + + return cnt; + } + + public void readVarintArray(int num, short[] array) throws IOException { + int bytes = decodeVarint32(); + + readBuffer(bytes); + + int cnt = 0; + + byte[] buf = buffer; + int pos = bufferPos; + int end = pos + bytes; + int val; + + while (pos < end) { + if (cnt == num) + throw new IOException("invalid array size " + num); + + if (buf[pos] >= 0) { + val = buf[pos++]; + } else if (buf[pos + 1] >= 0) { + val = (buf[pos++] & 0x7f) + | buf[pos++] << 7; + } else if (buf[pos + 2] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++]) << 14; + } else if (buf[pos + 3] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++]) << 21; + } else if (buf[pos + 4] >= 0){ + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++] & 0x7f) << 21 + | (buf[pos++]) << 28; + } else + throw new IOException("malformed VarInt32"); + + array[cnt++] = (short) val; + } + + if (pos != bufferPos + bytes) + throw new IOException("invalid array " + num); + + bufferPos = pos; + } + + private int decodeVarint32() throws IOException { + if (bufferPos + VARINT_MAX > bufferFill) + readBuffer(4096); + + byte[] buf = buffer; + int pos = bufferPos; + int val; + + if (buf[pos] >= 0) { + val = buf[pos++]; + } else { + + if (buf[pos + 1] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++]) << 7; + + } else if (buf[pos + 2] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++]) << 14; + + } else if (buf[pos + 3] >= 0) { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++]) << 21; + } else { + val = (buf[pos++] & 0x7f) + | (buf[pos++] & 0x7f) << 7 + | (buf[pos++] & 0x7f) << 14 + | (buf[pos++] & 0x7f) << 21 + | (buf[pos]) << 28; + + // 'Discard upper 32 bits' + int max = pos + VARINT_LIMIT; + while (pos < max) + if (buf[pos++] >= 0) + break; + + if (pos == max) + throw new IOException("malformed VarInt32"); + } + } + + bufferPos = pos; + + return val; + } + + public String decodeString() throws IOException { + final int size = decodeVarint32(); + readBuffer(size); + + String result; + + if (mStringDecoder == null) + result = new String(buffer,bufferPos, size, "UTF-8"); + else + result = mStringDecoder.decode(buffer, bufferPos, size); + + bufferPos += size; + + return result; + + } + public boolean hasData() { + return mBufferOffset + bufferPos < mReadEnd; + } + + public int position() { + return mBufferOffset + bufferPos; + } + + public void readBuffer(int size) throws IOException { + // check if buffer already contains the request bytes + if (bufferPos + size < bufferFill) + return; + + // check if inputstream is read to the end + if (mReadPos == mReadEnd) + return; + + int maxSize = buffer.length; + + if (size > maxSize) { + Log.d(TAG, "increase read buffer to " + size + " bytes"); + maxSize = size; + byte[] tmp = new byte[maxSize]; + bufferFill -= bufferPos; + System.arraycopy(buffer, bufferPos, tmp, 0, bufferFill); + mBufferOffset += bufferPos; + bufferPos = 0; + buffer = tmp; + } + + if (bufferFill == bufferPos) { + mBufferOffset += bufferPos; + bufferPos = 0; + bufferFill = 0; + } else if (bufferPos + size > maxSize) { + // copy bytes left to the beginning of buffer + bufferFill -= bufferPos; + System.arraycopy(buffer, bufferPos, buffer, 0, bufferFill); + mBufferOffset += bufferPos; + bufferPos = 0; + } + + int max = maxSize - bufferFill; + + while ((bufferFill - bufferPos) < size && max > 0) { + + max = maxSize - bufferFill; + if (max > mReadEnd - mReadPos) + max = (int) (mReadEnd - mReadPos); + + // read until requested size is available in buffer + int len = mInputStream.read(buffer, bufferFill, max); + + if (len < 0) { + // finished reading, mark end + buffer[bufferFill] = 0; + break; + } + + mReadPos += len; + + if (mReadPos == mReadEnd) + break; + + bufferFill += len; + } + } + +} diff --git a/src/org/oscim/database/oscimap4/Tags.java b/src/org/oscim/database/oscimap4/Tags.java new file mode 100644 index 00000000..e967c390 --- /dev/null +++ b/src/org/oscim/database/oscimap4/Tags.java @@ -0,0 +1,354 @@ +/* + * Copyright 2012 Hannes Janetzek + * + * This program is free software: you can redistribute it and/or modify it under the + * terms of the GNU Lesser General Public License as published by the Free Software + * Foundation, either version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A + * PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * this program. If not, see . + */ +package org.oscim.database.oscimap4; + +public class Tags { + // TODO this should be retrieved from tile 0/0/0 + + public final static int ATTRIB_OFFSET = 256; + + // the keys that were imported via osm2pgsql + some more + public final static String[] keys = { + "access", + "addr:housename", + "addr:housenumber", + "addr:interpolation", + "admin_level", + "aerialway", + "aeroway", + "amenity", + "area", + "barrier", + "bicycle", + "brand", + "bridge", + "boundary", + "building", + "construction", + "covered", + "culvert", + "cutting", + "denomination", + "disused", + "embankment", + "foot", + "generator:source", + "harbour", + "highway", + "historic", + "horse", + "intermittent", + "junction", + "landuse", + "layer", + "leisure", + "lock", + "man_made", + "military", + "motorcar", + "name", + "natural", + "oneway", + "operator", + "population", + "power", + "power_source", + "place", + "railway", + "ref", + "religion", + "route", + "service", + "shop", + "sport", + "surface", + "toll", + "tourism", + "tower:type", + "tracktype", + "tunnel", + "water", + "waterway", + "wetland", + "width", + "wood", + "height", + "min_height", + "scalerank" + }; + public final static int MAX_KEY = keys.length - 1; + + // most popular values for the selected key (created from taginfo db) + public final static String[] values = { + "yes", + "residential", + "service", + "unclassified", + "stream", + "track", + "water", + "footway", + "tertiary", + "private", + "tree", + "path", + "forest", + "secondary", + "house", + "no", + "asphalt", + "wood", + "grass", + "paved", + "primary", + "unpaved", + "bus_stop", + "parking", + "parking_aisle", + "rail", + "driveway", + "8", + "administrative", + "locality", + "turning_circle", + "crossing", + "village", + "fence", + "grade2", + "coastline", + "grade3", + "farmland", + "hamlet", + "hut", + "meadow", + "wetland", + "cycleway", + "river", + "school", + "trunk", + "gravel", + "place_of_worship", + "farm", + "grade1", + "traffic_signals", + "wall", + "garage", + "gate", + "motorway", + "living_street", + "pitch", + "grade4", + "industrial", + "road", + "ground", + "scrub", + "motorway_link", + "steps", + "ditch", + "swimming_pool", + "grade5", + "park", + "apartments", + "restaurant", + "designated", + "bench", + "survey_point", + "pedestrian", + "hedge", + "reservoir", + "riverbank", + "alley", + "farmyard", + "peak", + "level_crossing", + "roof", + "dirt", + "drain", + "garages", + "entrance", + "street_lamp", + "deciduous", + "fuel", + "trunk_link", + "information", + "playground", + "supermarket", + "primary_link", + "concrete", + "mixed", + "permissive", + "orchard", + "grave_yard", + "canal", + "garden", + "spur", + "paving_stones", + "rock", + "bollard", + "convenience", + "cemetery", + "post_box", + "commercial", + "pier", + "bank", + "hotel", + "cliff", + "retail", + "construction", + "-1", + "fast_food", + "coniferous", + "cafe", + "6", + "kindergarten", + "tower", + "hospital", + "yard", + "sand", + "public_building", + "cobblestone", + "destination", + "island", + "abandoned", + "vineyard", + "recycling", + "agricultural", + "isolated_dwelling", + "pharmacy", + "post_office", + "motorway_junction", + "pub", + "allotments", + "dam", + "secondary_link", + "lift_gate", + "siding", + "stop", + "main", + "farm_auxiliary", + "quarry", + "10", + "station", + "platform", + "taxiway", + "limited", + "sports_centre", + "cutline", + "detached", + "storage_tank", + "basin", + "bicycle_parking", + "telephone", + "terrace", + "town", + "suburb", + "bus", + "compacted", + "toilets", + "heath", + "works", + "tram", + "beach", + "culvert", + "fire_station", + "recreation_ground", + "bakery", + "police", + "atm", + "clothes", + "tertiary_link", + "waste_basket", + "attraction", + "viewpoint", + "bicycle", + "church", + "shelter", + "drinking_water", + "marsh", + "picnic_site", + "hairdresser", + "bridleway", + "retaining_wall", + "buffer_stop", + "nature_reserve", + "village_green", + "university", + "1", + "bar", + "townhall", + "mini_roundabout", + "camp_site", + "aerodrome", + "stile", + "9", + "car_repair", + "parking_space", + "library", + "pipeline", + "true", + "cycle_barrier", + "4", + "museum", + "spring", + "hunting_stand", + "disused", + "car", + "tram_stop", + "land", + "fountain", + "hiking", + "manufacture", + "vending_machine", + "kiosk", + "swamp", + "unknown", + "7", + "islet", + "shed", + "switch", + "rapids", + "office", + "bay", + "proposed", + "common", + "weir", + "grassland", + "customers", + "social_facility", + "hangar", + "doctors", + "stadium", + "give_way", + "greenhouse", + "guest_house", + "viaduct", + "doityourself", + "runway", + "bus_station", + "water_tower", + "golf_course", + "conservation", + "block", + "college", + "wastewater_plant", + "subway", + "halt", + "forestry", + "florist", + "butcher" + }; + public final static int MAX_VALUE = values.length - 1; + +} diff --git a/src/org/oscim/database/oscimap4/TileData_v4.proto b/src/org/oscim/database/oscimap4/TileData_v4.proto new file mode 100644 index 00000000..bc8d777a --- /dev/null +++ b/src/org/oscim/database/oscimap4/TileData_v4.proto @@ -0,0 +1,91 @@ +// Protocol Version 4 + +package org.oscim.database.oscimap4; + +message Data { + message Element { + + // number of geometry 'indices' + optional uint32 num_indices = 1 [default = 1]; + + // number of 'tags' + optional uint32 num_tags = 2 [default = 1]; + + // elevation per coordinate + // (pixel relative to ground meters) + // optional bool has_elevation = 3 [default = false]; + + // reference to tile.tags + repeated uint32 tags = 11 [packed = true]; + + // A list of number of coordinates for each geometry. + // - polygons are separated by one '0' index + // - for single points this can be omitted. + // e.g 2,2 for two lines with two points each, or + // 4,3,0,4,3 for two polygons with four points in + // the outer ring and 3 points in the inner. + + repeated uint32 indices = 12 [packed = true]; + + // single delta encoded coordinate x,y pairs scaled + // to a tile size of 4096 + // note: geometries start at x,y = tile size / 2 + + repeated sint32 coordinates = 13 [packed = true]; + + //---------------- optional items --------------- + // osm layer [-5 .. 5] -> [0 .. 10] + optional uint32 layer = 21 [default = 5]; + + // intended for symbol and label placement, not used + //optional uint32 rank = 32 [packed = true]; + + // elevation per coordinate + // (pixel relative to ground meters) + // repeated sint32 elevation = 33 [packed = true]; + + // building height, precision 1/10m + //repeated sint32 height = 34 [packed = true]; + + // building height, precision 1/10m + //repeated sint32 min_height = 35 [packed = true]; + } + + required uint32 version = 1; + + // tile creation time + optional uint64 timestamp = 2; + + // tile is completely water (not used yet) + optional bool water = 3; + + // number of 'tags' + required uint32 num_tags = 11; + optional uint32 num_keys = 12 [default = 0]; + optional uint32 num_vals = 13 [default = 0]; + + // strings referenced by tags + repeated string keys = 14; + // separate common attributes from label to + // allow + repeated string values = 15; + + // (key[0xfffffffc] | type[0x03]), value pairs + // key: uint32 -> reference to key-strings + // type 0: attribute -> uint32 reference to value-strings + // type 1: string -> uint32 reference to label-strings + // type 2: sint32 + // type 3: float + // value: uint32 interpreted according to 'type' + + repeated uint32 tags = 16 [packed = true]; + + // linestring + repeated Element lines = 21; + + // polygons (MUST be implicitly closed) + repeated Element polygons = 22; + + // points (POIs) + repeated Element points = 23; +}