xiaoyan159/vtm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

use LwHttp.Buffer to set limit on InputStream

Browse Source
This commit is contained in:
Hannes Janetzek 2014-01-20 22:43:57 +01:00
parent 9ed4bb6ec5
commit 402cbe7e57
10 changed files with 79 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
vtm-gdx-html/src/org/oscim/gdx/emu/org/oscim/tiling/source/common/LwHttp.java
View File

@ -116,9 +116,9 @@ public class LwHttp {
if (status == 200) { if (status == 200) {
Uint8Array buf = Uint8ArrayNative.create(xhr.getResponseArrayBuffer()); Uint8Array buf = Uint8ArrayNative.create(xhr.getResponseArrayBuffer());
mDataSource.process(new Buffer(buf), buf.byteLength()); mDataSource.process(new Buffer(buf));
} else { } else {
mDataSource.process(null, -1); mDataSource.process(null);
} }
} }
} }

6
vtm-gdx-html/src/org/oscim/gdx/emu/org/oscim/tiling/source/common/UrlTileDataSource.java
View File

@ -57,12 +57,12 @@ public abstract class UrlTileDataSource implements ITileDataSource {
return result; return result;
} }
public void process(InputStream is, int length) { public void process(InputStream is) {
boolean win = false; boolean win = false;
if (length >= 0) { if (is != null) {
try { try {
win = mTileDecoder.decode(mTile, mSink, is, length); win = mTileDecoder.decode(mTile, mSink, is);
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }

2
vtm/src/org/oscim/tiling/source/ITileDecoder.java
View File

@ -7,6 +7,6 @@ import org.oscim.core.Tile;
public interface ITileDecoder { public interface ITileDecoder {
boolean decode(Tile tile, ITileDataSink sink, InputStream is, int contentLength) boolean decode(Tile tile, ITileDataSink sink, InputStream is)
throws IOException; throws IOException;
} }

86
vtm/src/org/oscim/tiling/source/common/LwHttp.java
View File

@ -39,6 +39,7 @@ import org.slf4j.LoggerFactory;
*/ */
public class LwHttp { public class LwHttp {
static final Logger log = LoggerFactory.getLogger(LwHttp.class); static final Logger log = LoggerFactory.getLogger(LwHttp.class);
static final boolean DBG = false;
private final static byte[] HEADER_HTTP_OK = "200 OK".getBytes(); private final static byte[] HEADER_HTTP_OK = "200 OK".getBytes();
private final static byte[] HEADER_CONTENT_TYPE = "Content-Type".getBytes(); private final static byte[] HEADER_CONTENT_TYPE = "Content-Type".getBytes();
@ -46,7 +47,7 @@ public class LwHttp {
private final static int RESPONSE_EXPECTED_LIVES = 100; private final static int RESPONSE_EXPECTED_LIVES = 100;
private final static long RESPONSE_TIMEOUT = (long) 10E9; // 10 second in nanosecond private final static long RESPONSE_TIMEOUT = (long) 10E9; // 10 second in nanosecond
private final static int BUFFER_SIZE = 1024; private final static int BUFFER_SIZE = 4096;
private final byte[] buffer = new byte[BUFFER_SIZE]; private final byte[] buffer = new byte[BUFFER_SIZE];
private final String mHost; private final String mHost;
@ -55,7 +56,7 @@ public class LwHttp {
private int mMaxReq = 0; private int mMaxReq = 0;
private Socket mSocket; private Socket mSocket;
private OutputStream mCommandStream; private OutputStream mCommandStream;
private InputStream mResponseStream; private Buffer mResponseStream;
private OutputStream mCacheOutputStream; private OutputStream mCacheOutputStream;
private long mLastRequest = 0; private long mLastRequest = 0;
private SocketAddress mSockAddr; private SocketAddress mSockAddr;
@ -67,8 +68,6 @@ public class LwHttp {
private final boolean mInflateContent; private final boolean mInflateContent;
private final byte[] mContentType; private final byte[] mContentType;
private int mContentLength = -1;
/** /**
* @param url * @param url
* Base url for tiles * Base url for tiles
@ -106,18 +105,40 @@ public class LwHttp {
mRequestBuffer, 0, REQUEST_GET_START.length); mRequestBuffer, 0, REQUEST_GET_START.length);
} }
// TODO:
// to avoid a copy in PbfDecoder one could manage the buffer
// array directly and provide access to it.
static class Buffer extends BufferedInputStream { static class Buffer extends BufferedInputStream {
final OutputStream mCacheOutputstream; OutputStream mCacheOutputstream;
int sumRead = 0;
int mContentLength;
public Buffer(InputStream is, OutputStream cache) { public Buffer(InputStream is) {
super(is, 4096); super(is, BUFFER_SIZE);
}
public void setCache(OutputStream cache) {
mCacheOutputstream = cache; mCacheOutputstream = cache;
} }
public void start(int length) {
sumRead = 0;
mContentLength = length;
}
@Override @Override
public int read() throws IOException { public int read() throws IOException {
if (sumRead >= mContentLength)
return -1;
int data = super.read(); int data = super.read();
if (data >= 0)
sumRead += 1;
if (DBG)
log.debug("read {} {}", sumRead, mContentLength);
if (mCacheOutputstream != null)
mCacheOutputstream.write(data); mCacheOutputstream.write(data);
return data; return data;
@ -126,10 +147,18 @@ public class LwHttp {
@Override @Override
public int read(byte[] buffer, int offset, int byteCount) public int read(byte[] buffer, int offset, int byteCount)
throws IOException { throws IOException {
int len = super.read(buffer, offset, byteCount);
if (len >= 0) if (sumRead >= mContentLength)
mCacheOutputstream.write(buffer, offset, len); return -1;
int len = super.read(buffer, offset, byteCount);
sumRead += len;
if (DBG)
log.debug("read {} {} {}", len, sumRead, mContentLength);
if (mCacheOutputstream != null)
mCacheOutputstream.write(buffer, offset, byteCount);
return len; return len;
} }
@ -149,8 +178,9 @@ public class LwHttp {
public InputStream readHeader() throws IOException { public InputStream readHeader() throws IOException {
InputStream is = mResponseStream; Buffer is = mResponseStream;
is.mark(4096); is.mark(BUFFER_SIZE);
is.start(BUFFER_SIZE);
byte[] buf = buffer; byte[] buf = buffer;
boolean first = true; boolean first = true;
@ -161,7 +191,7 @@ public class LwHttp {
int end = 0; int end = 0;
int len = 0; int len = 0;
mContentLength = -1; int contentLength = -1;
// header may not be larger than BUFFER_SIZE for this to work // header may not be larger than BUFFER_SIZE for this to work
for (; (pos < read) || ((read < BUFFER_SIZE) && for (; (pos < read) || ((read < BUFFER_SIZE) &&
@ -202,11 +232,11 @@ public class LwHttp {
} else if (check(HEADER_CONTENT_LENGTH, buf, pos, end)) { } else if (check(HEADER_CONTENT_LENGTH, buf, pos, end)) {
// parse Content-Length // parse Content-Length
mContentLength = parseInt(buf, pos + contentLength = parseInt(buf, pos +
HEADER_CONTENT_LENGTH.length + 2, end - 1); HEADER_CONTENT_LENGTH.length + 2, end - 1);
} }
if (!ok) { if (!ok || DBG) {
String line = new String(buf, pos, end - pos - 1); String line = new String(buf, pos, end - pos - 1);
log.debug("> {} <", line); log.debug("> {} <", line);
} }
@ -223,9 +253,8 @@ public class LwHttp {
is.mark(0); is.mark(0);
is.skip(end); is.skip(end);
if (mCacheOutputStream != null) { is.setCache(mCacheOutputStream);
is = new Buffer(is, mCacheOutputStream); is.start(contentLength);
}
if (mInflateContent) if (mInflateContent)
return new InflaterInputStream(is); return new InflaterInputStream(is);
@ -238,14 +267,10 @@ public class LwHttp {
if (mSocket != null && ((mMaxReq-- <= 0) if (mSocket != null && ((mMaxReq-- <= 0)
|| (System.nanoTime() - mLastRequest > RESPONSE_TIMEOUT))) { || (System.nanoTime() - mLastRequest > RESPONSE_TIMEOUT))) {
try { close();
mSocket.close();
} catch (IOException e) {
log.debug(e.getMessage());
}
// log.debug("not alive - recreate connection " + mMaxReq); if (DBG)
mSocket = null; log.debug("not alive - recreate connection " + mMaxReq);
} }
if (mSocket == null) { if (mSocket == null) {
@ -284,6 +309,9 @@ public class LwHttp {
System.arraycopy(REQUEST_GET_END, 0, request, pos, len); System.arraycopy(REQUEST_GET_END, 0, request, pos, len);
len += pos; len += pos;
if (DBG)
log.debug("request: {}", new String(request, 0, len));
try { try {
mCommandStream.write(request, 0, len); mCommandStream.write(request, 0, len);
mCommandStream.flush(); mCommandStream.flush();
@ -309,7 +337,7 @@ public class LwHttp {
mSocket.setTcpNoDelay(true); mSocket.setTcpNoDelay(true);
mCommandStream = mSocket.getOutputStream(); mCommandStream = mSocket.getOutputStream();
mResponseStream = new BufferedInputStream(mSocket.getInputStream()); mResponseStream = new Buffer(mSocket.getInputStream());
return true; return true;
} }
@ -366,10 +394,6 @@ public class LwHttp {
close(); close();
} }
public int getContentLength() {
return mContentLength;
}
/** /**
* Write custom tile url * Write custom tile url
* *

24
vtm/src/org/oscim/tiling/source/common/PbfDecoder.java
View File

@ -72,9 +72,6 @@ public abstract class PbfDecoder implements ITileDecoder {
// offset of buffer in message // offset of buffer in message
private int mBufferOffset; private int mBufferOffset;
// max bytes to read: message = header + content
private int mMsgEnd;
// overall bytes of message read // overall bytes of message read
private int mMsgPos; private int mMsgPos;
@ -86,7 +83,7 @@ public abstract class PbfDecoder implements ITileDecoder {
mStringDecoder = new UTF8Decoder(); mStringDecoder = new UTF8Decoder();
} }
public void setInputStream(InputStream is, int contentLength) { public void setInputStream(InputStream is) {
mInputStream = is; mInputStream = is;
bufferFill = 0; bufferFill = 0;
@ -94,7 +91,6 @@ public abstract class PbfDecoder implements ITileDecoder {
mBufferOffset = 0; mBufferOffset = 0;
mMsgPos = 0; mMsgPos = 0;
mMsgEnd = contentLength;
} }
protected int decodeVarint32() throws IOException { protected int decodeVarint32() throws IOException {
@ -397,8 +393,8 @@ public abstract class PbfDecoder implements ITileDecoder {
} }
public boolean hasData() throws IOException { public boolean hasData() throws IOException {
if (mBufferOffset + bufferPos >= mMsgEnd) //if (mBufferOffset + bufferPos >= mMsgEnd)
return false; // return false;
return fillBuffer(1) > 0; return fillBuffer(1) > 0;
} }
@ -414,10 +410,6 @@ public abstract class PbfDecoder implements ITileDecoder {
if (bytesLeft >= size) if (bytesLeft >= size)
return bytesLeft; return bytesLeft;
// check if inputstream is read to the end
if (mMsgPos >= mMsgEnd)
return bytesLeft;
int maxSize = buffer.length; int maxSize = buffer.length;
if (size > maxSize) { if (size > maxSize) {
@ -454,10 +446,7 @@ public abstract class PbfDecoder implements ITileDecoder {
} }
while ((bufferFill - bufferPos) < size) { while ((bufferFill - bufferPos) < size) {
int max = maxSize - bufferFill; int max = maxSize - bufferFill;
if (max > mMsgEnd - mMsgPos)
max = mMsgEnd - mMsgPos;
if (max <= 0) { if (max <= 0) {
// should not be possible // should not be possible
@ -468,9 +457,8 @@ public abstract class PbfDecoder implements ITileDecoder {
int len = mInputStream.read(buffer, bufferFill, max); int len = mInputStream.read(buffer, bufferFill, max);
if (len < 0) { if (len < 0) {
mMsgEnd = mMsgPos;
if (debug) if (debug)
log.debug(" finished reading " + mMsgPos); log.debug("finished reading {}", mMsgPos);
// finished reading, mark end // finished reading, mark end
buffer[bufferFill] = 0; buffer[bufferFill] = 0;
@ -479,10 +467,6 @@ public abstract class PbfDecoder implements ITileDecoder {
mMsgPos += len; mMsgPos += len;
bufferFill += len; bufferFill += len;
if (mMsgPos == mMsgEnd)
break;
} }
return bufferFill - bufferPos; return bufferFill - bufferPos;
} }

7
vtm/src/org/oscim/tiling/source/common/UrlTileDataSource.java
View File

@ -55,7 +55,7 @@ public abstract class UrlTileDataSource implements ITileDataSource {
} else { } else {
InputStream is = c.getInputStream(); InputStream is = c.getInputStream();
try { try {
if (mTileDecoder.decode(tile, sink, is, c.getBytes())) { if (mTileDecoder.decode(tile, sink, is)) {
return QueryResult.SUCCESS; return QueryResult.SUCCESS;
} }
} catch (IOException e) { } catch (IOException e) {
@ -77,15 +77,14 @@ public abstract class UrlTileDataSource implements ITileDataSource {
} else if ((is = mConn.readHeader()) == null) { } else if ((is = mConn.readHeader()) == null) {
log.debug("{} Network Error", tile); log.debug("{} Network Error", tile);
} else { } else {
int bytes = mConn.getContentLength(); success = mTileDecoder.decode(tile, sink, is);
success = mTileDecoder.decode(tile, sink, is, bytes);
} }
} catch (SocketException e) { } catch (SocketException e) {
log.debug("{} Socket exception: {}", tile, e.getMessage()); log.debug("{} Socket exception: {}", tile, e.getMessage());
} catch (SocketTimeoutException e) { } catch (SocketTimeoutException e) {
log.debug("{} Socket Timeout", tile); log.debug("{} Socket Timeout", tile);
} catch (UnknownHostException e) { } catch (UnknownHostException e) {
log.debug("{} No Network", tile); log.debug("{} Unknown host: {}", tile, e.getMessage());
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} finally { } finally {

6
vtm/src/org/oscim/tiling/source/mapnik/TileDecoder.java
View File

@ -71,13 +71,13 @@ public class TileDecoder extends PbfDecoder {
private float mScale; private float mScale;
@Override @Override
public boolean decode(Tile tile, ITileDataSink mapDataCallback, InputStream is, public boolean decode(Tile tile, ITileDataSink mapDataCallback, InputStream is)
int contentLength)
throws IOException { throws IOException {
if (debug) if (debug)
log.debug(tile + " decode"); log.debug(tile + " decode");
setInputStream(is, Integer.MAX_VALUE); setInputStream(is);
mTile = tile; mTile = tile;
mMapDataCallback = mapDataCallback; mMapDataCallback = mapDataCallback;
mScale = REF_TILE_SIZE / Tile.SIZE; mScale = REF_TILE_SIZE / Tile.SIZE;

4
vtm/src/org/oscim/tiling/source/oscimap/TileDecoder.java
View File

@ -68,10 +68,10 @@ public class TileDecoder extends PbfDecoder {
} }
@Override @Override
public boolean decode(Tile tile, ITileDataSink sink, InputStream is, int contentLength) public boolean decode(Tile tile, ITileDataSink sink, InputStream is)
throws IOException { throws IOException {
setInputStream(is, contentLength); setInputStream(is);
mTile = tile; mTile = tile;
mSink = sink; mSink = sink;

11
vtm/src/org/oscim/tiling/source/oscimap2/OSciMap2TileSource.java
View File

@ -92,17 +92,12 @@ public class OSciMap2TileSource extends UrlTileSource {
} }
@Override @Override
public boolean decode(Tile tile, ITileDataSink sink, InputStream is, int contentLength) public boolean decode(Tile tile, ITileDataSink sink, InputStream is)
throws IOException { throws IOException {
int byteCount = readUnsignedInt(is, buffer); readUnsignedInt(is, buffer);
//log.debug(tile + " contentLength:" + byteCount);
if (byteCount < 0) {
log.debug(tile + " invalid content length: " + byteCount);
return false;
}
setInputStream(is, byteCount); setInputStream(is);
mTile = tile; mTile = tile;
mMapDataSink = sink; mMapDataSink = sink;

10
vtm/src/org/oscim/tiling/source/oscimap4/TileDecoder.java
View File

@ -76,17 +76,13 @@ public class TileDecoder extends PbfDecoder {
} }
@Override @Override
public boolean decode(Tile tile, ITileDataSink sink, InputStream is, int contentLength) public boolean decode(Tile tile, ITileDataSink sink, InputStream is)
throws IOException { throws IOException {
int byteCount = readUnsignedInt(is, buffer); readUnsignedInt(is, buffer);
//log.debug(tile + " contentLength:" + byteCount); //log.debug(tile + " contentLength:" + byteCount);
if (byteCount < 0) {
log.debug("invalid contentLength: " + byteCount);
return false;
}
setInputStream(is, byteCount); setInputStream(is);
mTile = tile; mTile = tile;
mMapDataSink = sink; mMapDataSink = sink;