xiaoyan159/vtm
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Mapsforge: map stream support

Browse Source
This commit is contained in:
Emux 2020-09-18 14:26:52 +03:00
parent 5c3a728d05
commit 6116ebb348
No known key found for this signature in database
GPG Key ID: 64ED9980896038C3
5 changed files with 189 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/Changelog.md
View File

@ -2,6 +2,7 @@
## New since 0.14.0 ## New since 0.14.0
- Mapsforge: map stream support [#784](https://github.com/mapsforge/vtm/pull/784)
- Render theme from Android content providers [#783](https://github.com/mapsforge/vtm/pull/783) - Render theme from Android content providers [#783](https://github.com/mapsforge/vtm/pull/783)
- Many other minor improvements and bug fixes - Many other minor improvements and bug fixes
- [Solved issues](https://github.com/mapsforge/vtm/issues?q=is%3Aclosed+milestone%3A0.15.0) - [Solved issues](https://github.com/mapsforge/vtm/issues?q=is%3Aclosed+milestone%3A0.15.0)

25
vtm/src/org/oscim/tiling/source/mapfile/IndexCache.java
View File

@ -1,5 +1,6 @@
/* /*
* Copyright 2010, 2011, 2012 mapsforge.org * Copyright 2010, 2011, 2012 mapsforge.org
* Copyright 2017-2020 devemux86
* *
* This file is part of the OpenScienceMap project (http://www.opensciencemap.org). * This file is part of the OpenScienceMap project (http://www.opensciencemap.org).
* *
@ -20,7 +21,8 @@ import org.oscim.tiling.source.mapfile.header.SubFileParameter;
import org.oscim.utils.LRUCache; import org.oscim.utils.LRUCache;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.Collections; import java.util.Collections;
import java.util.Map; import java.util.Map;
import java.util.logging.Level; import java.util.logging.Level;
@ -44,15 +46,15 @@ class IndexCache {
* SubFileParameter.BYTES_PER_INDEX_ENTRY; * SubFileParameter.BYTES_PER_INDEX_ENTRY;
private final Map<IndexCacheEntryKey, byte[]> map; private final Map<IndexCacheEntryKey, byte[]> map;
private final RandomAccessFile randomAccessFile; private final FileChannel fileChannel;
/** /**
* @param randomAccessFile the map file from which the index should be read and cached. * @param inputChannel the map file from which the index should be read and cached.
* @param capacity the maximum number of entries in the cache. * @param capacity the maximum number of entries in the cache.
* @throws IllegalArgumentException if the capacity is negative. * @throws IllegalArgumentException if the capacity is negative.
*/ */
IndexCache(RandomAccessFile randomAccessFile, int capacity) { IndexCache(FileChannel inputChannel, int capacity) {
this.randomAccessFile = randomAccessFile; this.fileChannel = inputChannel;
this.map = Collections.synchronizedMap(new LRUCache<IndexCacheEntryKey, byte[]>(capacity)); this.map = Collections.synchronizedMap(new LRUCache<IndexCacheEntryKey, byte[]>(capacity));
} }
@ -97,11 +99,14 @@ class IndexCache {
int remainingIndexSize = (int) (subFileParameter.indexEndAddress - indexBlockPosition); int remainingIndexSize = (int) (subFileParameter.indexEndAddress - indexBlockPosition);
int indexBlockSize = Math.min(SIZE_OF_INDEX_BLOCK, remainingIndexSize); int indexBlockSize = Math.min(SIZE_OF_INDEX_BLOCK, remainingIndexSize);
indexBlock = new byte[indexBlockSize]; indexBlock = new byte[indexBlockSize];
ByteBuffer indexBlockWrapper = ByteBuffer.wrap(indexBlock, 0, indexBlockSize);
this.randomAccessFile.seek(indexBlockPosition); synchronized (this.fileChannel) {
if (this.randomAccessFile.read(indexBlock, 0, indexBlockSize) != indexBlockSize) { this.fileChannel.position(indexBlockPosition);
LOG.warning("reading the current index block has failed"); if (this.fileChannel.read(indexBlockWrapper) != indexBlockSize) {
return -1; LOG.warning("reading the current index block has failed");
return -1;
}
} }
// put the index block in the map // put the index block in the map

165
vtm/src/org/oscim/tiling/source/mapfile/MapDatabase.java
View File

@ -2,7 +2,7 @@
* Copyright 2010, 2011, 2012 mapsforge.org * Copyright 2010, 2011, 2012 mapsforge.org
* Copyright 2013, 2014 Hannes Janetzek * Copyright 2013, 2014 Hannes Janetzek
* Copyright 2014-2015 Ludwig M Brinckmann * Copyright 2014-2015 Ludwig M Brinckmann
* Copyright 2016-2019 devemux86 * Copyright 2016-2020 devemux86
* Copyright 2016 Andrey Novikov * Copyright 2016 Andrey Novikov
* Copyright 2017-2018 Gustl22 * Copyright 2017-2018 Gustl22
* Copyright 2018 Bezzu * Copyright 2018 Bezzu
@ -38,8 +38,9 @@ import org.oscim.utils.geom.TileSeparator;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.nio.channels.FileChannel;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -202,8 +203,7 @@ public class MapDatabase implements ITileDataSource {
private long mFileSize; private long mFileSize;
private boolean mDebugFile; private boolean mDebugFile;
private RandomAccessFile mInputFile; private FileChannel mInputChannel;
private ReadBuffer mReadBuffer;
private String mSignatureBlock; private String mSignatureBlock;
private String mSignaturePoi; private String mSignaturePoi;
private String mSignatureWay; private String mSignatureWay;
@ -227,11 +227,15 @@ public class MapDatabase implements ITileDataSource {
public MapDatabase(MapFileTileSource tileSource) throws IOException { public MapDatabase(MapFileTileSource tileSource) throws IOException {
mTileSource = tileSource; mTileSource = tileSource;
try { try {
/* open the file in read only mode */ // false positive: stream gets closed when the channel is closed
mInputFile = new RandomAccessFile(tileSource.mapFile, "r"); // see e.g. http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4796385
mFileSize = mInputFile.length(); if (tileSource.mapFileInputStream != null)
mReadBuffer = new ReadBuffer(mInputFile); mInputChannel = tileSource.mapFileInputStream.getChannel();
else {
FileInputStream fis = new FileInputStream(tileSource.mapFile);
mInputChannel = fis.getChannel();
}
mFileSize = mInputChannel.size();
} catch (IOException e) { } catch (IOException e) {
log.error(e.getMessage()); log.error(e.getMessage());
/* make sure that the file is closed */ /* make sure that the file is closed */
@ -314,12 +318,10 @@ public class MapDatabase implements ITileDataSource {
@Override @Override
public void dispose() { public void dispose() {
mReadBuffer = null; if (mInputChannel != null) {
if (mInputFile != null) {
try { try {
mInputFile.close(); mInputChannel.close();
mInputFile = null; mInputChannel = null;
} catch (IOException e) { } catch (IOException e) {
log.error(e.getMessage()); log.error(e.getMessage());
} }
@ -351,13 +353,13 @@ public class MapDatabase implements ITileDataSource {
private void processBlock(QueryParameters queryParameters, private void processBlock(QueryParameters queryParameters,
SubFileParameter subFileParameter, ITileDataSink mapDataSink, SubFileParameter subFileParameter, ITileDataSink mapDataSink,
BoundingBox boundingBox, Selector selector, BoundingBox boundingBox, Selector selector,
MapReadResult mapReadResult) { MapReadResult mapReadResult, ReadBuffer readBuffer) {
if (!processBlockSignature()) { if (!processBlockSignature(readBuffer)) {
return; return;
} }
int[][] zoomTable = readZoomTable(subFileParameter); int[][] zoomTable = readZoomTable(subFileParameter, readBuffer);
if (zoomTable == null) { if (zoomTable == null) {
return; return;
} }
@ -366,7 +368,7 @@ public class MapDatabase implements ITileDataSource {
int waysOnQueryZoomLevel = zoomTable[zoomTableRow][1]; int waysOnQueryZoomLevel = zoomTable[zoomTableRow][1];
/* get the relative offset to the first stored way in the block */ /* get the relative offset to the first stored way in the block */
int firstWayOffset = mReadBuffer.readUnsignedInt(); int firstWayOffset = readBuffer.readUnsignedInt();
if (firstWayOffset < 0) { if (firstWayOffset < 0) {
log.warn(INVALID_FIRST_WAY_OFFSET + firstWayOffset); log.warn(INVALID_FIRST_WAY_OFFSET + firstWayOffset);
if (mDebugFile) { if (mDebugFile) {
@ -376,8 +378,8 @@ public class MapDatabase implements ITileDataSource {
} }
/* add the current buffer position to the relative first way offset */ /* add the current buffer position to the relative first way offset */
firstWayOffset += mReadBuffer.getBufferPosition(); firstWayOffset += readBuffer.getBufferPosition();
if (firstWayOffset > mReadBuffer.getBufferSize()) { if (firstWayOffset > readBuffer.getBufferSize()) {
log.warn(INVALID_FIRST_WAY_OFFSET + firstWayOffset); log.warn(INVALID_FIRST_WAY_OFFSET + firstWayOffset);
if (mDebugFile) { if (mDebugFile) {
log.warn(DEBUG_SIGNATURE_BLOCK + mSignatureBlock); log.warn(DEBUG_SIGNATURE_BLOCK + mSignatureBlock);
@ -391,13 +393,13 @@ public class MapDatabase implements ITileDataSource {
if (mapReadResult != null) if (mapReadResult != null)
pois = new ArrayList<>(); pois = new ArrayList<>();
if (!processPOIs(mapDataSink, poisOnQueryZoomLevel, boundingBox, filterRequired, pois)) { if (!processPOIs(mapDataSink, poisOnQueryZoomLevel, boundingBox, filterRequired, pois, readBuffer)) {
return; return;
} }
/* finished reading POIs, check if the current buffer position is valid */ /* finished reading POIs, check if the current buffer position is valid */
if (mReadBuffer.getBufferPosition() > firstWayOffset) { if (readBuffer.getBufferPosition() > firstWayOffset) {
log.warn("invalid buffer position: " + mReadBuffer.getBufferPosition()); log.warn("invalid buffer position: " + readBuffer.getBufferPosition());
if (mDebugFile) { if (mDebugFile) {
log.warn(DEBUG_SIGNATURE_BLOCK + mSignatureBlock); log.warn(DEBUG_SIGNATURE_BLOCK + mSignatureBlock);
} }
@ -405,13 +407,13 @@ public class MapDatabase implements ITileDataSource {
} }
/* move the pointer to the first way */ /* move the pointer to the first way */
mReadBuffer.setBufferPosition(firstWayOffset); readBuffer.setBufferPosition(firstWayOffset);
List<Way> ways = null; List<Way> ways = null;
if (mapReadResult != null && Selector.POIS != selector) if (mapReadResult != null && Selector.POIS != selector)
ways = new ArrayList<>(); ways = new ArrayList<>();
if (!processWays(queryParameters, mapDataSink, waysOnQueryZoomLevel, boundingBox, filterRequired, selector, ways)) { if (!processWays(queryParameters, mapDataSink, waysOnQueryZoomLevel, boundingBox, filterRequired, selector, ways, readBuffer)) {
return; return;
} }
@ -576,10 +578,9 @@ public class MapDatabase implements ITileDataSource {
} }
/* seek to the current block in the map file */ /* seek to the current block in the map file */
mInputFile.seek(subFileParameter.startAddress + blockPointer);
/* read the current block into the buffer */ /* read the current block into the buffer */
if (!mReadBuffer.readFromFile(blockSize)) { ReadBuffer readBuffer = new ReadBuffer(mInputChannel);
if (!readBuffer.readFromFile(subFileParameter.startAddress + blockPointer, blockSize)) {
/* skip the current block */ /* skip the current block */
log.warn("reading current block has failed: " + blockSize); log.warn("reading current block has failed: " + blockSize);
return; return;
@ -596,7 +597,7 @@ public class MapDatabase implements ITileDataSource {
mTileLatitude = (int) (tileLatitudeDeg * 1E6); mTileLatitude = (int) (tileLatitudeDeg * 1E6);
mTileLongitude = (int) (tileLongitudeDeg * 1E6); mTileLongitude = (int) (tileLongitudeDeg * 1E6);
processBlock(queryParams, subFileParameter, mapDataSink, boundingBox, selector, mapReadResult); processBlock(queryParams, subFileParameter, mapDataSink, boundingBox, selector, mapReadResult, readBuffer);
} }
} }
} }
@ -607,10 +608,10 @@ public class MapDatabase implements ITileDataSource {
* @return true if the block signature could be processed successfully, * @return true if the block signature could be processed successfully,
* false otherwise. * false otherwise.
*/ */
private boolean processBlockSignature() { private boolean processBlockSignature(ReadBuffer readBuffer) {
if (mDebugFile) { if (mDebugFile) {
/* get and check the block signature */ /* get and check the block signature */
mSignatureBlock = mReadBuffer.readUTF8EncodedString(SIGNATURE_LENGTH_BLOCK); mSignatureBlock = readBuffer.readUTF8EncodedString(SIGNATURE_LENGTH_BLOCK);
if (!mSignatureBlock.startsWith("###TileStart")) { if (!mSignatureBlock.startsWith("###TileStart")) {
log.warn("invalid block signature: " + mSignatureBlock); log.warn("invalid block signature: " + mSignatureBlock);
return false; return false;
@ -628,7 +629,7 @@ public class MapDatabase implements ITileDataSource {
* otherwise. * otherwise.
*/ */
private boolean processPOIs(ITileDataSink mapDataSink, int numberOfPois, BoundingBox boundingBox, private boolean processPOIs(ITileDataSink mapDataSink, int numberOfPois, BoundingBox boundingBox,
boolean filterRequired, List<PointOfInterest> pois) { boolean filterRequired, List<PointOfInterest> pois, ReadBuffer readBuffer) {
Tag[] poiTags = mTileSource.fileInfo.poiTags; Tag[] poiTags = mTileSource.fileInfo.poiTags;
MapElement e = mElem; MapElement e = mElem;
@ -638,7 +639,7 @@ public class MapDatabase implements ITileDataSource {
if (mDebugFile) { if (mDebugFile) {
/* get and check the POI signature */ /* get and check the POI signature */
mSignaturePoi = mReadBuffer.readUTF8EncodedString(SIGNATURE_LENGTH_POI); mSignaturePoi = readBuffer.readUTF8EncodedString(SIGNATURE_LENGTH_POI);
if (!mSignaturePoi.startsWith("***POIStart")) { if (!mSignaturePoi.startsWith("***POIStart")) {
log.warn("invalid POI signature: " + mSignaturePoi); log.warn("invalid POI signature: " + mSignaturePoi);
log.warn(DEBUG_SIGNATURE_BLOCK + mSignatureBlock); log.warn(DEBUG_SIGNATURE_BLOCK + mSignatureBlock);
@ -647,12 +648,12 @@ public class MapDatabase implements ITileDataSource {
} }
/* get the POI latitude offset (VBE-S) */ /* get the POI latitude offset (VBE-S) */
int latitude = mTileLatitude + mReadBuffer.readSignedInt(); int latitude = mTileLatitude + readBuffer.readSignedInt();
/* get the POI longitude offset (VBE-S) */ /* get the POI longitude offset (VBE-S) */
int longitude = mTileLongitude + mReadBuffer.readSignedInt(); int longitude = mTileLongitude + readBuffer.readSignedInt();
/* get the special byte which encodes multiple flags */ /* get the special byte which encodes multiple flags */
byte specialByte = mReadBuffer.readByte(); byte specialByte = readBuffer.readByte();
/* bit 1-4 represent the layer */ /* bit 1-4 represent the layer */
byte layer = (byte) ((specialByte & POI_LAYER_BITMASK) >>> POI_LAYER_SHIFT); byte layer = (byte) ((specialByte & POI_LAYER_BITMASK) >>> POI_LAYER_SHIFT);
@ -661,29 +662,29 @@ public class MapDatabase implements ITileDataSource {
byte numberOfTags = (byte) (specialByte & POI_NUMBER_OF_TAGS_BITMASK); byte numberOfTags = (byte) (specialByte & POI_NUMBER_OF_TAGS_BITMASK);
if (numberOfTags != 0) { if (numberOfTags != 0) {
if (!mReadBuffer.readTags(e.tags, poiTags, numberOfTags)) if (!readBuffer.readTags(e.tags, poiTags, numberOfTags))
return false; return false;
} }
/* get the feature bitmask (1 byte) */ /* get the feature bitmask (1 byte) */
byte featureByte = mReadBuffer.readByte(); byte featureByte = readBuffer.readByte();
/* bit 1-3 enable optional features /* bit 1-3 enable optional features
* check if the POI has a name */ * check if the POI has a name */
if ((featureByte & POI_FEATURE_NAME) != 0) { if ((featureByte & POI_FEATURE_NAME) != 0) {
String str = mTileSource.extractLocalized(mReadBuffer.readUTF8EncodedString()); String str = mTileSource.extractLocalized(readBuffer.readUTF8EncodedString());
e.tags.add(new Tag(Tag.KEY_NAME, str, false)); e.tags.add(new Tag(Tag.KEY_NAME, str, false));
} }
/* check if the POI has a house number */ /* check if the POI has a house number */
if ((featureByte & POI_FEATURE_HOUSE_NUMBER) != 0) { if ((featureByte & POI_FEATURE_HOUSE_NUMBER) != 0) {
String str = mReadBuffer.readUTF8EncodedString(); String str = readBuffer.readUTF8EncodedString();
e.tags.add(new Tag(Tag.KEY_HOUSE_NUMBER, str, false)); e.tags.add(new Tag(Tag.KEY_HOUSE_NUMBER, str, false));
} }
/* check if the POI has an elevation */ /* check if the POI has an elevation */
if ((featureByte & POI_FEATURE_ELEVATION) != 0) { if ((featureByte & POI_FEATURE_ELEVATION) != 0) {
String str = Integer.toString(mReadBuffer.readSignedInt()); String str = Integer.toString(readBuffer.readSignedInt());
e.tags.add(new Tag(Tag.KEY_ELE, str, false)); e.tags.add(new Tag(Tag.KEY_ELE, str, false));
} }
mTileProjection.projectPoint(latitude, longitude, e); mTileProjection.projectPoint(latitude, longitude, e);
@ -712,9 +713,9 @@ public class MapDatabase implements ITileDataSource {
return true; return true;
} }
private boolean processWayDataBlock(MapElement e, boolean doubleDeltaEncoding, boolean isLine, List<GeoPoint[]> wayCoordinates, int[] labelPosition) { private boolean processWayDataBlock(MapElement e, boolean doubleDeltaEncoding, boolean isLine, List<GeoPoint[]> wayCoordinates, int[] labelPosition, ReadBuffer readBuffer) {
/* get and check the number of way coordinate blocks (VBE-U) */ /* get and check the number of way coordinate blocks (VBE-U) */
int numBlocks = mReadBuffer.readUnsignedInt(); int numBlocks = readBuffer.readUnsignedInt();
if (numBlocks < 1 || numBlocks > Short.MAX_VALUE) { if (numBlocks < 1 || numBlocks > Short.MAX_VALUE) {
log.warn("invalid number of way coordinate blocks: " + numBlocks); log.warn("invalid number of way coordinate blocks: " + numBlocks);
return false; return false;
@ -726,7 +727,7 @@ public class MapDatabase implements ITileDataSource {
/* read the way coordinate blocks */ /* read the way coordinate blocks */
for (int coordinateBlock = 0; coordinateBlock < numBlocks; ++coordinateBlock) { for (int coordinateBlock = 0; coordinateBlock < numBlocks; ++coordinateBlock) {
int numWayNodes = mReadBuffer.readUnsignedInt(); int numWayNodes = readBuffer.readUnsignedInt();
if (numWayNodes < 2 || numWayNodes > Short.MAX_VALUE) { if (numWayNodes < 2 || numWayNodes > Short.MAX_VALUE) {
log.warn("invalid number of way nodes: " + numWayNodes); log.warn("invalid number of way nodes: " + numWayNodes);
@ -744,7 +745,7 @@ public class MapDatabase implements ITileDataSource {
// label position must be set on first coordinate block // label position must be set on first coordinate block
wayLengths[coordinateBlock] = decodeWayNodes(doubleDeltaEncoding, e, len, isLine, wayLengths[coordinateBlock] = decodeWayNodes(doubleDeltaEncoding, e, len, isLine,
coordinateBlock == 0 ? labelPosition : null, waySegment); coordinateBlock == 0 ? labelPosition : null, waySegment, readBuffer);
if (wayCoordinates != null) if (wayCoordinates != null)
wayCoordinates.add(waySegment); wayCoordinates.add(waySegment);
@ -753,9 +754,9 @@ public class MapDatabase implements ITileDataSource {
return true; return true;
} }
private int decodeWayNodes(boolean doubleDelta, MapElement e, int length, boolean isLine, int[] labelPosition, GeoPoint[] waySegment) { private int decodeWayNodes(boolean doubleDelta, MapElement e, int length, boolean isLine, int[] labelPosition, GeoPoint[] waySegment, ReadBuffer readBuffer) {
int[] buffer = mIntBuffer; int[] buffer = mIntBuffer;
mReadBuffer.readSignedInt(buffer, length); readBuffer.readSignedInt(buffer, length);
float[] outBuffer = e.ensurePointSize(e.pointNextPos + length, true); float[] outBuffer = e.ensurePointSize(e.pointNextPos + length, true);
int outPos = e.pointNextPos; int outPos = e.pointNextPos;
@ -848,7 +849,7 @@ public class MapDatabase implements ITileDataSource {
*/ */
private boolean processWays(QueryParameters queryParameters, ITileDataSink mapDataSink, private boolean processWays(QueryParameters queryParameters, ITileDataSink mapDataSink,
int numberOfWays, BoundingBox boundingBox, boolean filterRequired, int numberOfWays, BoundingBox boundingBox, boolean filterRequired,
Selector selector, List<Way> ways) { Selector selector, List<Way> ways, ReadBuffer readBuffer) {
Tag[] wayTags = mTileSource.fileInfo.wayTags; Tag[] wayTags = mTileSource.fileInfo.wayTags;
MapElement e = mElem; MapElement e = mElem;
@ -860,9 +861,9 @@ public class MapDatabase implements ITileDataSource {
stringOffset = 0; stringOffset = 0;
if (mTileSource.experimental) { if (mTileSource.experimental) {
stringsSize = mReadBuffer.readUnsignedInt(); stringsSize = readBuffer.readUnsignedInt();
stringOffset = mReadBuffer.getBufferPosition(); stringOffset = readBuffer.getBufferPosition();
mReadBuffer.skipBytes(stringsSize); readBuffer.skipBytes(stringsSize);
} }
//setTileClipping(queryParameters); //setTileClipping(queryParameters);
@ -873,7 +874,7 @@ public class MapDatabase implements ITileDataSource {
if (mDebugFile) { if (mDebugFile) {
// get and check the way signature // get and check the way signature
mSignatureWay = mReadBuffer.readUTF8EncodedString(SIGNATURE_LENGTH_WAY); mSignatureWay = readBuffer.readUTF8EncodedString(SIGNATURE_LENGTH_WAY);
if (!mSignatureWay.startsWith("---WayStart")) { if (!mSignatureWay.startsWith("---WayStart")) {
log.warn("invalid way signature: " + mSignatureWay); log.warn("invalid way signature: " + mSignatureWay);
log.warn(DEBUG_SIGNATURE_BLOCK + mSignatureBlock); log.warn(DEBUG_SIGNATURE_BLOCK + mSignatureBlock);
@ -882,7 +883,7 @@ public class MapDatabase implements ITileDataSource {
} }
if (queryParameters.useTileBitmask) { if (queryParameters.useTileBitmask) {
elementCounter = mReadBuffer.skipWays(queryParameters.queryTileBitmask, elementCounter = readBuffer.skipWays(queryParameters.queryTileBitmask,
elementCounter); elementCounter);
if (elementCounter == 0) if (elementCounter == 0)
@ -891,19 +892,19 @@ public class MapDatabase implements ITileDataSource {
if (elementCounter < 0) if (elementCounter < 0)
return false; return false;
if (mTileSource.experimental && mReadBuffer.lastTagPosition > 0) { if (mTileSource.experimental && readBuffer.lastTagPosition > 0) {
int pos = mReadBuffer.getBufferPosition(); int pos = readBuffer.getBufferPosition();
mReadBuffer.setBufferPosition(mReadBuffer.lastTagPosition); readBuffer.setBufferPosition(readBuffer.lastTagPosition);
byte numberOfTags = byte numberOfTags =
(byte) (mReadBuffer.readByte() & WAY_NUMBER_OF_TAGS_BITMASK); (byte) (readBuffer.readByte() & WAY_NUMBER_OF_TAGS_BITMASK);
if (!mReadBuffer.readTags(e.tags, wayTags, numberOfTags)) if (!readBuffer.readTags(e.tags, wayTags, numberOfTags))
return false; return false;
mReadBuffer.setBufferPosition(pos); readBuffer.setBufferPosition(pos);
} }
} else { } else {
int wayDataSize = mReadBuffer.readUnsignedInt(); int wayDataSize = readBuffer.readUnsignedInt();
if (wayDataSize < 0) { if (wayDataSize < 0) {
log.warn("invalid way data size: " + wayDataSize); log.warn("invalid way data size: " + wayDataSize);
if (mDebugFile) { if (mDebugFile) {
@ -914,11 +915,11 @@ public class MapDatabase implements ITileDataSource {
} }
/* ignore the way tile bitmask (2 bytes) */ /* ignore the way tile bitmask (2 bytes) */
mReadBuffer.skipBytes(2); readBuffer.skipBytes(2);
} }
/* get the special byte which encodes multiple flags */ /* get the special byte which encodes multiple flags */
byte specialByte = mReadBuffer.readByte(); byte specialByte = readBuffer.readByte();
/* bit 1-4 represent the layer */ /* bit 1-4 represent the layer */
byte layer = (byte) ((specialByte & WAY_LAYER_BITMASK) >>> WAY_LAYER_SHIFT); byte layer = (byte) ((specialByte & WAY_LAYER_BITMASK) >>> WAY_LAYER_SHIFT);
@ -926,12 +927,12 @@ public class MapDatabase implements ITileDataSource {
byte numberOfTags = (byte) (specialByte & WAY_NUMBER_OF_TAGS_BITMASK); byte numberOfTags = (byte) (specialByte & WAY_NUMBER_OF_TAGS_BITMASK);
if (numberOfTags != 0) { if (numberOfTags != 0) {
if (!mReadBuffer.readTags(e.tags, wayTags, numberOfTags)) if (!readBuffer.readTags(e.tags, wayTags, numberOfTags))
return false; return false;
} }
/* get the feature bitmask (1 byte) */ /* get the feature bitmask (1 byte) */
byte featureByte = mReadBuffer.readByte(); byte featureByte = readBuffer.readByte();
/* bit 1-6 enable optional features */ /* bit 1-6 enable optional features */
boolean featureWayDoubleDeltaEncoding = boolean featureWayDoubleDeltaEncoding =
@ -943,42 +944,42 @@ public class MapDatabase implements ITileDataSource {
if (mTileSource.experimental) { if (mTileSource.experimental) {
if (hasName) { if (hasName) {
int textPos = mReadBuffer.readUnsignedInt(); int textPos = readBuffer.readUnsignedInt();
String str = mTileSource.extractLocalized(mReadBuffer.readUTF8EncodedStringAt(stringOffset + textPos)); String str = mTileSource.extractLocalized(readBuffer.readUTF8EncodedStringAt(stringOffset + textPos));
e.tags.add(new Tag(Tag.KEY_NAME, str, false)); e.tags.add(new Tag(Tag.KEY_NAME, str, false));
} }
if (hasHouseNr) { if (hasHouseNr) {
int textPos = mReadBuffer.readUnsignedInt(); int textPos = readBuffer.readUnsignedInt();
String str = mReadBuffer.readUTF8EncodedStringAt(stringOffset + textPos); String str = readBuffer.readUTF8EncodedStringAt(stringOffset + textPos);
e.tags.add(new Tag(Tag.KEY_HOUSE_NUMBER, str, false)); e.tags.add(new Tag(Tag.KEY_HOUSE_NUMBER, str, false));
} }
if (hasRef) { if (hasRef) {
int textPos = mReadBuffer.readUnsignedInt(); int textPos = readBuffer.readUnsignedInt();
String str = mReadBuffer.readUTF8EncodedStringAt(stringOffset + textPos); String str = readBuffer.readUTF8EncodedStringAt(stringOffset + textPos);
e.tags.add(new Tag(Tag.KEY_REF, str, false)); e.tags.add(new Tag(Tag.KEY_REF, str, false));
} }
} else { } else {
if (hasName) { if (hasName) {
String str = mTileSource.extractLocalized(mReadBuffer.readUTF8EncodedString()); String str = mTileSource.extractLocalized(readBuffer.readUTF8EncodedString());
e.tags.add(new Tag(Tag.KEY_NAME, str, false)); e.tags.add(new Tag(Tag.KEY_NAME, str, false));
} }
if (hasHouseNr) { if (hasHouseNr) {
String str = mReadBuffer.readUTF8EncodedString(); String str = readBuffer.readUTF8EncodedString();
e.tags.add(new Tag(Tag.KEY_HOUSE_NUMBER, str, false)); e.tags.add(new Tag(Tag.KEY_HOUSE_NUMBER, str, false));
} }
if (hasRef) { if (hasRef) {
String str = mReadBuffer.readUTF8EncodedString(); String str = readBuffer.readUTF8EncodedString();
e.tags.add(new Tag(Tag.KEY_REF, str, false)); e.tags.add(new Tag(Tag.KEY_REF, str, false));
} }
} }
int[] labelPosition = null; int[] labelPosition = null;
if ((featureByte & WAY_FEATURE_LABEL_POSITION) != 0) { if ((featureByte & WAY_FEATURE_LABEL_POSITION) != 0) {
labelPosition = readOptionalLabelPosition(); labelPosition = readOptionalLabelPosition(readBuffer);
} }
if ((featureByte & WAY_FEATURE_DATA_BLOCKS_BYTE) != 0) { if ((featureByte & WAY_FEATURE_DATA_BLOCKS_BYTE) != 0) {
wayDataBlocks = mReadBuffer.readUnsignedInt(); wayDataBlocks = readBuffer.readUnsignedInt();
if (wayDataBlocks < 1) { if (wayDataBlocks < 1) {
log.warn("invalid number of way data blocks: " + wayDataBlocks); log.warn("invalid number of way data blocks: " + wayDataBlocks);
@ -999,7 +1000,7 @@ public class MapDatabase implements ITileDataSource {
if (ways != null) if (ways != null)
wayNodes = new ArrayList<>(); wayNodes = new ArrayList<>();
if (!processWayDataBlock(e, featureWayDoubleDeltaEncoding, linearFeature, wayNodes, labelPosition)) if (!processWayDataBlock(e, featureWayDoubleDeltaEncoding, linearFeature, wayNodes, labelPosition, readBuffer))
return false; return false;
/* drop invalid outer ring */ /* drop invalid outer ring */
@ -1152,14 +1153,14 @@ public class MapDatabase implements ITileDataSource {
return mapReadResult; return mapReadResult;
} }
private int[] readOptionalLabelPosition() { private int[] readOptionalLabelPosition(ReadBuffer readBuffer) {
int[] labelPosition = new int[2]; int[] labelPosition = new int[2];
/* get the label position latitude offset (VBE-S) */ /* get the label position latitude offset (VBE-S) */
labelPosition[1] = mReadBuffer.readSignedInt(); labelPosition[1] = readBuffer.readSignedInt();
/* get the label position longitude offset (VBE-S) */ /* get the label position longitude offset (VBE-S) */
labelPosition[0] = mReadBuffer.readSignedInt(); labelPosition[0] = readBuffer.readSignedInt();
return labelPosition; return labelPosition;
} }
@ -1187,7 +1188,7 @@ public class MapDatabase implements ITileDataSource {
return readMapData(upperLeft, lowerRight, Selector.POIS); return readMapData(upperLeft, lowerRight, Selector.POIS);
} }
private int[][] readZoomTable(SubFileParameter subFileParameter) { private int[][] readZoomTable(SubFileParameter subFileParameter, ReadBuffer readBuffer) {
int rows = subFileParameter.zoomLevelMax - subFileParameter.zoomLevelMin + 1; int rows = subFileParameter.zoomLevelMax - subFileParameter.zoomLevelMin + 1;
int[][] zoomTable = new int[rows][2]; int[][] zoomTable = new int[rows][2];
@ -1195,8 +1196,8 @@ public class MapDatabase implements ITileDataSource {
int cumulatedNumberOfWays = 0; int cumulatedNumberOfWays = 0;
for (int row = 0; row < rows; row++) { for (int row = 0; row < rows; row++) {
cumulatedNumberOfPois += mReadBuffer.readUnsignedInt(); cumulatedNumberOfPois += readBuffer.readUnsignedInt();
cumulatedNumberOfWays += mReadBuffer.readUnsignedInt(); cumulatedNumberOfWays += readBuffer.readUnsignedInt();
zoomTable[row][0] = cumulatedNumberOfPois; zoomTable[row][0] = cumulatedNumberOfPois;
zoomTable[row][1] = cumulatedNumberOfWays; zoomTable[row][1] = cumulatedNumberOfWays;

70
vtm/src/org/oscim/tiling/source/mapfile/MapFileTileSource.java
View File

@ -1,7 +1,7 @@
/* /*
* Copyright 2013 mapsforge.org * Copyright 2013 mapsforge.org
* Copyright 2013 Hannes Janetzek * Copyright 2013 Hannes Janetzek
* Copyright 2016-2018 devemux86 * Copyright 2016-2020 devemux86
* *
* This file is part of the OpenScienceMap project (http://www.opensciencemap.org). * This file is part of the OpenScienceMap project (http://www.opensciencemap.org).
* *
@ -25,13 +25,13 @@ import org.oscim.tiling.OverzoomTileDataSource;
import org.oscim.tiling.TileSource; import org.oscim.tiling.TileSource;
import org.oscim.tiling.source.mapfile.header.MapFileHeader; import org.oscim.tiling.source.mapfile.header.MapFileHeader;
import org.oscim.tiling.source.mapfile.header.MapFileInfo; import org.oscim.tiling.source.mapfile.header.MapFileInfo;
import org.oscim.utils.IOUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.nio.channels.FileChannel;
public class MapFileTileSource extends TileSource implements IMapFileTileSource { public class MapFileTileSource extends TileSource implements IMapFileTileSource {
private static final Logger log = LoggerFactory.getLogger(MapFileTileSource.class); private static final Logger log = LoggerFactory.getLogger(MapFileTileSource.class);
@ -40,14 +40,14 @@ public class MapFileTileSource extends TileSource implements IMapFileTileSource
* Amount of cache blocks that the index cache should store. * Amount of cache blocks that the index cache should store.
*/ */
private static final int INDEX_CACHE_SIZE = 64; private static final int INDEX_CACHE_SIZE = 64;
private static final String READ_ONLY_MODE = "r";
MapFileHeader fileHeader; MapFileHeader fileHeader;
MapFileInfo fileInfo; MapFileInfo fileInfo;
IndexCache databaseIndexCache; IndexCache databaseIndexCache;
boolean experimental; boolean experimental;
File mapFile; File mapFile;
private RandomAccessFile mInputFile; FileInputStream mapFileInputStream;
private FileChannel inputChannel;
/** /**
* The preferred language when extracting labels from this tile source. * The preferred language when extracting labels from this tile source.
@ -98,6 +98,10 @@ public class MapFileTileSource extends TileSource implements IMapFileTileSource
return true; return true;
} }
public void setMapFileInputStream(FileInputStream fileInputStream) {
this.mapFileInputStream = fileInputStream;
}
@Override @Override
public void setPreferredLanguage(String preferredLanguage) { public void setPreferredLanguage(String preferredLanguage) {
this.preferredLanguage = preferredLanguage; this.preferredLanguage = preferredLanguage;
@ -105,31 +109,38 @@ public class MapFileTileSource extends TileSource implements IMapFileTileSource
@Override @Override
public OpenResult open() { public OpenResult open() {
if (!options.containsKey("file")) if (mapFileInputStream == null && !options.containsKey("file"))
return new OpenResult("no map file set"); return new OpenResult("no map file set");
try { try {
// make sure to close any previously opened file first // false positive: stream gets closed when the channel is closed
//close(); // see e.g. http://bugs.java.com/bugdatabase/view_bug.do?bug_id=4796385
File file = null;
if (mapFileInputStream != null)
inputChannel = mapFileInputStream.getChannel();
else {
// make sure to close any previously opened file first
//close();
File file = new File(options.get("file")); file = new File(options.get("file"));
// check if the file exists and is readable // check if the file exists and is readable
if (!file.exists()) { if (!file.exists()) {
return new OpenResult("file does not exist: " + file); return new OpenResult("file does not exist: " + file);
} else if (!file.isFile()) { } else if (!file.isFile()) {
return new OpenResult("not a file: " + file); return new OpenResult("not a file: " + file);
} else if (!file.canRead()) { } else if (!file.canRead()) {
return new OpenResult("cannot read file: " + file); return new OpenResult("cannot read file: " + file);
}
FileInputStream fis = new FileInputStream(file);
inputChannel = fis.getChannel();
} }
long fileSize = inputChannel.size();
// open the file in read only mode ReadBuffer readBuffer = new ReadBuffer(inputChannel);
mInputFile = new RandomAccessFile(file, READ_ONLY_MODE);
long mFileSize = mInputFile.length();
ReadBuffer mReadBuffer = new ReadBuffer(mInputFile);
fileHeader = new MapFileHeader(); fileHeader = new MapFileHeader();
OpenResult openResult = fileHeader.readHeader(mReadBuffer, mFileSize); OpenResult openResult = fileHeader.readHeader(readBuffer, fileSize);
if (!openResult.isSuccess()) { if (!openResult.isSuccess()) {
close(); close();
@ -137,10 +148,7 @@ public class MapFileTileSource extends TileSource implements IMapFileTileSource
} }
fileInfo = fileHeader.getMapFileInfo(); fileInfo = fileHeader.getMapFileInfo();
mapFile = file; mapFile = file;
databaseIndexCache = new IndexCache(mInputFile, INDEX_CACHE_SIZE); databaseIndexCache = new IndexCache(inputChannel, INDEX_CACHE_SIZE);
// Experimental?
//experimental = fileInfo.fileVersion == 4;
log.debug("File version: " + fileInfo.fileVersion); log.debug("File version: " + fileInfo.fileVersion);
return OpenResult.SUCCESS; return OpenResult.SUCCESS;
@ -164,8 +172,14 @@ public class MapFileTileSource extends TileSource implements IMapFileTileSource
@Override @Override
public void close() { public void close() {
IOUtils.closeQuietly(mInputFile); if (inputChannel != null) {
mInputFile = null; try {
inputChannel.close();
inputChannel = null;
} catch (IOException e) {
log.error(e.getMessage());
}
}
fileHeader = null; fileHeader = null;
fileInfo = null; fileInfo = null;
mapFile = null; mapFile = null;

52
vtm/src/org/oscim/tiling/source/mapfile/ReadBuffer.java
View File

@ -25,6 +25,8 @@ import org.oscim.utils.Parameters;
import java.io.IOException; import java.io.IOException;
import java.io.RandomAccessFile; import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.logging.Level; import java.util.logging.Level;
@ -39,12 +41,13 @@ public class ReadBuffer {
private byte[] mBufferData; private byte[] mBufferData;
private int mBufferPosition; private int mBufferPosition;
private final RandomAccessFile mInputFile; private ByteBuffer mBufferWrapper;
private final FileChannel mInputChannel;
private final List<Integer> mTagIds = new ArrayList<>(); private final List<Integer> mTagIds = new ArrayList<>();
ReadBuffer(RandomAccessFile inputFile) { ReadBuffer(FileChannel inputChannel) {
mInputFile = inputFile; mInputChannel = inputChannel;
} }
/** /**
@ -91,13 +94,54 @@ public class ReadBuffer {
LOG.log(Level.SEVERE, t.getMessage(), t); LOG.log(Level.SEVERE, t.getMessage(), t);
return false; return false;
} }
mBufferWrapper = ByteBuffer.wrap(mBufferData, 0, length);
} }
mBufferPosition = 0; mBufferPosition = 0;
mBufferWrapper.clear();
// reset the buffer position and read the data into the buffer // reset the buffer position and read the data into the buffer
// bufferPosition = 0; // bufferPosition = 0;
return mInputFile.read(mBufferData, 0, length) == length; return mInputChannel.read(mBufferWrapper) == length;
}
/**
* Reads the given amount of bytes from the file into the read buffer and
* resets the internal buffer position. If
* the capacity of the read buffer is too small, a larger one is created
* automatically.
*
* @param offset the offset position, measured in bytes from the beginning of the file, at which to set the file pointer.
* @param length the amount of bytes to read from the file.
* @return true if the whole data was read successfully, false otherwise.
* @throws IOException if an error occurs while reading the file.
*/
public boolean readFromFile(long offset, int length) throws IOException {
// ensure that the read buffer is large enough
if (mBufferData == null || mBufferData.length < length) {
// ensure that the read buffer is not too large
if (length > Parameters.MAXIMUM_BUFFER_SIZE) {
LOG.warning("invalid read length: " + length);
return false;
}
try {
mBufferData = new byte[length];
} catch (Throwable t) {
LOG.log(Level.SEVERE, t.getMessage(), t);
return false;
}
mBufferWrapper = ByteBuffer.wrap(mBufferData, 0, length);
}
mBufferPosition = 0;
mBufferWrapper.clear();
// reset the buffer position and read the data into the buffer
// bufferPosition = 0;
synchronized (mInputChannel) {
mInputChannel.position(offset);
return mInputChannel.read(mBufferWrapper) == length;
}
} }
/** /**