From 86794c88381a108a1c145c9f5da0cb2607a4f09a Mon Sep 17 00:00:00 2001 From: Emux Date: Tue, 8 Feb 2022 14:33:40 +0200 Subject: [PATCH] Mapsforge: deduplicate maps, fix overlapping map regions (#903) --- docs/Changelog.md | 2 + .../src/org/oscim/test/MapsforgeTest.java | 13 ++-- vtm/src/org/oscim/core/MapElement.java | 6 +- .../layers/tile/vector/VectorTileLoader.java | 25 ++------ vtm/src/org/oscim/tiling/TileDataSink.java | 8 ++- .../tiling/source/mapfile/MapDatabase.java | 41 ++++++++++--- .../tiling/source/mapfile/MapReadResult.java | 19 ++++-- .../source/mapfile/MultiMapDatabase.java | 60 +++++++++++++------ .../mapfile/MultiMapFileTileSource.java | 9 ++- 9 files changed, 121 insertions(+), 62 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index 02157008..a0a99980 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -2,6 +2,8 @@ ## New since 0.17.0 +- Mapsforge: deduplicate maps [#903](https://github.com/mapsforge/vtm/pull/903) +- Fix overlapping map regions [#903](https://github.com/mapsforge/vtm/pull/903) - Minor improvements and bug fixes - [Solved issues](https://github.com/mapsforge/vtm/issues?q=is%3Aclosed+milestone%3A0.18.0) diff --git a/vtm-playground/src/org/oscim/test/MapsforgeTest.java b/vtm-playground/src/org/oscim/test/MapsforgeTest.java index de423dbd..ada687cf 100644 --- a/vtm-playground/src/org/oscim/test/MapsforgeTest.java +++ b/vtm-playground/src/org/oscim/test/MapsforgeTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2019 devemux86 + * Copyright 2016-2022 devemux86 * Copyright 2018-2019 Gustl22 * * This file is part of the OpenScienceMap project (http://www.opensciencemap.org). @@ -67,6 +67,7 @@ public class MapsforgeTest extends GdxMapApp { mapFileTileSource.setMapFile(mapFile.getAbsolutePath()); tileSource.add(mapFileTileSource); } + tileSource.setDeduplicate(true); //tileSource.setPreferredLanguage("en"); VectorTileLayer l = mMap.setBaseMap(tileSource); @@ -139,13 +140,13 @@ public class MapsforgeTest extends GdxMapApp { for (String arg : args) { File mapFile = new File(arg); if (!mapFile.exists()) { - throw new IllegalArgumentException("file does not exist: " + mapFile); + System.err.println("file does not exist: " + mapFile); } else if (!mapFile.isFile()) { - throw new IllegalArgumentException("not a file: " + mapFile); + System.err.println("not a file: " + mapFile); } else if (!mapFile.canRead()) { - throw new IllegalArgumentException("cannot read file: " + mapFile); - } - result.add(mapFile); + System.err.println("cannot read file: " + mapFile); + } else + result.add(mapFile); } return result; } diff --git a/vtm/src/org/oscim/core/MapElement.java b/vtm/src/org/oscim/core/MapElement.java index 0f9f9a47..65e2e92e 100644 --- a/vtm/src/org/oscim/core/MapElement.java +++ b/vtm/src/org/oscim/core/MapElement.java @@ -2,7 +2,7 @@ * Copyright 2012 Hannes Janetzek * Copyright 2016 Andrey Novikov * Copyright 2017-2019 Gustl22 - * Copyright 2018-2019 devemux86 + * Copyright 2018-2022 devemux86 * Copyright 2019 marq24 * * This file is part of the OpenScienceMap project (http://www.opensciencemap.org). @@ -38,6 +38,8 @@ public class MapElement extends GeometryBuffer { */ public int layer; + public int level; + public final TagSet tags = new TagSet(); public MapElement() { @@ -61,6 +63,7 @@ public class MapElement extends GeometryBuffer { this.centroidPosition = element.centroidPosition; this.labelPosition = element.labelPosition; this.setLayer(element.layer); + this.level = element.level; } /** @@ -121,6 +124,7 @@ public class MapElement extends GeometryBuffer { @Override public MapElement clear() { layer = 5; + level = 0; super.clear(); return this; } diff --git a/vtm/src/org/oscim/layers/tile/vector/VectorTileLoader.java b/vtm/src/org/oscim/layers/tile/vector/VectorTileLoader.java index 011ab3ad..1418d909 100644 --- a/vtm/src/org/oscim/layers/tile/vector/VectorTileLoader.java +++ b/vtm/src/org/oscim/layers/tile/vector/VectorTileLoader.java @@ -1,6 +1,6 @@ /* * Copyright 2012-2014 Hannes Janetzek - * Copyright 2016-2019 devemux86 + * Copyright 2016-2022 devemux86 * * This file is part of the OpenScienceMap project (http://www.opensciencemap.org). * @@ -18,28 +18,13 @@ package org.oscim.layers.tile.vector; import org.oscim.core.GeometryBuffer.GeometryType; -import org.oscim.core.MapElement; -import org.oscim.core.MercatorProjection; -import org.oscim.core.Tag; -import org.oscim.core.TagSet; -import org.oscim.core.Tile; +import org.oscim.core.*; import org.oscim.layers.tile.MapTile; import org.oscim.layers.tile.TileLoader; -import org.oscim.renderer.bucket.CircleBucket; -import org.oscim.renderer.bucket.LineBucket; -import org.oscim.renderer.bucket.LineTexBucket; -import org.oscim.renderer.bucket.MeshBucket; -import org.oscim.renderer.bucket.PolygonBucket; -import org.oscim.renderer.bucket.RenderBuckets; +import org.oscim.renderer.bucket.*; import org.oscim.theme.IRenderTheme; import org.oscim.theme.RenderTheme; -import org.oscim.theme.styles.AreaStyle; -import org.oscim.theme.styles.CircleStyle; -import org.oscim.theme.styles.ExtrusionStyle; -import org.oscim.theme.styles.LineStyle; -import org.oscim.theme.styles.RenderStyle; -import org.oscim.theme.styles.SymbolStyle; -import org.oscim.theme.styles.TextStyle; +import org.oscim.theme.styles.*; import org.oscim.tiling.ITileDataSource; import org.oscim.tiling.QueryResult; import org.slf4j.Logger; @@ -209,7 +194,7 @@ public class VectorTileLoader extends TileLoader implements RenderStyle.Callback if (element.type == GeometryType.POINT) { renderNode(renderTheme.matchElement(element.type, tags, mTile.zoomLevel)); } else { - mCurBucket = getValidLayer(element.layer) * renderTheme.getLevels(); + mCurBucket = getValidLayer(element.layer - (element.isPoly() ? element.level : 0)) * renderTheme.getLevels(); renderWay(renderTheme.matchElement(element.type, tags, mTile.zoomLevel)); } clearState(); diff --git a/vtm/src/org/oscim/tiling/TileDataSink.java b/vtm/src/org/oscim/tiling/TileDataSink.java index d46267bb..5393da6d 100644 --- a/vtm/src/org/oscim/tiling/TileDataSink.java +++ b/vtm/src/org/oscim/tiling/TileDataSink.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2018 devemux86 + * Copyright 2016-2022 devemux86 * * This program is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free Software @@ -17,8 +17,14 @@ package org.oscim.tiling; import org.oscim.backend.canvas.Bitmap; import org.oscim.core.MapElement; +import java.util.HashSet; +import java.util.Set; + public class TileDataSink implements ITileDataSink { + public final Set hashPois = new HashSet<>(); + public final Set hashWays = new HashSet<>(); + private QueryResult result; private final ITileDataSink sink; diff --git a/vtm/src/org/oscim/tiling/source/mapfile/MapDatabase.java b/vtm/src/org/oscim/tiling/source/mapfile/MapDatabase.java index 4385b9c0..deba15da 100644 --- a/vtm/src/org/oscim/tiling/source/mapfile/MapDatabase.java +++ b/vtm/src/org/oscim/tiling/source/mapfile/MapDatabase.java @@ -2,7 +2,7 @@ * Copyright 2010, 2011, 2012 mapsforge.org * Copyright 2013, 2014 Hannes Janetzek * Copyright 2014-2015 Ludwig M Brinckmann - * Copyright 2016-2020 devemux86 + * Copyright 2016-2022 devemux86 * Copyright 2016 Andrey Novikov * Copyright 2017-2018 Gustl22 * Copyright 2018 Bezzu @@ -31,6 +31,7 @@ import org.oscim.layers.tile.MapTile; import org.oscim.layers.tile.buildings.BuildingLayer; import org.oscim.tiling.ITileDataSink; import org.oscim.tiling.ITileDataSource; +import org.oscim.tiling.TileDataSink; import org.oscim.tiling.source.mapfile.header.SubFileParameter; import org.oscim.utils.Parameters; import org.oscim.utils.geom.TileClipper; @@ -224,6 +225,9 @@ public class MapDatabase implements ITileDataSource { private int zoomLevelMin = 0; private int zoomLevelMax = Byte.MAX_VALUE; + private boolean deduplicate; + private int level; + public MapDatabase(MapFileTileSource tileSource) throws IOException { mTileSource = tileSource; try { @@ -306,7 +310,10 @@ public class MapDatabase implements ITileDataSource { QueryCalculations.calculateBaseTiles(queryParameters, tile, subFileParameter); QueryCalculations.calculateBlocks(queryParameters, subFileParameter); - processBlocks(sink, queryParameters, subFileParameter); + if (deduplicate) + processBlocks(sink, queryParameters, subFileParameter, tile.getBoundingBox(), Selector.ALL, new MapReadResult()); + else + processBlocks(sink, queryParameters, subFileParameter); } catch (IOException e) { log.error(e.getMessage()); sink.completed(FAILED); @@ -424,6 +431,14 @@ public class MapDatabase implements ITileDataSource { } } + void setDeduplicate(boolean deduplicate) { + this.deduplicate = deduplicate; + } + + void setLevel(int level) { + this.level = level; + } + private void setTileClipping(QueryParameters queryParameters, SubFileParameter subFileParameter, long currentRow, long currentCol) { long numRows = queryParameters.toBlockY - queryParameters.fromBlockY; @@ -693,7 +708,9 @@ public class MapDatabase implements ITileDataSource { continue; e.setLayer(layer); + e.level = level; + PointOfInterest poi = null; if (pois != null) { List tags = new ArrayList<>(); for (int i = 0; i < e.tags.size(); i++) @@ -702,12 +719,15 @@ public class MapDatabase implements ITileDataSource { // depending on the zoom level configuration the poi can lie outside // the tile requested, we filter them out here if (!filterRequired || boundingBox.contains(position)) { - pois.add(new PointOfInterest(layer, tags, position)); + poi = new PointOfInterest(layer, tags, position); + pois.add(poi); } } - if (mapDataSink != null) - mapDataSink.process(e); + if (mapDataSink != null) { + if (!deduplicate || poi == null || ((TileDataSink) mapDataSink).hashPois.add(poi.hashCode())) + mapDataSink.process(e); + } } return true; @@ -1046,7 +1066,9 @@ public class MapDatabase implements ITileDataSource { e.simplify(1, true); e.setLayer(layer); + e.level = level; + Way way = null; if (ways != null) { BoundingBox wayFilterBbox = boundingBox.extendMeters(wayFilterDistance); GeoPoint[][] wayNodesArray = wayNodes.toArray(new GeoPoint[wayNodes.size()][]); @@ -1056,13 +1078,16 @@ public class MapDatabase implements ITileDataSource { tags.add(e.tags.get(i)); if (Selector.ALL == selector || hasName || hasHouseNr || hasRef || wayAsLabelTagFilter(tags)) { GeoPoint labelPos = labelPosition != null ? new GeoPoint(labelPosition[1] / 1E6, labelPosition[0] / 1E6) : null; - ways.add(new Way(layer, tags, wayNodesArray, labelPos, e.type)); + way = new Way(layer, tags, wayNodesArray, labelPos, e.type); + ways.add(way); } } } - if (mapDataSink != null) - mapDataSink.process(e); + if (mapDataSink != null) { + if (!deduplicate || way == null || ((TileDataSink) mapDataSink).hashWays.add(way.hashCode())) + mapDataSink.process(e); + } } } diff --git a/vtm/src/org/oscim/tiling/source/mapfile/MapReadResult.java b/vtm/src/org/oscim/tiling/source/mapfile/MapReadResult.java index 06bd183f..2278c983 100644 --- a/vtm/src/org/oscim/tiling/source/mapfile/MapReadResult.java +++ b/vtm/src/org/oscim/tiling/source/mapfile/MapReadResult.java @@ -1,7 +1,7 @@ /* * Copyright 2010, 2011, 2012, 2013 mapsforge.org * Copyright 2014-2015 Ludwig M Brinckmann - * Copyright 2017 devemux86 + * Copyright 2015-2022 devemux86 * * This program is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free Software @@ -17,13 +17,21 @@ package org.oscim.tiling.source.mapfile; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * An immutable container for the data returned from a MapDataStore. */ public class MapReadResult { + /** + * Hash codes. + */ + private final Set hashPois = new HashSet<>(); + private final Set hashWays = new HashSet<>(); + /** * True if the read area is completely covered by water, false otherwise. */ @@ -50,8 +58,8 @@ public class MapReadResult { } /** - * Adds other MapReadResult by combining pois and ways. Optionally, deduplication can - * be requested (much more expensive). + * Adds other MapReadResult by combining pois and ways. + * Optionally deduplication can be requested (more expensive). * * @param other the MapReadResult to add to this. * @param deduplicate true if check for duplicates is required. @@ -59,12 +67,12 @@ public class MapReadResult { public void add(MapReadResult other, boolean deduplicate) { if (deduplicate) { for (PointOfInterest poi : other.pointOfInterests) { - if (!this.pointOfInterests.contains(poi)) { + if (this.hashPois.add(poi.hashCode())) { this.pointOfInterests.add(poi); } } for (Way way : other.ways) { - if (!this.ways.contains(way)) { + if (this.hashWays.add(way.hashCode())) { this.ways.add(way); } } @@ -73,5 +81,4 @@ public class MapReadResult { this.ways.addAll(other.ways); } } - } diff --git a/vtm/src/org/oscim/tiling/source/mapfile/MultiMapDatabase.java b/vtm/src/org/oscim/tiling/source/mapfile/MultiMapDatabase.java index 8bc2b521..960f417f 100644 --- a/vtm/src/org/oscim/tiling/source/mapfile/MultiMapDatabase.java +++ b/vtm/src/org/oscim/tiling/source/mapfile/MultiMapDatabase.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2018 devemux86 + * Copyright 2016-2022 devemux86 * * This program is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free Software @@ -18,6 +18,7 @@ import org.oscim.core.Tile; import org.oscim.layers.tile.MapTile; import org.oscim.tiling.ITileDataSink; import org.oscim.tiling.ITileDataSource; +import org.oscim.tiling.QueryResult; import org.oscim.tiling.TileDataSink; import java.util.ArrayList; @@ -25,9 +26,15 @@ import java.util.List; public class MultiMapDatabase implements ITileDataSource { + private final boolean deduplicate; private final List mapDatabases = new ArrayList<>(); public MultiMapDatabase() { + this(false); + } + + public MultiMapDatabase(boolean deduplicate) { + this.deduplicate = deduplicate; } public boolean add(MapDatabase mapDatabase) { @@ -39,12 +46,29 @@ public class MultiMapDatabase implements ITileDataSource { @Override public void query(MapTile tile, ITileDataSink sink) { - TileDataSink dataSink = new TileDataSink(sink); - for (MapDatabase mapDatabase : mapDatabases) { - if (mapDatabase.supportsTile(tile)) - mapDatabase.query(tile, dataSink); + boolean deduplicate = this.deduplicate; + if (deduplicate) { + int n = 0; + for (MapDatabase mapDatabase : mapDatabases) { + if (mapDatabase.supportsTile(tile)) { + if (++n > 1) { + break; + } + } + } + deduplicate = n > 1; } - sink.completed(dataSink.getResult()); + + TileDataSink dataSink = new TileDataSink(sink); + for (int i = 0; i < mapDatabases.size(); i++) { + MapDatabase mapDatabase = mapDatabases.get(i); + if (mapDatabase.supportsTile(tile)) { + mapDatabase.setDeduplicate(deduplicate); + mapDatabase.setLevel(i); + mapDatabase.query(tile, dataSink); + } + } + sink.completed(QueryResult.SUCCESS); } @Override @@ -61,7 +85,7 @@ public class MultiMapDatabase implements ITileDataSource { } } - public MapReadResult readLabels(Tile tile) { + public MapReadResult readLabels(Tile tile, boolean deduplicate) { MapReadResult mapReadResult = new MapReadResult(); for (MapDatabase mdb : mapDatabases) { if (mdb.supportsTile(tile)) { @@ -71,13 +95,13 @@ public class MultiMapDatabase implements ITileDataSource { } boolean isWater = mapReadResult.isWater & result.isWater; mapReadResult.isWater = isWater; - mapReadResult.add(result, false); + mapReadResult.add(result, deduplicate); } } return mapReadResult; } - public MapReadResult readLabels(Tile upperLeft, Tile lowerRight) { + public MapReadResult readLabels(Tile upperLeft, Tile lowerRight, boolean deduplicate) { MapReadResult mapReadResult = new MapReadResult(); for (MapDatabase mdb : mapDatabases) { if (mdb.supportsTile(upperLeft)) { @@ -87,13 +111,13 @@ public class MultiMapDatabase implements ITileDataSource { } boolean isWater = mapReadResult.isWater & result.isWater; mapReadResult.isWater = isWater; - mapReadResult.add(result, false); + mapReadResult.add(result, deduplicate); } } return mapReadResult; } - public MapReadResult readMapData(Tile tile) { + public MapReadResult readMapData(Tile tile, boolean deduplicate) { MapReadResult mapReadResult = new MapReadResult(); for (MapDatabase mdb : mapDatabases) { if (mdb.supportsTile(tile)) { @@ -103,13 +127,13 @@ public class MultiMapDatabase implements ITileDataSource { } boolean isWater = mapReadResult.isWater & result.isWater; mapReadResult.isWater = isWater; - mapReadResult.add(result, false); + mapReadResult.add(result, deduplicate); } } return mapReadResult; } - public MapReadResult readMapData(Tile upperLeft, Tile lowerRight) { + public MapReadResult readMapData(Tile upperLeft, Tile lowerRight, boolean deduplicate) { MapReadResult mapReadResult = new MapReadResult(); for (MapDatabase mdb : mapDatabases) { if (mdb.supportsTile(upperLeft)) { @@ -119,13 +143,13 @@ public class MultiMapDatabase implements ITileDataSource { } boolean isWater = mapReadResult.isWater & result.isWater; mapReadResult.isWater = isWater; - mapReadResult.add(result, false); + mapReadResult.add(result, deduplicate); } } return mapReadResult; } - public MapReadResult readPoiData(Tile tile) { + public MapReadResult readPoiData(Tile tile, boolean deduplicate) { MapReadResult mapReadResult = new MapReadResult(); for (MapDatabase mdb : mapDatabases) { if (mdb.supportsTile(tile)) { @@ -135,13 +159,13 @@ public class MultiMapDatabase implements ITileDataSource { } boolean isWater = mapReadResult.isWater & result.isWater; mapReadResult.isWater = isWater; - mapReadResult.add(result, false); + mapReadResult.add(result, deduplicate); } } return mapReadResult; } - public MapReadResult readPoiData(Tile upperLeft, Tile lowerRight) { + public MapReadResult readPoiData(Tile upperLeft, Tile lowerRight, boolean deduplicate) { MapReadResult mapReadResult = new MapReadResult(); for (MapDatabase mdb : mapDatabases) { if (mdb.supportsTile(upperLeft)) { @@ -151,7 +175,7 @@ public class MultiMapDatabase implements ITileDataSource { } boolean isWater = mapReadResult.isWater & result.isWater; mapReadResult.isWater = isWater; - mapReadResult.add(result, false); + mapReadResult.add(result, deduplicate); } } return mapReadResult; diff --git a/vtm/src/org/oscim/tiling/source/mapfile/MultiMapFileTileSource.java b/vtm/src/org/oscim/tiling/source/mapfile/MultiMapFileTileSource.java index 150770fc..4bc6c0b6 100644 --- a/vtm/src/org/oscim/tiling/source/mapfile/MultiMapFileTileSource.java +++ b/vtm/src/org/oscim/tiling/source/mapfile/MultiMapFileTileSource.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2018 devemux86 + * Copyright 2016-2022 devemux86 * * This program is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free Software @@ -33,6 +33,7 @@ public class MultiMapFileTileSource extends TileSource implements IMapFileTileSo private static final Logger log = LoggerFactory.getLogger(MultiMapFileTileSource.class); + private boolean deduplicate; private final List mapFileTileSources = new ArrayList<>(); private final Map zoomsByTileSource = new HashMap<>(); @@ -72,7 +73,7 @@ public class MultiMapFileTileSource extends TileSource implements IMapFileTileSo @Override public ITileDataSource getDataSource() { - MultiMapDatabase multiMapDatabase = new MultiMapDatabase(); + MultiMapDatabase multiMapDatabase = new MultiMapDatabase(deduplicate); for (MapFileTileSource mapFileTileSource : mapFileTileSources) { try { MapDatabase mapDatabase = new MapDatabase(mapFileTileSource); @@ -112,6 +113,10 @@ public class MultiMapFileTileSource extends TileSource implements IMapFileTileSo } } + public void setDeduplicate(boolean deduplicate) { + this.deduplicate = deduplicate; + } + @Override public void setPreferredLanguage(String preferredLanguage) { for (MapFileTileSource mapFileTileSource : mapFileTileSources) {