feat: global data download scripts + port 9982
- scripts/download-data.sh: OSM planet (Planetiler) + satellite orchestrator - scripts/download-satellite.py: Sentinel-2 cloudless zoom 0-10 -> MBTiles resumable, 8 threads parallel, direct SQLite WAL write - docker-compose: port 9982:3000
This commit is contained in:
parent
cd73f91021
commit
6f8c9258b8
3 changed files with 160 additions and 119 deletions
|
|
@ -3,7 +3,7 @@ services:
|
|||
image: ghcr.io/maplibre/martin:v1.10.1
|
||||
container_name: rd13_martin
|
||||
ports:
|
||||
- "3000:3000"
|
||||
- "9982:3000"
|
||||
volumes:
|
||||
- ./data:/data
|
||||
- ./config/martin.yaml:/config/martin.yaml:ro
|
||||
|
|
|
|||
157
scripts/download-data.sh
Normal file → Executable file
157
scripts/download-data.sh
Normal file → Executable file
|
|
@ -1,150 +1,75 @@
|
|||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# download-data.sh – Kartendaten für den rd13 Tile Server herunterladen
|
||||
# download-data.sh -- OSM Planet + Satellit herunterladen
|
||||
#
|
||||
# Verwendung:
|
||||
# ./scripts/download-data.sh [region]
|
||||
# ./scripts/download-data.sh [osm|satellite|all]
|
||||
#
|
||||
# Regionen:
|
||||
# europe-dach – Deutschland, Österreich, Schweiz (Standard)
|
||||
# germany – nur Deutschland
|
||||
# planet – gesamter Planet (groß!)
|
||||
# satellite – Sentinel-2 cloudless Satellitenkacheln (niedrig/mittel Zoom)
|
||||
# Optionen:
|
||||
# osm -- OSM Planet via Planetiler (min. 8 GB RAM, mehrere Std.)
|
||||
# satellite -- Sentinel-2 cloudless Zoom 0-10 (ca. 10 GB, ca. 3-8 Std.)
|
||||
# all -- beides sequentiell (Standard)
|
||||
#
|
||||
# Umgebungsvariablen:
|
||||
# PLANETILER_RAM -- Java Heap fuer Planetiler (Standard: 8g)
|
||||
# SAT_MAX_ZOOM -- Max Zoom Satellit (Standard: 10)
|
||||
# SAT_THREADS -- Parallele Downloads Satellit (Standard: 8)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
DATA_DIR="$SCRIPT_DIR/../data"
|
||||
mkdir -p "$DATA_DIR"
|
||||
DATA_DIR="$(realpath "$SCRIPT_DIR/../data")"
|
||||
LOG_DIR="$(realpath "$SCRIPT_DIR/../logs")"
|
||||
mkdir -p "$DATA_DIR" "$LOG_DIR"
|
||||
|
||||
REGION="${1:-europe-dach}"
|
||||
MODE="${1:-all}"
|
||||
PLANETILER_RAM="${PLANETILER_RAM:-8g}"
|
||||
|
||||
# ---- OSM-Vektorkacheln via Planetiler ------------------------------------
|
||||
download_osm() {
|
||||
local region="$1"
|
||||
local jar="$DATA_DIR/planetiler.jar"
|
||||
|
||||
# Planetiler herunterladen falls nicht vorhanden
|
||||
local out="$DATA_DIR/osm.mbtiles"
|
||||
echo "[OSM] Ziel: $out RAM: -Xmx${PLANETILER_RAM}"
|
||||
if [[ ! -f "$jar" ]]; then
|
||||
echo "[OSM] Lade Planetiler herunter..."
|
||||
curl -L -o "$jar" \
|
||||
"https://github.com/onthegomap/planetiler/releases/latest/download/planetiler.jar"
|
||||
echo "[OSM] Lade Planetiler JAR..."
|
||||
curl -L --progress-bar -o "$jar" "https://github.com/onthegomap/planetiler/releases/latest/download/planetiler.jar"
|
||||
fi
|
||||
|
||||
case "$region" in
|
||||
germany)
|
||||
AREA="germany"
|
||||
DOWNLOAD="https://download.geofabrik.de/europe/germany-latest.osm.pbf"
|
||||
;;
|
||||
europe-dach)
|
||||
# DACH als Extrakt
|
||||
AREA="dach"
|
||||
DOWNLOAD="https://download.geofabrik.de/europe/dach-latest.osm.pbf"
|
||||
;;
|
||||
planet)
|
||||
AREA="planet"
|
||||
DOWNLOAD="" # Planetiler lädt selbst herunter
|
||||
;;
|
||||
*)
|
||||
echo "Unbekannte Region: $region"; exit 1 ;;
|
||||
esac
|
||||
|
||||
echo "[OSM] Erzeuge osm.mbtiles für Region: $region"
|
||||
if [[ "$region" == "planet" ]]; then
|
||||
java -Xmx8g -jar "$jar" \
|
||||
--download \
|
||||
--output="$DATA_DIR/osm.mbtiles"
|
||||
else
|
||||
# OSM-PBF herunterladen
|
||||
local pbf="$DATA_DIR/${AREA}.osm.pbf"
|
||||
if [[ ! -f "$pbf" ]]; then
|
||||
echo "[OSM] Lade PBF-Datei: $DOWNLOAD"
|
||||
curl -L -o "$pbf" "$DOWNLOAD"
|
||||
fi
|
||||
java -Xmx4g -jar "$jar" \
|
||||
--area="$AREA" \
|
||||
--osm-path="$pbf" \
|
||||
--output="$DATA_DIR/osm.mbtiles"
|
||||
fi
|
||||
echo "[OSM] Fertig: $DATA_DIR/osm.mbtiles"
|
||||
java "-Xmx${PLANETILER_RAM}" -jar "$jar" --download --output="$out" --force 2>&1 | tee "$LOG_DIR/osm-planetiler.log"
|
||||
echo "[OSM] Fertig: $out"
|
||||
}
|
||||
|
||||
# ---- Satellit: Sentinel-2 cloudless (OpenMapTiles / maptiler) ------------
|
||||
download_satellite() {
|
||||
echo ""
|
||||
echo "[Satellit] Hinweis:"
|
||||
echo " Kostenlose Satelliten-MBTiles sind nur für niedrige Zoomstufen (0-10)"
|
||||
echo " verfügbar. Für höhere Auflösung gibt es zwei Optionen:"
|
||||
echo ""
|
||||
echo " 1) NASA GIBS (kostenlos, TMS/WMTS, kein Download nötig):"
|
||||
echo " https://gibs.earthdata.nasa.gov/wmts/"
|
||||
echo " → Einfach in config.json als 'tilejson'-Source eintragen."
|
||||
echo ""
|
||||
echo " 2) Sentinel-2 cloudless (maptiler.com, kostenloser Account für self-hosted):"
|
||||
echo " https://www.maptiler.com/data/satellite-mediumres/"
|
||||
echo " → MBTiles manuell herunterladen und als data/satellite.mbtiles ablegen."
|
||||
echo ""
|
||||
echo " 3) Eigene GeoTIFFs → MBTiles mit gdal2tiles:"
|
||||
echo " gdal2tiles.py --zoom=0-14 input.tif data/satellite/"
|
||||
echo " mb-util data/satellite/ data/satellite.mbtiles"
|
||||
echo ""
|
||||
local out="$DATA_DIR/satellite.mbtiles"
|
||||
echo "[Sat] Ziel: $out MaxZoom: ${SAT_MAX_ZOOM:-10} Threads: ${SAT_THREADS:-8}"
|
||||
python3 "$SCRIPT_DIR/download-satellite.py" "$out" 2>&1 | tee "$LOG_DIR/satellite.log"
|
||||
echo "[Sat] Fertig: $out"
|
||||
}
|
||||
|
||||
# ---- Fonts & Sprites für OSM Bright Style --------------------------------
|
||||
download_assets() {
|
||||
local fonts_dir="$DATA_DIR/fonts"
|
||||
local sprites_dir="$DATA_DIR/sprites"
|
||||
local styles_dir="$DATA_DIR/styles"
|
||||
|
||||
mkdir -p "$fonts_dir" "$sprites_dir" "$styles_dir"
|
||||
|
||||
if [[ ! -d "$fonts_dir/Open Sans Regular" ]]; then
|
||||
echo "[Assets] Lade OpenMapTiles Fonts herunter..."
|
||||
mkdir -p "$fonts_dir" "$styles_dir"
|
||||
if [[ ! "$(ls -A "$fonts_dir" 2>/dev/null)" ]]; then
|
||||
echo "[Assets] Lade Fonts..."
|
||||
TMP=$(mktemp -d)
|
||||
curl -L -o "$TMP/fonts.zip" \
|
||||
"https://github.com/openmaptiles/fonts/releases/latest/download/v3.0.zip" \
|
||||
|| { echo "[Assets] Font-Download fehlgeschlagen – bitte manuell von"; \
|
||||
echo " https://github.com/openmaptiles/fonts/releases herunterladen"; \
|
||||
rm -rf "$TMP"; }
|
||||
if [[ -f "$TMP/fonts.zip" ]]; then
|
||||
unzip -q "$TMP/fonts.zip" -d "$fonts_dir"
|
||||
curl -L --progress-bar -o "$TMP/fonts.zip" "https://github.com/openmaptiles/fonts/releases/latest/download/v3.0.zip" && unzip -q "$TMP/fonts.zip" -d "$fonts_dir" && echo "[Assets] Fonts installiert." || echo "[Assets] WARNUNG: Font-Download fehlgeschlagen."
|
||||
rm -rf "$TMP"
|
||||
echo "[Assets] Fonts installiert."
|
||||
fi
|
||||
else
|
||||
echo "[Assets] Fonts bereits vorhanden."
|
||||
fi
|
||||
|
||||
if [[ ! -d "$styles_dir/osm-bright" ]]; then
|
||||
echo "[Assets] Lade OSM Bright GL Style herunter..."
|
||||
echo "[Assets] Lade OSM Bright Style..."
|
||||
TMP=$(mktemp -d)
|
||||
curl -L -o "$TMP/style.zip" \
|
||||
"https://github.com/openmaptiles/osm-bright-gl-style/releases/latest/download/v1.9.zip" \
|
||||
|| { echo "[Assets] Style-Download fehlgeschlagen."; rm -rf "$TMP"; }
|
||||
if [[ -f "$TMP/style.zip" ]]; then
|
||||
unzip -q "$TMP/style.zip" -d "$TMP/extracted"
|
||||
mv "$TMP/extracted/"*/ "$styles_dir/osm-bright"
|
||||
curl -L --progress-bar -o "$TMP/style.zip" "https://github.com/openmaptiles/osm-bright-gl-style/releases/latest/download/v1.9.zip" && unzip -q "$TMP/style.zip" -d "$TMP/x" && mv "$TMP/x/"*/ "$styles_dir/osm-bright" && echo "[Assets] Style installiert." || echo "[Assets] WARNUNG: Style-Download fehlgeschlagen."
|
||||
rm -rf "$TMP"
|
||||
echo "[Assets] OSM Bright Style installiert."
|
||||
fi
|
||||
else
|
||||
echo "[Assets] OSM Bright Style bereits vorhanden."
|
||||
fi
|
||||
}
|
||||
|
||||
# ---- Hauptlogik ----------------------------------------------------------
|
||||
echo "=== rd13 Tile Server – Daten-Download ==="
|
||||
echo "Region: $REGION"
|
||||
echo ""
|
||||
echo "=== rd13 Tile Server -- Daten-Download === Modus: $MODE Start: $(date)"
|
||||
|
||||
if [[ "$REGION" == "satellite" ]]; then
|
||||
download_satellite
|
||||
else
|
||||
download_osm "$REGION"
|
||||
download_assets
|
||||
download_satellite
|
||||
fi
|
||||
case "$MODE" in
|
||||
osm) download_assets; download_osm ;;
|
||||
satellite) download_satellite ;;
|
||||
all) download_assets; download_osm; download_satellite ;;
|
||||
*) echo "Unbekannter Modus: $MODE (osm|satellite|all)"; exit 1 ;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "=== Abgeschlossen ==="
|
||||
echo "Starte den Server mit: docker compose up -d"
|
||||
echo "=== Abgeschlossen: $(date) ==="
|
||||
echo " docker compose up -d && curl http://localhost:9982/catalog"
|
||||
|
|
|
|||
116
scripts/download-satellite.py
Executable file
116
scripts/download-satellite.py
Executable file
|
|
@ -0,0 +1,116 @@
|
|||
#!/usr/bin/env python3
|
||||
# Sentinel-2 cloudless tiles zoom 0-10 -> MBTiles (resume-capable)
|
||||
# Quelle: EOX s2cloudless-2021 CC BY 4.0 https://s2maps.eu
|
||||
import os, sys, sqlite3, time, threading, urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
MAX_ZOOM = int(os.environ.get("SAT_MAX_ZOOM", "10"))
|
||||
THREADS = int(os.environ.get("SAT_THREADS", "8"))
|
||||
OUTPUT = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/data/satellite.mbtiles")
|
||||
TILE_URL = ("https://tiles.maps.eox.at/wmts/1.0.0/"
|
||||
"s2cloudless-2021_3857/default/GoogleMapsCompatible/{z}/{y}/{x}.jpg")
|
||||
HEADERS = {"User-Agent": "rd13-tileserver/1.0 (self-hosted)", "Referer": "https://s2maps.eu"}
|
||||
_lock = threading.Lock()
|
||||
stats = {"ok": 0, "skip": 0, "err": 0}
|
||||
pending = []
|
||||
|
||||
|
||||
def init_db(path):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
c = sqlite3.connect(str(path), check_same_thread=False)
|
||||
c.execute("PRAGMA journal_mode=WAL")
|
||||
c.execute("PRAGMA synchronous=NORMAL")
|
||||
c.execute("CREATE TABLE IF NOT EXISTS metadata(name TEXT PRIMARY KEY, value TEXT)")
|
||||
c.execute("CREATE TABLE IF NOT EXISTS tiles("
|
||||
"zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB,"
|
||||
"PRIMARY KEY(zoom_level, tile_column, tile_row))")
|
||||
c.executemany("INSERT OR REPLACE INTO metadata VALUES(?,?)", [
|
||||
("name", "satellite"),
|
||||
("format", "jpg"),
|
||||
("type", "baselayer"),
|
||||
("description", "Sentinel-2 cloudless 2021 (EOX CC BY 4.0)"),
|
||||
("version", "1"),
|
||||
("minzoom", "0"),
|
||||
("maxzoom", str(MAX_ZOOM)),
|
||||
("bounds", "-180,-85.051129,180,85.051129"),
|
||||
("center", "0,0,2"),
|
||||
])
|
||||
c.commit()
|
||||
return c
|
||||
|
||||
|
||||
def fetch(url):
|
||||
req = urllib.request.Request(url, headers=HEADERS)
|
||||
for i in range(3):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.read() if r.status == 200 else None
|
||||
except Exception:
|
||||
if i < 2:
|
||||
time.sleep(2 ** i)
|
||||
return None
|
||||
|
||||
|
||||
def do_tile(conn, z, x, y):
|
||||
y_tms = (1 << z) - 1 - y
|
||||
with _lock:
|
||||
if conn.execute(
|
||||
"SELECT 1 FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?",
|
||||
(z, x, y_tms)
|
||||
).fetchone():
|
||||
stats["skip"] += 1
|
||||
return
|
||||
data = fetch(TILE_URL.format(z=z, y=y, x=x))
|
||||
with _lock:
|
||||
if data:
|
||||
pending.append((z, x, y_tms, data))
|
||||
if len(pending) >= 500:
|
||||
conn.executemany("INSERT OR REPLACE INTO tiles VALUES(?,?,?,?)", pending)
|
||||
conn.commit()
|
||||
pending.clear()
|
||||
stats["ok"] += 1
|
||||
else:
|
||||
stats["err"] += 1
|
||||
|
||||
|
||||
def main():
|
||||
total = sum(4 ** z for z in range(MAX_ZOOM + 1))
|
||||
print("[Sat] Output: %s" % OUTPUT)
|
||||
print("[Sat] Zoom: 0-%d | Threads: %d | Tiles: %d" % (MAX_ZOOM, THREADS, total))
|
||||
print("[Sat] Quelle: Sentinel-2 cloudless 2021 (EOX, CC BY 4.0)")
|
||||
conn = init_db(OUTPUT)
|
||||
t0 = time.time()
|
||||
futs = [
|
||||
do_tile
|
||||
for z in range(MAX_ZOOM + 1)
|
||||
for x in range(1 << z)
|
||||
for y in range(1 << z)
|
||||
]
|
||||
with ThreadPoolExecutor(max_workers=THREADS) as ex:
|
||||
submitted = [ex.submit(do_tile, conn, z, x, y)
|
||||
for z in range(MAX_ZOOM + 1)
|
||||
for x in range(1 << z)
|
||||
for y in range(1 << z)]
|
||||
for n, f in enumerate(as_completed(submitted), 1):
|
||||
f.result()
|
||||
if n % 1000 == 0 or n == total:
|
||||
dt = time.time() - t0 or 0.001
|
||||
eta = (total - n) / n * dt / 3600
|
||||
print("[Sat] %5.1f%% %d/%d ok=%d skip=%d err=%d %.0ft/s ETA=%.1fh" % (
|
||||
n / total * 100, n, total,
|
||||
stats["ok"], stats["skip"], stats["err"],
|
||||
n / dt, eta))
|
||||
with _lock:
|
||||
if pending:
|
||||
conn.executemany("INSERT OR REPLACE INTO tiles VALUES(?,?,?,?)", pending)
|
||||
conn.commit()
|
||||
print("[Sat] Erstelle Index...")
|
||||
conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx ON tiles(zoom_level, tile_column, tile_row)")
|
||||
conn.execute("ANALYZE")
|
||||
conn.close()
|
||||
print("[Sat] Fertig %.1fh -- %s" % ((time.time() - t0) / 3600, OUTPUT))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Reference in a new issue