From 6f8c9258b88fa48949096f7fac03325f8041fb40 Mon Sep 17 00:00:00 2001 From: Conrad Schulz Date: Sun, 31 May 2026 06:50:56 +0000 Subject: [PATCH] feat: global data download scripts + port 9982 - scripts/download-data.sh: OSM planet (Planetiler) + satellite orchestrator - scripts/download-satellite.py: Sentinel-2 cloudless zoom 0-10 -> MBTiles resumable, 8 threads parallel, direct SQLite WAL write - docker-compose: port 9982:3000 --- docker-compose.yml | 2 +- scripts/download-data.sh | 161 +++++++++------------------------- scripts/download-satellite.py | 116 ++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 119 deletions(-) mode change 100644 => 100755 scripts/download-data.sh create mode 100755 scripts/download-satellite.py diff --git a/docker-compose.yml b/docker-compose.yml index 1f5220b..404df9b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: image: ghcr.io/maplibre/martin:v1.10.1 container_name: rd13_martin ports: - - "3000:3000" + - "9982:3000" volumes: - ./data:/data - ./config/martin.yaml:/config/martin.yaml:ro diff --git a/scripts/download-data.sh b/scripts/download-data.sh old mode 100644 new mode 100755 index 409da66..7673398 --- a/scripts/download-data.sh +++ b/scripts/download-data.sh @@ -1,150 +1,75 @@ #!/usr/bin/env bash # ============================================================================= -# download-data.sh – Kartendaten für den rd13 Tile Server herunterladen +# download-data.sh -- OSM Planet + Satellit herunterladen # # Verwendung: -# ./scripts/download-data.sh [region] +# ./scripts/download-data.sh [osm|satellite|all] # -# Regionen: -# europe-dach – Deutschland, Österreich, Schweiz (Standard) -# germany – nur Deutschland -# planet – gesamter Planet (groß!) -# satellite – Sentinel-2 cloudless Satellitenkacheln (niedrig/mittel Zoom) +# Optionen: +# osm -- OSM Planet via Planetiler (min. 8 GB RAM, mehrere Std.) +# satellite -- Sentinel-2 cloudless Zoom 0-10 (ca. 10 GB, ca. 3-8 Std.) +# all -- beides sequentiell (Standard) +# +# Umgebungsvariablen: +# PLANETILER_RAM -- Java Heap fuer Planetiler (Standard: 8g) +# SAT_MAX_ZOOM -- Max Zoom Satellit (Standard: 10) +# SAT_THREADS -- Parallele Downloads Satellit (Standard: 8) # ============================================================================= set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -DATA_DIR="$SCRIPT_DIR/../data" -mkdir -p "$DATA_DIR" +DATA_DIR="$(realpath "$SCRIPT_DIR/../data")" +LOG_DIR="$(realpath "$SCRIPT_DIR/../logs")" +mkdir -p "$DATA_DIR" "$LOG_DIR" -REGION="${1:-europe-dach}" +MODE="${1:-all}" +PLANETILER_RAM="${PLANETILER_RAM:-8g}" -# ---- OSM-Vektorkacheln via Planetiler ------------------------------------ download_osm() { - local region="$1" local jar="$DATA_DIR/planetiler.jar" - - # Planetiler herunterladen falls nicht vorhanden + local out="$DATA_DIR/osm.mbtiles" + echo "[OSM] Ziel: $out RAM: -Xmx${PLANETILER_RAM}" if [[ ! -f "$jar" ]]; then - echo "[OSM] Lade Planetiler herunter..." - curl -L -o "$jar" \ - "https://github.com/onthegomap/planetiler/releases/latest/download/planetiler.jar" + echo "[OSM] Lade Planetiler JAR..." + curl -L --progress-bar -o "$jar" "https://github.com/onthegomap/planetiler/releases/latest/download/planetiler.jar" fi - - case "$region" in - germany) - AREA="germany" - DOWNLOAD="https://download.geofabrik.de/europe/germany-latest.osm.pbf" - ;; - europe-dach) - # DACH als Extrakt - AREA="dach" - DOWNLOAD="https://download.geofabrik.de/europe/dach-latest.osm.pbf" - ;; - planet) - AREA="planet" - DOWNLOAD="" # Planetiler lädt selbst herunter - ;; - *) - echo "Unbekannte Region: $region"; exit 1 ;; - esac - - echo "[OSM] Erzeuge osm.mbtiles für Region: $region" - if [[ "$region" == "planet" ]]; then - java -Xmx8g -jar "$jar" \ - --download \ - --output="$DATA_DIR/osm.mbtiles" - else - # OSM-PBF herunterladen - local pbf="$DATA_DIR/${AREA}.osm.pbf" - if [[ ! -f "$pbf" ]]; then - echo "[OSM] Lade PBF-Datei: $DOWNLOAD" - curl -L -o "$pbf" "$DOWNLOAD" - fi - java -Xmx4g -jar "$jar" \ - --area="$AREA" \ - --osm-path="$pbf" \ - --output="$DATA_DIR/osm.mbtiles" - fi - echo "[OSM] Fertig: $DATA_DIR/osm.mbtiles" + java "-Xmx${PLANETILER_RAM}" -jar "$jar" --download --output="$out" --force 2>&1 | tee "$LOG_DIR/osm-planetiler.log" + echo "[OSM] Fertig: $out" } -# ---- Satellit: Sentinel-2 cloudless (OpenMapTiles / maptiler) ------------ download_satellite() { - echo "" - echo "[Satellit] Hinweis:" - echo " Kostenlose Satelliten-MBTiles sind nur für niedrige Zoomstufen (0-10)" - echo " verfügbar. Für höhere Auflösung gibt es zwei Optionen:" - echo "" - echo " 1) NASA GIBS (kostenlos, TMS/WMTS, kein Download nötig):" - echo " https://gibs.earthdata.nasa.gov/wmts/" - echo " → Einfach in config.json als 'tilejson'-Source eintragen." - echo "" - echo " 2) Sentinel-2 cloudless (maptiler.com, kostenloser Account für self-hosted):" - echo " https://www.maptiler.com/data/satellite-mediumres/" - echo " → MBTiles manuell herunterladen und als data/satellite.mbtiles ablegen." - echo "" - echo " 3) Eigene GeoTIFFs → MBTiles mit gdal2tiles:" - echo " gdal2tiles.py --zoom=0-14 input.tif data/satellite/" - echo " mb-util data/satellite/ data/satellite.mbtiles" - echo "" + local out="$DATA_DIR/satellite.mbtiles" + echo "[Sat] Ziel: $out MaxZoom: ${SAT_MAX_ZOOM:-10} Threads: ${SAT_THREADS:-8}" + python3 "$SCRIPT_DIR/download-satellite.py" "$out" 2>&1 | tee "$LOG_DIR/satellite.log" + echo "[Sat] Fertig: $out" } -# ---- Fonts & Sprites für OSM Bright Style -------------------------------- download_assets() { local fonts_dir="$DATA_DIR/fonts" - local sprites_dir="$DATA_DIR/sprites" local styles_dir="$DATA_DIR/styles" - - mkdir -p "$fonts_dir" "$sprites_dir" "$styles_dir" - - if [[ ! -d "$fonts_dir/Open Sans Regular" ]]; then - echo "[Assets] Lade OpenMapTiles Fonts herunter..." + mkdir -p "$fonts_dir" "$styles_dir" + if [[ ! "$(ls -A "$fonts_dir" 2>/dev/null)" ]]; then + echo "[Assets] Lade Fonts..." TMP=$(mktemp -d) - curl -L -o "$TMP/fonts.zip" \ - "https://github.com/openmaptiles/fonts/releases/latest/download/v3.0.zip" \ - || { echo "[Assets] Font-Download fehlgeschlagen – bitte manuell von"; \ - echo " https://github.com/openmaptiles/fonts/releases herunterladen"; \ - rm -rf "$TMP"; } - if [[ -f "$TMP/fonts.zip" ]]; then - unzip -q "$TMP/fonts.zip" -d "$fonts_dir" - rm -rf "$TMP" - echo "[Assets] Fonts installiert." - fi - else - echo "[Assets] Fonts bereits vorhanden." + curl -L --progress-bar -o "$TMP/fonts.zip" "https://github.com/openmaptiles/fonts/releases/latest/download/v3.0.zip" && unzip -q "$TMP/fonts.zip" -d "$fonts_dir" && echo "[Assets] Fonts installiert." || echo "[Assets] WARNUNG: Font-Download fehlgeschlagen." + rm -rf "$TMP" fi - if [[ ! -d "$styles_dir/osm-bright" ]]; then - echo "[Assets] Lade OSM Bright GL Style herunter..." + echo "[Assets] Lade OSM Bright Style..." TMP=$(mktemp -d) - curl -L -o "$TMP/style.zip" \ - "https://github.com/openmaptiles/osm-bright-gl-style/releases/latest/download/v1.9.zip" \ - || { echo "[Assets] Style-Download fehlgeschlagen."; rm -rf "$TMP"; } - if [[ -f "$TMP/style.zip" ]]; then - unzip -q "$TMP/style.zip" -d "$TMP/extracted" - mv "$TMP/extracted/"*/ "$styles_dir/osm-bright" - rm -rf "$TMP" - echo "[Assets] OSM Bright Style installiert." - fi - else - echo "[Assets] OSM Bright Style bereits vorhanden." + curl -L --progress-bar -o "$TMP/style.zip" "https://github.com/openmaptiles/osm-bright-gl-style/releases/latest/download/v1.9.zip" && unzip -q "$TMP/style.zip" -d "$TMP/x" && mv "$TMP/x/"*/ "$styles_dir/osm-bright" && echo "[Assets] Style installiert." || echo "[Assets] WARNUNG: Style-Download fehlgeschlagen." + rm -rf "$TMP" fi } -# ---- Hauptlogik ---------------------------------------------------------- -echo "=== rd13 Tile Server – Daten-Download ===" -echo "Region: $REGION" -echo "" +echo "=== rd13 Tile Server -- Daten-Download === Modus: $MODE Start: $(date)" -if [[ "$REGION" == "satellite" ]]; then - download_satellite -else - download_osm "$REGION" - download_assets - download_satellite -fi +case "$MODE" in + osm) download_assets; download_osm ;; + satellite) download_satellite ;; + all) download_assets; download_osm; download_satellite ;; + *) echo "Unbekannter Modus: $MODE (osm|satellite|all)"; exit 1 ;; +esac -echo "" -echo "=== Abgeschlossen ===" -echo "Starte den Server mit: docker compose up -d" +echo "=== Abgeschlossen: $(date) ===" +echo " docker compose up -d && curl http://localhost:9982/catalog" diff --git a/scripts/download-satellite.py b/scripts/download-satellite.py new file mode 100755 index 0000000..c2a6859 --- /dev/null +++ b/scripts/download-satellite.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# Sentinel-2 cloudless tiles zoom 0-10 -> MBTiles (resume-capable) +# Quelle: EOX s2cloudless-2021 CC BY 4.0 https://s2maps.eu +import os, sys, sqlite3, time, threading, urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +MAX_ZOOM = int(os.environ.get("SAT_MAX_ZOOM", "10")) +THREADS = int(os.environ.get("SAT_THREADS", "8")) +OUTPUT = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/data/satellite.mbtiles") +TILE_URL = ("https://tiles.maps.eox.at/wmts/1.0.0/" + "s2cloudless-2021_3857/default/GoogleMapsCompatible/{z}/{y}/{x}.jpg") +HEADERS = {"User-Agent": "rd13-tileserver/1.0 (self-hosted)", "Referer": "https://s2maps.eu"} +_lock = threading.Lock() +stats = {"ok": 0, "skip": 0, "err": 0} +pending = [] + + +def init_db(path): + path.parent.mkdir(parents=True, exist_ok=True) + c = sqlite3.connect(str(path), check_same_thread=False) + c.execute("PRAGMA journal_mode=WAL") + c.execute("PRAGMA synchronous=NORMAL") + c.execute("CREATE TABLE IF NOT EXISTS metadata(name TEXT PRIMARY KEY, value TEXT)") + c.execute("CREATE TABLE IF NOT EXISTS tiles(" + "zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB," + "PRIMARY KEY(zoom_level, tile_column, tile_row))") + c.executemany("INSERT OR REPLACE INTO metadata VALUES(?,?)", [ + ("name", "satellite"), + ("format", "jpg"), + ("type", "baselayer"), + ("description", "Sentinel-2 cloudless 2021 (EOX CC BY 4.0)"), + ("version", "1"), + ("minzoom", "0"), + ("maxzoom", str(MAX_ZOOM)), + ("bounds", "-180,-85.051129,180,85.051129"), + ("center", "0,0,2"), + ]) + c.commit() + return c + + +def fetch(url): + req = urllib.request.Request(url, headers=HEADERS) + for i in range(3): + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.read() if r.status == 200 else None + except Exception: + if i < 2: + time.sleep(2 ** i) + return None + + +def do_tile(conn, z, x, y): + y_tms = (1 << z) - 1 - y + with _lock: + if conn.execute( + "SELECT 1 FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?", + (z, x, y_tms) + ).fetchone(): + stats["skip"] += 1 + return + data = fetch(TILE_URL.format(z=z, y=y, x=x)) + with _lock: + if data: + pending.append((z, x, y_tms, data)) + if len(pending) >= 500: + conn.executemany("INSERT OR REPLACE INTO tiles VALUES(?,?,?,?)", pending) + conn.commit() + pending.clear() + stats["ok"] += 1 + else: + stats["err"] += 1 + + +def main(): + total = sum(4 ** z for z in range(MAX_ZOOM + 1)) + print("[Sat] Output: %s" % OUTPUT) + print("[Sat] Zoom: 0-%d | Threads: %d | Tiles: %d" % (MAX_ZOOM, THREADS, total)) + print("[Sat] Quelle: Sentinel-2 cloudless 2021 (EOX, CC BY 4.0)") + conn = init_db(OUTPUT) + t0 = time.time() + futs = [ + do_tile + for z in range(MAX_ZOOM + 1) + for x in range(1 << z) + for y in range(1 << z) + ] + with ThreadPoolExecutor(max_workers=THREADS) as ex: + submitted = [ex.submit(do_tile, conn, z, x, y) + for z in range(MAX_ZOOM + 1) + for x in range(1 << z) + for y in range(1 << z)] + for n, f in enumerate(as_completed(submitted), 1): + f.result() + if n % 1000 == 0 or n == total: + dt = time.time() - t0 or 0.001 + eta = (total - n) / n * dt / 3600 + print("[Sat] %5.1f%% %d/%d ok=%d skip=%d err=%d %.0ft/s ETA=%.1fh" % ( + n / total * 100, n, total, + stats["ok"], stats["skip"], stats["err"], + n / dt, eta)) + with _lock: + if pending: + conn.executemany("INSERT OR REPLACE INTO tiles VALUES(?,?,?,?)", pending) + conn.commit() + print("[Sat] Erstelle Index...") + conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx ON tiles(zoom_level, tile_column, tile_row)") + conn.execute("ANALYZE") + conn.close() + print("[Sat] Fertig %.1fh -- %s" % ((time.time() - t0) / 3600, OUTPUT)) + + +if __name__ == "__main__": + main()