feat: global data download scripts + port 9982

- scripts/download-data.sh: OSM planet (Planetiler) + satellite orchestrator
- scripts/download-satellite.py: Sentinel-2 cloudless zoom 0-10 -> MBTiles
  resumable, 8 threads parallel, direct SQLite WAL write
- docker-compose: port 9982:3000

docker-compose.yml

@@ -3,7 +3,7 @@ services:
     image: ghcr.io/maplibre/martin:v1.10.1
     container_name: rd13_martin
     ports:
-      - "3000:3000"
+      - "9982:3000"
     volumes:
       - ./data:/data
       - ./config/martin.yaml:/config/martin.yaml:ro

scripts/download-data.sh

@@ -1,150 +1,75 @@
 #!/usr/bin/env bash
 # =============================================================================
-# download-data.sh – Kartendaten für den rd13 Tile Server herunterladen
+# download-data.sh -- OSM Planet + Satellit herunterladen
 #
 # Verwendung:
-#   ./scripts/download-data.sh [region]
+#   ./scripts/download-data.sh [osm|satellite|all]
 #
-# Regionen:
+# Optionen:
-#   europe-dach  – Deutschland, Österreich, Schweiz (Standard)
+#   osm        -- OSM Planet via Planetiler (min. 8 GB RAM, mehrere Std.)
-#   germany      – nur Deutschland
+#   satellite  -- Sentinel-2 cloudless Zoom 0-10 (ca. 10 GB, ca. 3-8 Std.)
-#   planet       – gesamter Planet (groß!)
+#   all        -- beides sequentiell (Standard)
-#   satellite    – Sentinel-2 cloudless Satellitenkacheln (niedrig/mittel Zoom)
+#
+# Umgebungsvariablen:
+#   PLANETILER_RAM  -- Java Heap fuer Planetiler (Standard: 8g)
+#   SAT_MAX_ZOOM    -- Max Zoom Satellit (Standard: 10)
+#   SAT_THREADS     -- Parallele Downloads Satellit (Standard: 8)
 # =============================================================================
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-DATA_DIR="$SCRIPT_DIR/../data"
+DATA_DIR="$(realpath "$SCRIPT_DIR/../data")"
-mkdir -p "$DATA_DIR"
+LOG_DIR="$(realpath "$SCRIPT_DIR/../logs")"
+mkdir -p "$DATA_DIR" "$LOG_DIR"
 
-REGION="${1:-europe-dach}"
+MODE="${1:-all}"
+PLANETILER_RAM="${PLANETILER_RAM:-8g}"
 
-# ---- OSM-Vektorkacheln via Planetiler ------------------------------------
 download_osm() {
-  local region="$1"
   local jar="$DATA_DIR/planetiler.jar"
-
+  local out="$DATA_DIR/osm.mbtiles"
-  # Planetiler herunterladen falls nicht vorhanden
+  echo "[OSM] Ziel: $out  RAM: -Xmx${PLANETILER_RAM}"
   if [[ ! -f "$jar" ]]; then
-    echo "[OSM] Lade Planetiler herunter..."
+    echo "[OSM] Lade Planetiler JAR..."
-    curl -L -o "$jar" \
+    curl -L --progress-bar -o "$jar"       "https://github.com/onthegomap/planetiler/releases/latest/download/planetiler.jar"
-      "https://github.com/onthegomap/planetiler/releases/latest/download/planetiler.jar"
   fi
-
+  java "-Xmx${PLANETILER_RAM}" -jar "$jar"     --download --output="$out" --force 2>&1 | tee "$LOG_DIR/osm-planetiler.log"
-  case "$region" in
+  echo "[OSM] Fertig: $out"
-    germany)
-      AREA="germany"
-      DOWNLOAD="https://download.geofabrik.de/europe/germany-latest.osm.pbf"
-      ;;
-    europe-dach)
-      # DACH als Extrakt
-      AREA="dach"
-      DOWNLOAD="https://download.geofabrik.de/europe/dach-latest.osm.pbf"
-      ;;
-    planet)
-      AREA="planet"
-      DOWNLOAD=""  # Planetiler lädt selbst herunter
-      ;;
-    *)
-      echo "Unbekannte Region: $region"; exit 1 ;;
-  esac
-
-  echo "[OSM] Erzeuge osm.mbtiles für Region: $region"
-  if [[ "$region" == "planet" ]]; then
-    java -Xmx8g -jar "$jar" \
-      --download \
-      --output="$DATA_DIR/osm.mbtiles"
-  else
-    # OSM-PBF herunterladen
-    local pbf="$DATA_DIR/${AREA}.osm.pbf"
-    if [[ ! -f "$pbf" ]]; then
-      echo "[OSM] Lade PBF-Datei: $DOWNLOAD"
-      curl -L -o "$pbf" "$DOWNLOAD"
-    fi
-    java -Xmx4g -jar "$jar" \
-      --area="$AREA" \
-      --osm-path="$pbf" \
-      --output="$DATA_DIR/osm.mbtiles"
-  fi
-  echo "[OSM] Fertig: $DATA_DIR/osm.mbtiles"
 }
 
-# ---- Satellit: Sentinel-2 cloudless (OpenMapTiles / maptiler) ------------
 download_satellite() {
-  echo ""
+  local out="$DATA_DIR/satellite.mbtiles"
-  echo "[Satellit] Hinweis:"
+  echo "[Sat] Ziel: $out  MaxZoom: ${SAT_MAX_ZOOM:-10}  Threads: ${SAT_THREADS:-8}"
-  echo "  Kostenlose Satelliten-MBTiles sind nur für niedrige Zoomstufen (0-10)"
+  python3 "$SCRIPT_DIR/download-satellite.py" "$out" 2>&1 | tee "$LOG_DIR/satellite.log"
-  echo "  verfügbar. Für höhere Auflösung gibt es zwei Optionen:"
+  echo "[Sat] Fertig: $out"
-  echo ""
-  echo "  1) NASA GIBS (kostenlos, TMS/WMTS, kein Download nötig):"
-  echo "     https://gibs.earthdata.nasa.gov/wmts/"
-  echo "     → Einfach in config.json als 'tilejson'-Source eintragen."
-  echo ""
-  echo "  2) Sentinel-2 cloudless (maptiler.com, kostenloser Account für self-hosted):"
-  echo "     https://www.maptiler.com/data/satellite-mediumres/"
-  echo "     → MBTiles manuell herunterladen und als data/satellite.mbtiles ablegen."
-  echo ""
-  echo "  3) Eigene GeoTIFFs → MBTiles mit gdal2tiles:"
-  echo "     gdal2tiles.py --zoom=0-14 input.tif data/satellite/"
-  echo "     mb-util data/satellite/ data/satellite.mbtiles"
-  echo ""
 }
 
-# ---- Fonts & Sprites für OSM Bright Style --------------------------------
 download_assets() {
   local fonts_dir="$DATA_DIR/fonts"
-  local sprites_dir="$DATA_DIR/sprites"
   local styles_dir="$DATA_DIR/styles"
-
+  mkdir -p "$fonts_dir" "$styles_dir"
-  mkdir -p "$fonts_dir" "$sprites_dir" "$styles_dir"
+  if [[ ! "$(ls -A "$fonts_dir" 2>/dev/null)" ]]; then
-
+    echo "[Assets] Lade Fonts..."
-  if [[ ! -d "$fonts_dir/Open Sans Regular" ]]; then
-    echo "[Assets] Lade OpenMapTiles Fonts herunter..."
     TMP=$(mktemp -d)
-    curl -L -o "$TMP/fonts.zip" \
+    curl -L --progress-bar -o "$TMP/fonts.zip"       "https://github.com/openmaptiles/fonts/releases/latest/download/v3.0.zip"     && unzip -q "$TMP/fonts.zip" -d "$fonts_dir"     && echo "[Assets] Fonts installiert."     || echo "[Assets] WARNUNG: Font-Download fehlgeschlagen."
-      "https://github.com/openmaptiles/fonts/releases/latest/download/v3.0.zip" \
+    rm -rf "$TMP"
-      || { echo "[Assets] Font-Download fehlgeschlagen – bitte manuell von"; \
-           echo "  https://github.com/openmaptiles/fonts/releases herunterladen"; \
-           rm -rf "$TMP"; }
-    if [[ -f "$TMP/fonts.zip" ]]; then
-      unzip -q "$TMP/fonts.zip" -d "$fonts_dir"
-      rm -rf "$TMP"
-      echo "[Assets] Fonts installiert."
-    fi
-  else
-    echo "[Assets] Fonts bereits vorhanden."
   fi
-
   if [[ ! -d "$styles_dir/osm-bright" ]]; then
-    echo "[Assets] Lade OSM Bright GL Style herunter..."
+    echo "[Assets] Lade OSM Bright Style..."
     TMP=$(mktemp -d)
-    curl -L -o "$TMP/style.zip" \
+    curl -L --progress-bar -o "$TMP/style.zip"       "https://github.com/openmaptiles/osm-bright-gl-style/releases/latest/download/v1.9.zip"     && unzip -q "$TMP/style.zip" -d "$TMP/x"     && mv "$TMP/x/"*/ "$styles_dir/osm-bright"     && echo "[Assets] Style installiert."     || echo "[Assets] WARNUNG: Style-Download fehlgeschlagen."
-      "https://github.com/openmaptiles/osm-bright-gl-style/releases/latest/download/v1.9.zip" \
+    rm -rf "$TMP"
-      || { echo "[Assets] Style-Download fehlgeschlagen."; rm -rf "$TMP"; }
-    if [[ -f "$TMP/style.zip" ]]; then
-      unzip -q "$TMP/style.zip" -d "$TMP/extracted"
-      mv "$TMP/extracted/"*/ "$styles_dir/osm-bright"
-      rm -rf "$TMP"
-      echo "[Assets] OSM Bright Style installiert."
-    fi
-  else
-    echo "[Assets] OSM Bright Style bereits vorhanden."
   fi
 }
 
-# ---- Hauptlogik ----------------------------------------------------------
+echo "=== rd13 Tile Server -- Daten-Download === Modus: $MODE  Start: $(date)"
-echo "=== rd13 Tile Server – Daten-Download ==="
-echo "Region: $REGION"
-echo ""
 
-if [[ "$REGION" == "satellite" ]]; then
+case "$MODE" in
-  download_satellite
+  osm)       download_assets; download_osm ;;
-else
+  satellite) download_satellite ;;
-  download_osm "$REGION"
+  all)       download_assets; download_osm; download_satellite ;;
-  download_assets
+  *)         echo "Unbekannter Modus: $MODE  (osm|satellite|all)"; exit 1 ;;
-  download_satellite
+esac
-fi
 
-echo ""
+echo "=== Abgeschlossen: $(date) ==="
-echo "=== Abgeschlossen ==="
+echo "  docker compose up -d  &&  curl http://localhost:9982/catalog"
-echo "Starte den Server mit: docker compose up -d"

scripts/download-satellite.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+# Sentinel-2 cloudless tiles zoom 0-10 -> MBTiles  (resume-capable)
+# Quelle: EOX s2cloudless-2021 CC BY 4.0  https://s2maps.eu
+import os, sys, sqlite3, time, threading, urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+MAX_ZOOM = int(os.environ.get("SAT_MAX_ZOOM", "10"))
+THREADS  = int(os.environ.get("SAT_THREADS",  "8"))
+OUTPUT   = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/data/satellite.mbtiles")
+TILE_URL = ("https://tiles.maps.eox.at/wmts/1.0.0/"
+            "s2cloudless-2021_3857/default/GoogleMapsCompatible/{z}/{y}/{x}.jpg")
+HEADERS  = {"User-Agent": "rd13-tileserver/1.0 (self-hosted)", "Referer": "https://s2maps.eu"}
+_lock    = threading.Lock()
+stats    = {"ok": 0, "skip": 0, "err": 0}
+pending  = []
+
+
+def init_db(path):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    c = sqlite3.connect(str(path), check_same_thread=False)
+    c.execute("PRAGMA journal_mode=WAL")
+    c.execute("PRAGMA synchronous=NORMAL")
+    c.execute("CREATE TABLE IF NOT EXISTS metadata(name TEXT PRIMARY KEY, value TEXT)")
+    c.execute("CREATE TABLE IF NOT EXISTS tiles("
+              "zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB,"
+              "PRIMARY KEY(zoom_level, tile_column, tile_row))")
+    c.executemany("INSERT OR REPLACE INTO metadata VALUES(?,?)", [
+        ("name",        "satellite"),
+        ("format",      "jpg"),
+        ("type",        "baselayer"),
+        ("description", "Sentinel-2 cloudless 2021 (EOX CC BY 4.0)"),
+        ("version",     "1"),
+        ("minzoom",     "0"),
+        ("maxzoom",     str(MAX_ZOOM)),
+        ("bounds",      "-180,-85.051129,180,85.051129"),
+        ("center",      "0,0,2"),
+    ])
+    c.commit()
+    return c
+
+
+def fetch(url):
+    req = urllib.request.Request(url, headers=HEADERS)
+    for i in range(3):
+        try:
+            with urllib.request.urlopen(req, timeout=30) as r:
+                return r.read() if r.status == 200 else None
+        except Exception:
+            if i < 2:
+                time.sleep(2 ** i)
+    return None
+
+
+def do_tile(conn, z, x, y):
+    y_tms = (1 << z) - 1 - y
+    with _lock:
+        if conn.execute(
+            "SELECT 1 FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?",
+            (z, x, y_tms)
+        ).fetchone():
+            stats["skip"] += 1
+            return
+    data = fetch(TILE_URL.format(z=z, y=y, x=x))
+    with _lock:
+        if data:
+            pending.append((z, x, y_tms, data))
+            if len(pending) >= 500:
+                conn.executemany("INSERT OR REPLACE INTO tiles VALUES(?,?,?,?)", pending)
+                conn.commit()
+                pending.clear()
+            stats["ok"] += 1
+        else:
+            stats["err"] += 1
+
+
+def main():
+    total = sum(4 ** z for z in range(MAX_ZOOM + 1))
+    print("[Sat] Output:  %s" % OUTPUT)
+    print("[Sat] Zoom:    0-%d  |  Threads: %d  |  Tiles: %d" % (MAX_ZOOM, THREADS, total))
+    print("[Sat] Quelle:  Sentinel-2 cloudless 2021 (EOX, CC BY 4.0)")
+    conn = init_db(OUTPUT)
+    t0 = time.time()
+    futs = [
+        do_tile
+        for z in range(MAX_ZOOM + 1)
+        for x in range(1 << z)
+        for y in range(1 << z)
+    ]
+    with ThreadPoolExecutor(max_workers=THREADS) as ex:
+        submitted = [ex.submit(do_tile, conn, z, x, y)
+                     for z in range(MAX_ZOOM + 1)
+                     for x in range(1 << z)
+                     for y in range(1 << z)]
+        for n, f in enumerate(as_completed(submitted), 1):
+            f.result()
+            if n % 1000 == 0 or n == total:
+                dt  = time.time() - t0 or 0.001
+                eta = (total - n) / n * dt / 3600
+                print("[Sat] %5.1f%%  %d/%d  ok=%d skip=%d err=%d  %.0ft/s  ETA=%.1fh" % (
+                    n / total * 100, n, total,
+                    stats["ok"], stats["skip"], stats["err"],
+                    n / dt, eta))
+    with _lock:
+        if pending:
+            conn.executemany("INSERT OR REPLACE INTO tiles VALUES(?,?,?,?)", pending)
+            conn.commit()
+    print("[Sat] Erstelle Index...")
+    conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx ON tiles(zoom_level, tile_column, tile_row)")
+    conn.execute("ANALYZE")
+    conn.close()
+    print("[Sat] Fertig  %.1fh  --  %s" % ((time.time() - t0) / 3600, OUTPUT))
+
+
+if __name__ == "__main__":
+    main()