131 lines
4.8 KiB
Python
Executable file
131 lines
4.8 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# Sentinel-2 cloudless tiles zoom 0-10 -> MBTiles (resume-capable)
|
|
# Quelle: EOX s2cloudless-2021 CC BY 4.0 https://s2maps.eu
|
|
import os, sys, sqlite3, time, threading, urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
MAX_ZOOM = int(os.environ.get("SAT_MAX_ZOOM", "10"))
|
|
THREADS = int(os.environ.get("SAT_THREADS", "8"))
|
|
# Schreibt in .part-Datei → Martin ignoriert diese während des Downloads
|
|
_FINAL = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/data/satellite.mbtiles")
|
|
OUTPUT = _FINAL.with_suffix(".mbtiles.part")
|
|
TILE_URL = ("https://tiles.maps.eox.at/wmts/1.0.0/"
|
|
"s2cloudless-2021_3857/default/GoogleMapsCompatible/{z}/{y}/{x}.jpg")
|
|
HEADERS = {"User-Agent": "rd13-tileserver/1.0 (self-hosted)", "Referer": "https://s2maps.eu"}
|
|
_lock = threading.Lock()
|
|
stats = {"ok": 0, "skip": 0, "err": 0}
|
|
pending = []
|
|
|
|
|
|
def init_db(path):
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
c = sqlite3.connect(str(path), check_same_thread=False)
|
|
c.execute("PRAGMA journal_mode=WAL")
|
|
c.execute("PRAGMA synchronous=NORMAL")
|
|
c.execute("CREATE TABLE IF NOT EXISTS metadata(name TEXT PRIMARY KEY, value TEXT)")
|
|
c.execute("CREATE TABLE IF NOT EXISTS tiles("
|
|
"zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_data BLOB,"
|
|
"PRIMARY KEY(zoom_level, tile_column, tile_row))")
|
|
c.executemany("INSERT OR REPLACE INTO metadata VALUES(?,?)", [
|
|
("name", "satellite"),
|
|
("format", "jpg"),
|
|
("type", "baselayer"),
|
|
("description", "Sentinel-2 cloudless 2021 (EOX CC BY 4.0)"),
|
|
("version", "1"),
|
|
("minzoom", "0"),
|
|
("maxzoom", str(MAX_ZOOM)),
|
|
("bounds", "-180,-85.051129,180,85.051129"),
|
|
("center", "0,0,2"),
|
|
])
|
|
c.commit()
|
|
return c
|
|
|
|
|
|
def fetch(url):
|
|
req = urllib.request.Request(url, headers=HEADERS)
|
|
for i in range(3):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
return r.read() if r.status == 200 else None
|
|
except Exception:
|
|
if i < 2:
|
|
time.sleep(2 ** i)
|
|
return None
|
|
|
|
|
|
def do_tile(conn, z, x, y):
|
|
y_tms = (1 << z) - 1 - y
|
|
with _lock:
|
|
if conn.execute(
|
|
"SELECT 1 FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?",
|
|
(z, x, y_tms)
|
|
).fetchone():
|
|
stats["skip"] += 1
|
|
return
|
|
data = fetch(TILE_URL.format(z=z, y=y, x=x))
|
|
with _lock:
|
|
if data:
|
|
pending.append((z, x, y_tms, data))
|
|
if len(pending) >= 500:
|
|
conn.executemany("INSERT OR REPLACE INTO tiles VALUES(?,?,?,?)", pending)
|
|
conn.commit()
|
|
pending.clear()
|
|
stats["ok"] += 1
|
|
else:
|
|
stats["err"] += 1
|
|
|
|
|
|
def log(msg):
|
|
print(msg, flush=True)
|
|
|
|
|
|
def main():
|
|
total = sum(4 ** z for z in range(MAX_ZOOM + 1))
|
|
log("[Sat] Output: %s" % OUTPUT)
|
|
log("[Sat] Zoom: 0-%d | Threads: %d | Tiles: %d" % (MAX_ZOOM, THREADS, total))
|
|
log("[Sat] Quelle: Sentinel-2 cloudless 2021 (EOX, CC BY 4.0)")
|
|
log("[Sat] Resume: bereits heruntergeladene Tiles werden übersprungen")
|
|
conn = init_db(OUTPUT)
|
|
t0 = time.time()
|
|
n = 0
|
|
# Bounded queue: max THREADS*4 Futures gleichzeitig im Speicher
|
|
# verhindert OOM bei 5.5 Mio Tasks
|
|
sem = threading.Semaphore(THREADS * 4)
|
|
|
|
def submit_tile(ex, z, x, y):
|
|
sem.acquire()
|
|
fut = ex.submit(do_tile, conn, z, x, y)
|
|
fut.add_done_callback(lambda _: sem.release())
|
|
return fut
|
|
|
|
with ThreadPoolExecutor(max_workers=THREADS) as ex:
|
|
futs = []
|
|
for z in range(MAX_ZOOM + 1):
|
|
for x in range(1 << z):
|
|
for y in range(1 << z):
|
|
futs.append(submit_tile(ex, z, x, y))
|
|
for n, f in enumerate(as_completed(futs), 1):
|
|
f.result()
|
|
if n % 1000 == 0 or n == total:
|
|
dt = time.time() - t0 or 0.001
|
|
eta = (total - n) / n * dt / 3600
|
|
log("[Sat] %5.1f%% %d/%d ok=%d skip=%d err=%d %.0ft/s ETA=%.1fh" % (
|
|
n / total * 100, n, total,
|
|
stats["ok"], stats["skip"], stats["err"],
|
|
n / dt, eta))
|
|
with _lock:
|
|
if pending:
|
|
conn.executemany("INSERT OR REPLACE INTO tiles VALUES(?,?,?,?)", pending)
|
|
conn.commit()
|
|
log("[Sat] Erstelle Index...")
|
|
conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx ON tiles(zoom_level, tile_column, tile_row)")
|
|
conn.execute("ANALYZE")
|
|
conn.close()
|
|
log("[Sat] Benenne um: %s → %s" % (OUTPUT, _FINAL))
|
|
OUTPUT.rename(_FINAL)
|
|
log("[Sat] Fertig %.1fh -- %s" % ((time.time() - t0) / 3600, _FINAL))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|