149 lines
5.2 KiB
Python
Executable file
149 lines
5.2 KiB
Python
Executable file
#!/usr/bin/env python3
|
||
"""Pre-fetch satellite (MODIS/VIIRS TrueColor) tiles for key regions.
|
||
|
||
Downloads directly from NASA GIBS with concurrent workers and local caching.
|
||
Regions: Southern California, GCC, Iran, Israel
|
||
"""
|
||
|
||
import math
|
||
import requests
|
||
from pathlib import Path
|
||
import sys
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from threading import Lock
|
||
|
||
CACHE_DIR = Path("/home/raider1984/synq-data/pmtiles/tiles/satellite")
|
||
TILE_URL = "https://gibs.earthdata.nasa.gov/wmts/epsg3857/best/VIIRS_SNPP_CorrectedReflectance_TrueColor/default/GoogleMapsCompatible_Level9/{z}/{x}/{y}.jpeg"
|
||
MAX_ZOOM = 9
|
||
|
||
REGIONS = {
|
||
"southern_california": (32.0, 35.5, -121.0, -114.0),
|
||
"gcc": (16.0, 32.5, 47.0, 57.0),
|
||
"iran": (25.0, 40.0, 44.0, 63.5),
|
||
"israel": (29.5, 34.0, 34.0, 36.5),
|
||
}
|
||
|
||
ZOOM_LEVELS = [5, 6, 7, 8, 9]
|
||
MAX_WORKERS = 16
|
||
SESSION = requests.Session()
|
||
SESSION.headers.update({"User-Agent": "Synq-Intel-TilePrefetch/1.0"})
|
||
|
||
stats_lock = Lock()
|
||
|
||
|
||
def latlng_to_tile(lat, lng, zoom):
|
||
n = 2 ** zoom
|
||
x = int((lng + 180.0) / 360.0 * n)
|
||
lat_rad = math.radians(lat)
|
||
y = int((1.0 - math.log(math.tan(lat_rad) + 1.0 / math.cos(lat_rad)) / math.pi) / 2.0 * n)
|
||
return x, y
|
||
|
||
|
||
def get_tile_range(lat_min, lat_max, lng_min, lng_max, zoom):
|
||
x1, y1 = latlng_to_tile(lat_max, lng_min, zoom)
|
||
x2, y2 = latlng_to_tile(lat_min, lng_max, zoom)
|
||
return min(x1, x2), max(x1, x2), min(y1, y2), max(y1, y2)
|
||
|
||
|
||
def fetch_tile(z, x, y):
|
||
cache_path = CACHE_DIR / str(z) / str(x) / f"{y}.jpg"
|
||
if cache_path.exists():
|
||
return True, "cached", z
|
||
|
||
url = TILE_URL.format(z=z, x=x, y=y)
|
||
try:
|
||
r = SESSION.get(url, timeout=20)
|
||
if r.status_code == 200:
|
||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with open(cache_path, "wb") as f:
|
||
f.write(r.content)
|
||
return True, "downloaded", z
|
||
else:
|
||
return False, f"http_{r.status_code}", z
|
||
except Exception as e:
|
||
return False, str(e)[:50], z
|
||
|
||
|
||
def prefetch_region(name, lat_min, lat_max, lng_min, lng_max, zoom_levels):
|
||
print(f"\n📍 {name.upper()}")
|
||
print(f" Bounds: lat {lat_min}–{lat_max}, lng {lng_min}–{lng_max}")
|
||
sys.stdout.flush()
|
||
|
||
all_tiles = []
|
||
for z in zoom_levels:
|
||
if z > MAX_ZOOM:
|
||
continue
|
||
x_min, x_max, y_min, y_max = get_tile_range(lat_min, lat_max, lng_min, lng_max, z)
|
||
max_tile = (2 ** z) - 1
|
||
x_min = max(0, x_min)
|
||
x_max = min(max_tile, x_max)
|
||
y_min = max(0, y_min)
|
||
y_max = min(max_tile, y_max)
|
||
for x in range(x_min, x_max + 1):
|
||
for y in range(y_min, y_max + 1):
|
||
all_tiles.append((z, x, y))
|
||
|
||
zoom_stats = {z: {"total": 0, "downloaded": 0, "cached": 0, "errors": 0} for z in zoom_levels if z <= MAX_ZOOM}
|
||
total_tiles = len(all_tiles)
|
||
completed = 0
|
||
downloaded = 0
|
||
cached = 0
|
||
errors = 0
|
||
|
||
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
|
||
future_to_tile = {executor.submit(fetch_tile, z, x, y): (z, x, y) for z, x, y in all_tiles}
|
||
for future in as_completed(future_to_tile):
|
||
success, status, z = future.result()
|
||
with stats_lock:
|
||
completed += 1
|
||
zoom_stats[z]["total"] += 1
|
||
if success:
|
||
if status == "downloaded":
|
||
downloaded += 1
|
||
zoom_stats[z]["downloaded"] += 1
|
||
else:
|
||
cached += 1
|
||
zoom_stats[z]["cached"] += 1
|
||
else:
|
||
errors += 1
|
||
zoom_stats[z]["errors"] += 1
|
||
|
||
if completed % 100 == 0 or completed == total_tiles:
|
||
pct = completed * 100 // total_tiles if total_tiles > 0 else 0
|
||
print(f"\r Progress: {completed}/{total_tiles} ({pct}%) | DL:{downloaded} | Cache:{cached} | Err:{errors}", end="")
|
||
sys.stdout.flush()
|
||
|
||
print()
|
||
for z in zoom_levels:
|
||
if z > MAX_ZOOM:
|
||
continue
|
||
st = zoom_stats[z]
|
||
if st["total"] > 0:
|
||
print(f" Zoom {z:2d}: {st['total']:5d} tiles | {st['downloaded']:5d} downloaded | {st['cached']:5d} cached | {st['errors']:3d} errors")
|
||
|
||
print(f" TOTAL: {total_tiles} tiles | {downloaded} downloaded | {cached} cached | {errors} errors")
|
||
return total_tiles, downloaded, cached, errors
|
||
|
||
|
||
if __name__ == "__main__":
|
||
print("=" * 70)
|
||
print("NASA GIBS SATELLITE TILE PREFETCH (VIIRS SNPP TrueColor)")
|
||
print(f"Workers: {MAX_WORKERS} | Zooms: {ZOOM_LEVELS} | MaxZoom: {MAX_ZOOM}")
|
||
print("=" * 70)
|
||
sys.stdout.flush()
|
||
|
||
grand_total = [0, 0, 0, 0]
|
||
|
||
for name, (lat_min, lat_max, lng_min, lng_max) in REGIONS.items():
|
||
counts = prefetch_region(name, lat_min, lat_max, lng_min, lng_max, ZOOM_LEVELS)
|
||
for i in range(4):
|
||
grand_total[i] += counts[i]
|
||
|
||
print("\n" + "=" * 70)
|
||
print("GRAND TOTAL")
|
||
print(f" {grand_total[0]} tiles checked")
|
||
print(f" {grand_total[1]} downloaded")
|
||
print(f" {grand_total[2]} already cached")
|
||
print(f" {grand_total[3]} errors")
|
||
print(f"\nCache location: {CACHE_DIR}")
|
||
print("=" * 70)
|