108 lines
3.1 KiB
Python
108 lines
3.1 KiB
Python
import argparse
|
|
import sys
|
|
import urllib.request
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
from pipeline.transform.transform_poi import NAPTAN_EMOJIS, _CATEGORIES
|
|
|
|
GLYPHS_BASE = "https://protomaps.github.io/basemaps-assets/fonts"
|
|
TWEMOJI_BASE = "https://cdn.jsdelivr.net/gh/twitter/twemoji@14.0.2/assets/72x72"
|
|
|
|
# Font stacks used by @protomaps/basemaps with lang='en'
|
|
FONT_STACKS = ["Noto Sans Regular", "Noto Sans Italic", "Noto Sans Medium"]
|
|
|
|
# Fallback emoji not in any category
|
|
_FALLBACK_EMOJIS = ["📍"]
|
|
|
|
|
|
def collect_twemoji_codes() -> list[str]:
|
|
"""Derive twemoji hex codes from transform_poi categories.
|
|
|
|
Matches the frontend's emojiToTwemojiUrl() which does
|
|
emoji.codePointAt(0).toString(16).
|
|
"""
|
|
emojis: set[str] = set()
|
|
|
|
for _group, _name, emoji, _osm_keys in _CATEGORIES:
|
|
emojis.add(emoji)
|
|
|
|
for emoji in NAPTAN_EMOJIS.values():
|
|
emojis.add(emoji)
|
|
|
|
for emoji in _FALLBACK_EMOJIS:
|
|
emojis.add(emoji)
|
|
|
|
# First codepoint hex, matching frontend logic
|
|
return sorted({f"{ord(e[0]):x}" for e in emojis})
|
|
|
|
|
|
def download_file(url: str, dest: Path) -> tuple[bool, str]:
|
|
"""Download a single file. Returns (success, url)."""
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
urllib.request.urlretrieve(url, dest)
|
|
return True, url
|
|
except urllib.error.HTTPError as e:
|
|
print(f" {e.code} {url}", file=sys.stderr)
|
|
return False, url
|
|
except Exception as e:
|
|
print(f" ERROR {url}: {e}", file=sys.stderr)
|
|
return False, url
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
required=True,
|
|
help="Output directory",
|
|
)
|
|
args = parser.parse_args()
|
|
out: Path = args.output
|
|
|
|
twemoji_codes = collect_twemoji_codes()
|
|
|
|
# Build download list
|
|
tasks: list[tuple[str, Path]] = []
|
|
|
|
# Font glyphs: 256 range files per font stack
|
|
for font in FONT_STACKS:
|
|
font_encoded = font.replace(" ", "%20")
|
|
font_dir = out / "fonts" / font
|
|
for start in range(0, 65536, 256):
|
|
end = start + 255
|
|
name = f"{start}-{end}.pbf"
|
|
url = f"{GLYPHS_BASE}/{font_encoded}/{name}"
|
|
tasks.append((url, font_dir / name))
|
|
|
|
# Twemoji PNGs
|
|
twemoji_dir = out / "twemoji"
|
|
for code in twemoji_codes:
|
|
url = f"{TWEMOJI_BASE}/{code}.png"
|
|
tasks.append((url, twemoji_dir / f"{code}.png"))
|
|
|
|
# Skip already-downloaded files
|
|
remaining = [(url, dest) for url, dest in tasks]
|
|
|
|
print(f"Downloading {len(remaining)} assets")
|
|
|
|
ok = 0
|
|
fail = 0
|
|
with ThreadPoolExecutor(max_workers=20) as pool:
|
|
futures = {
|
|
pool.submit(download_file, url, dest): url for url, dest in remaining
|
|
}
|
|
for future in as_completed(futures):
|
|
success, url = future.result()
|
|
if success:
|
|
ok += 1
|
|
else:
|
|
fail += 1
|
|
|
|
print(f"Done: {ok} downloaded, {fail} failed")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|