#!/usr/bin/env python3
"""Generate src/index/languages_table.rs from GitHub Linguist's data.

Dev-only; the output is committed. Needs PyYAML.

    tools/gen-languages/gen.py [VERSION]      # default: the pinned version

Downloads languages.yml, vendor.yml and documentation.yml for VERSION and
writes the table. Re-run when bumping Linguist.
"""

import pathlib
import sys
import urllib.request

import yaml

VERSION = sys.argv[1] if len(sys.argv) > 1 else "v9.7.0"
BASE = f"https://raw.githubusercontent.com/github-linguist/linguist/{VERSION}/lib/linguist/"
OUT = pathlib.Path(__file__).resolve().parents[2] / "src/index/languages_table.rs"

# Extensions claimed by several languages, settled by hand (Linguist uses
# content heuristics we don't run). Everything else: see `pick`.
OVERRIDES = {
    ".h": "C",
    ".m": "Objective-C",
    ".pl": "Perl",
    ".pm": "Perl",
    ".t": "Perl",
    ".md": "Markdown",
    ".ts": "TypeScript",
    ".sql": "SQL",
    ".inc": "PHP",
    ".v": "Verilog",
    ".fs": "F#",
    ".r": "R",
    ".d": "D",
    ".cls": "TeX",
    ".pro": "Prolog",
    ".s": "Unix Assembly",
    ".asm": "Assembly",
    ".l": "Lex",
    ".ms": "Roff",
    ".bf": "Brainfuck",
    ".cs": "C#",
    ".rs": "Rust",
    ".html": "HTML",
    ".txt": "Text",
    ".json": "JSON",
    ".yml": "YAML",
    ".xml": "XML",
    ".sh": "Shell",
    ".mod": "Modula-2",
    ".gs": "JavaScript",
    ".ex": "Elixir",
    ".nl": "NewLisp",
    ".lisp": "Common Lisp",
    ".cl": "Common Lisp",
    ".tst": "Scilab",
    ".x": "Logos",
    ".b": "Brainfuck",
    ".n": "Roff",
    ".i": "SWIG",
    ".tpl": "Smarty",
    ".sc": "Scala",
    ".st": "Smalltalk",
    ".frm": "VBA",
    ".bas": "BASIC",
    ".fx": "HLSL",
    ".mask": "Unity3D Asset",
    ".plt": "Gnuplot",
    ".ncl": "NCL",
    ".lsp": "Common Lisp",
    ".brd": "Eagle",
    ".mdx": "MDX",
    ".cake": "C#",
    ".builds": "XML",
    ".ice": "JSON",
    ".rpy": "Ren'Py",
    ".sol": "Solidity",
    ".yaml": "YAML",
    ".cfg": "INI",
    ".f": "Fortran",
    ".es": "JavaScript",
}

KIND_RANK = {"programming": 0, "markup": 1, "data": 2, "prose": 3}


def fetch(name):
    with urllib.request.urlopen(BASE + name) as res:
        return yaml.safe_load(res.read())


def rust_str(s):
    return '"' + s.replace("\\", "\\\\").replace('"', '\\"') + '"'


def main():
    languages = fetch("languages.yml")
    vendor = fetch("vendor.yml")
    documentation = fetch("documentation.yml")

    names = sorted(languages)
    index = {name: i for i, name in enumerate(names)}

    def pick(ext, candidates):
        if ext in OVERRIDES and OVERRIDES[ext] in candidates:
            return OVERRIDES[ext]
        primary = [c for c in candidates if languages[c].get("extensions", [None])[0] == ext]
        pool = primary or candidates
        return min(pool, key=lambda c: (KIND_RANK.get(languages[c]["type"], 9), c))

    by_ext = {}
    by_name = {}
    for name in names:
        lang = languages[name]
        for ext in lang.get("extensions", []):
            by_ext.setdefault(ext.lower(), []).append(name)
        for filename in lang.get("filenames", []):
            by_name.setdefault(filename, []).append(name)

    extensions = sorted((ext, index[pick(ext, c)]) for ext, c in by_ext.items())
    filenames = sorted(
        (f, index[min(c, key=lambda n: (KIND_RANK.get(languages[n]["type"], 9), n))])
        for f, c in by_name.items()
    )

    out = []
    w = out.append
    w(f"// @generated by tools/gen-languages from GitHub Linguist {VERSION}. Do not edit.\n")
    w("// Linguist is MIT licensed: https://github.com/github-linguist/linguist\n\n")
    w("use super::languages::{Kind, Language};\n\n")
    w(f"pub const LINGUIST_VERSION: &str = {rust_str(VERSION)};\n\n")
    w("/// Every language; `group` folds a language into its parent for statistics.\n")
    w("pub static LANGUAGES: &[Language] = &[\n")
    for name in names:
        lang = languages[name]
        kind = lang["type"].capitalize()
        group = lang.get("group")
        group = f"Some({index[group]})" if group in index else "None"
        w(f"    Language {{ name: {rust_str(name)}, kind: Kind::{kind}, group: {group} }},\n")
    w("];\n\n")
    w("/// Lowercase extensions (with the dot), sorted, to an index into `LANGUAGES`.\n")
    w("pub static EXTENSIONS: &[(&str, u16)] = &[\n")
    for ext, i in extensions:
        w(f"    ({rust_str(ext)}, {i}),\n")
    w("];\n\n")
    w("/// Exact file names, sorted, to an index into `LANGUAGES`.\n")
    w("pub static FILENAMES: &[(&str, u16)] = &[\n")
    for f, i in filenames:
        w(f"    ({rust_str(f)}, {i}),\n")
    w("];\n\n")
    for title, const, patterns in [
        ("Paths of vendored code (regular expressions).", "VENDOR", vendor),
        ("Paths of documentation (regular expressions).", "DOCUMENTATION", documentation),
    ]:
        w(f"/// {title}\n")
        w(f"pub static {const}: &[&str] = &[\n")
        for p in patterns:
            w(f"    {rust_str(p)},\n")
        w("];\n\n")
    OUT.write_text("".join(out).rstrip() + "\n")
    print(f"wrote {OUT} ({len(names)} languages, {len(extensions)} extensions, "
          f"{len(filenames)} file names)")


if __name__ == "__main__":
    main()