U/kostmo/gen circle conf (pytorch#17189)

Summary:
Diagram preview:
![binarysmoketests-config-dimensions](https://user-images.githubusercontent.com/261693/53040977-a0f88d00-3437-11e9-9190-796cc243e0f9.png)
Pull Request resolved: pytorch#17189

Differential Revision: D14141362

Pulled By: kostmo

fbshipit-source-id: 0625a1234d0307c6be79f17e756ddb1cc445b374

.circleci/.gitignore

@@ -0,0 +1,2 @@
+*.svg
+*.png

.circleci/binary_build_definitions.py

@@ -0,0 +1,187 @@
+from collections import OrderedDict
+
+import conf_tree
+import miniutils
+import make_build_configs
+
+
+class Conf:
+    def __init__(self, os, cuda_version, pydistro, parms, smoke=False, libtorch_variant=None):
+
+        self.os = os
+        self.cuda_version = cuda_version
+        self.pydistro = pydistro
+        self.parms = parms
+        self.smoke = smoke
+        self.libtorch_variant = libtorch_variant
+
+    def genBuildEnvParms(self):
+        return [self.pydistro] + self.parms + [make_build_configs.get_processor_arch_name(self.cuda_version)]
+
+    def genDockerImage(self):
+
+        docker_word_substitution = {
+            "manywheel": "manylinux",
+            "libtorch": "manylinux",
+        }
+
+        docker_distro_prefix = miniutils.override(self.pydistro, docker_word_substitution)
+
+        alt_docker_suffix = self.cuda_version or "80"
+        docker_distro_suffix = "" if self.pydistro == "conda" else alt_docker_suffix
+        return miniutils.quote("soumith/" + docker_distro_prefix + "-cuda" + docker_distro_suffix)
+
+    def getNamePrefix(self):
+        return "smoke" if self.smoke else "binary"
+
+    def genBuildName(self, build_or_test):
+        parts = [self.getNamePrefix(), self.os] + self.genBuildEnvParms()
+
+        if self.smoke:
+            if self.libtorch_variant:
+                parts.append(self.libtorch_variant)
+        else:
+            parts.append(build_or_test)
+
+        return "_".join(parts)
+
+    def genYamlTree(self, build_or_test):
+
+        env_dict = OrderedDict({
+            "BUILD_ENVIRONMENT": miniutils.quote(" ".join(self.genBuildEnvParms())),
+        })
+
+        if self.libtorch_variant:
+            env_dict["LIBTORCH_VARIANT"] = miniutils.quote(self.libtorch_variant)
+
+        os_word_substitution = {
+            "macos": "mac",
+        }
+
+        os_name = miniutils.override(self.os, os_word_substitution)
+
+        d = {
+            "environment": env_dict,
+            "<<": "*" + "_".join([self.getNamePrefix(), os_name, build_or_test]),
+        }
+
+        if build_or_test == "test":
+            tuples = []
+            if self.cuda_version:
+                tuples.append(("USE_CUDA_DOCKER_RUNTIME", miniutils.quote("1")))
+
+            if not (self.smoke and self.os == "macos"):
+                tuples.append(("DOCKER_IMAGE", self.genDockerImage()))
+
+            if self.smoke:
+                # TODO: Fix this discrepancy upstream
+                tuples.reverse()
+
+            for (k, v) in tuples:
+                env_dict[k] = v
+
+        else:
+            if self.os == "linux" and build_or_test != "upload":
+                d["docker"] = [{"image": self.genDockerImage()}]
+
+        if build_or_test == "test":
+            if self.cuda_version:
+                d["resource_class"] = "gpu.medium"
+
+        return d
+
+
+def gen_build_env_list(smoke):
+
+    root = make_build_configs.TopLevelNode(
+        "Builds",
+        make_build_configs.CONFIG_TREE_DATA,
+        smoke,
+    )
+
+    config_list, dot = conf_tree.dfs(root)
+
+    newlist = []
+    for c in config_list:
+        conf = Conf(
+            c.find_prop("os_name"),
+            c.find_prop("cu"),
+            c.find_prop("package_format"),
+            [c.find_prop("pyver")],
+            c.find_prop("smoke"),
+            c.find_prop("libtorch_variant")
+        )
+        newlist.append(conf)
+
+    return newlist, dot
+
+
+def add_build_entries(jobs_dict, phase, smoke):
+
+    configs, _ = gen_build_env_list(smoke)
+    for conf_options in configs:
+        jobs_dict[conf_options.genBuildName(phase)] = conf_options.genYamlTree(phase)
+
+
+def add_binary_build_specs(jobs_dict):
+    add_build_entries(jobs_dict, "build", False)
+
+
+def add_binary_build_uploads(jobs_dict):
+    add_build_entries(jobs_dict, "upload", False)
+
+
+def add_smoke_test_specs(jobs_dict):
+    add_build_entries(jobs_dict, "test", True)
+
+
+def add_binary_build_tests(jobs_dict):
+
+    def testable_binary_predicate(x):
+        return x.os == "linux" and (x.smoke or x.pydistro != "libtorch")
+
+    configs, _ = gen_build_env_list(False)
+    filtered_configs = filter(testable_binary_predicate, configs)
+
+    for conf_options in filtered_configs:
+        jobs_dict[conf_options.genBuildName("test")] = conf_options.genYamlTree("test")
+
+
+def gen_schedule_tree(cron_timing):
+    return [{
+        "schedule": {
+            "cron": miniutils.quote(cron_timing),
+            "filters": {
+                "branches": {
+                    "only": ["master"],
+                },
+            },
+        },
+    }]
+
+
+def add_jobs_and_render(jobs_dict, toplevel_key, smoke, cron_schedule):
+
+    jobs_list = []
+
+    configs, graph = gen_build_env_list(smoke)
+    for build_config in configs:
+        build_name = build_config.genBuildName("build")
+        jobs_list.append(build_name)
+
+    d = OrderedDict(
+        triggers=gen_schedule_tree(cron_schedule),
+        jobs=jobs_list,
+    )
+
+    jobs_dict[toplevel_key] = d
+
+    graph.draw(toplevel_key + "-config-dimensions.png", prog="twopi")
+
+
+def add_binary_build_jobs(jobs_dict):
+    add_jobs_and_render(jobs_dict, "binarybuilds", False, "5 5 * * *")
+
+
+def add_binary_smoke_test_jobs(jobs_dict):
+    add_jobs_and_render(jobs_dict, "binarysmoketests", True, "15 16 * * *")

.circleci/build_env_definitions.py

@@ -0,0 +1,195 @@
+from collections import OrderedDict
+
+import miniutils
+
+
+DOCKER_IMAGE_PATH_BASE = "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/"
+
+DEFAULT_DOCKER_VERSION = 282
+
+
+class DockerHide:
+    """Hides element for construction of docker path"""
+    def __init__(self, val):
+        self.val = val
+
+    def __str__(self):
+        return self.val
+
+
+class Conf:
+    def __init__(self,
+                 distro,
+                 parms,
+                 pyver=None,
+                 cuda_version=None,
+                 is_xla=False,
+                 restrict_phases=None,
+                 cuda_docker_phases=None,
+                 gpu_resource=None,
+                 docker_version_override=None):
+
+        self.distro = distro
+        self.pyver = pyver
+        self.parms = parms
+
+        self.cuda_version = cuda_version
+        self.is_xla = is_xla
+        self.restrict_phases = restrict_phases
+
+        # FIXME does the build phase ever need CUDA runtime?
+        self.cuda_docker_phases = cuda_docker_phases or []
+
+        self.gpu_resource = gpu_resource
+
+        # FIXME is this different docker version intentional?
+        self.docker_version_override = docker_version_override
+
+    def getParms(self):
+        leading = ["pytorch"]
+        if self.is_xla:
+            leading.append(DockerHide("xla"))
+
+        cuda_parms = []
+        if self.cuda_version:
+            cuda_parms.extend(["cuda" + self.cuda_version, "cudnn7"])
+        return leading + ["linux", self.distro] + cuda_parms + self.parms
+
+    # TODO: Eliminate this special casing in docker paths
+    def genDockerImagePath(self, build_or_test):
+
+        build_env_pieces = self.getParms()
+        build_env_pieces = list(filter(lambda x: type(x) is not DockerHide, build_env_pieces))
+
+        build_job_name_pieces = build_env_pieces + [build_or_test]
+
+        base_build_env_name = "-".join(build_env_pieces)
+
+        docker_version = self.docker_version_override or DEFAULT_DOCKER_VERSION
+
+        return miniutils.quote(DOCKER_IMAGE_PATH_BASE + base_build_env_name + ":" + str(docker_version))
+
+    def getBuildJobNamePieces(self, build_or_test):
+        return self.getParms() + [build_or_test]
+
+    def genBuildName(self, build_or_test):
+        return ("_".join(map(str, self.getBuildJobNamePieces(build_or_test)))).replace(".", "_")
+
+    def genYamlTree(self, build_or_test):
+
+        build_job_name_pieces = self.getBuildJobNamePieces(build_or_test)
+
+        base_build_env_name = "-".join(map(str, self.getParms()))
+        build_env_name = "-".join(map(str, build_job_name_pieces))
+
+        env_dict = {
+            "BUILD_ENVIRONMENT": build_env_name,
+            "DOCKER_IMAGE": self.genDockerImagePath(build_or_test),
+        }
+
+        if self.pyver:
+            env_dict["PYTHON_VERSION"] = miniutils.quote(self.pyver)
+
+        if build_or_test in self.cuda_docker_phases:
+            env_dict["USE_CUDA_DOCKER_RUNTIME"] = miniutils.quote("1")
+
+        d = {
+            "environment": env_dict,
+            "<<": "*" + "_".join(["pytorch", "linux", build_or_test, "defaults"]),
+        }
+
+        if build_or_test == "test":
+            resource_class = "large"
+            if self.gpu_resource:
+                resource_class = "gpu." + self.gpu_resource
+
+                if self.gpu_resource == "large":
+                    env_dict["MULTI_GPU"] = miniutils.quote("1")
+
+            d["resource_class"] = resource_class
+
+        return d
+
+
+BUILD_ENV_LIST = [
+    Conf("trusty", ["py2.7.9"]),
+    Conf("trusty", ["py2.7"]),
+    Conf("trusty", ["py3.5"]),
+    Conf("trusty", ["py3.5"]),
+    Conf("trusty", ["py3.6", "gcc4.8"]),
+    Conf("trusty", ["py3.6", "gcc5.4"]),
+    Conf("trusty", ["py3.6", "gcc5.4"], is_xla=True, docker_version_override=278),
+    Conf("trusty", ["py3.6", "gcc7"]),
+    Conf("trusty", ["pynightly"]),
+    Conf("xenial", ["py3", "clang5", "asan"], pyver="3.6"),
+    Conf("xenial",
+         ["py3"],
+         pyver="3.6",
+         cuda_version="8",
+         gpu_resource="medium",
+         cuda_docker_phases=["test"]),
+    Conf("xenial",
+         ["py3", DockerHide("multigpu")],
+         pyver="3.6",
+         cuda_version="8",
+         restrict_phases=["test"],
+         cuda_docker_phases=["build", "test"],
+         gpu_resource="large"),
+    Conf("xenial",
+         ["py3", DockerHide("NO_AVX2")],
+         pyver="3.6",
+         cuda_version="8",
+         restrict_phases=["test"],
+         cuda_docker_phases=["build", "test"],
+         gpu_resource="medium"),
+    Conf("xenial",
+         ["py3", DockerHide("NO_AVX"), DockerHide("NO_AVX2")],
+         pyver="3.6",
+         cuda_version="8",
+         restrict_phases=["test"],
+         cuda_docker_phases=["build", "test"],
+         gpu_resource="medium"),
+    Conf("xenial",
+         ["py2"],
+         pyver="2.7",
+         cuda_version="9",
+         cuda_docker_phases=["test"],
+         gpu_resource="medium"),
+    Conf("xenial",
+         ["py3"],
+         pyver="3.6",
+         cuda_version="9",
+         gpu_resource="medium",
+         cuda_docker_phases=["test"]),
+    Conf("xenial",
+         ["py3", "gcc7"],
+         pyver="3.6",
+         cuda_version="9.2",
+         gpu_resource="medium",
+         cuda_docker_phases=["test"]),
+    Conf("xenial",
+         ["py3", "gcc7"],
+         pyver="3.6",
+         cuda_version="10",
+         restrict_phases=["build"]),
+]
+
+
+def add_build_env_defs(jobs_dict):
+
+    mydict = OrderedDict()
+    for conf_options in BUILD_ENV_LIST:
+
+        def append_environment_dict(build_or_test):
+            d = conf_options.genYamlTree(build_or_test)
+            mydict[conf_options.genBuildName(build_or_test)] = d
+
+        phases = ["build", "test"]
+        if conf_options.restrict_phases:
+            phases = conf_options.restrict_phases
+
+        for phase in phases:
+            append_environment_dict(phase)
+
+    jobs_dict["version"] = 2
+    jobs_dict["jobs"] = mydict