From f11926aa8bbf4d259f05b2866d221ac7660ec9bb Mon Sep 17 00:00:00 2001 From: Ian O Connell Date: Sun, 11 Sep 2016 12:12:29 -0700 Subject: [PATCH] WIP -- have the jar creator code inlined --- scala/scala.bzl | 302 ++++++++++-------- src/java/io/bazel/rulesscala/jar/BUILD | 10 +- .../io/bazel/rulesscala/jar/JarCreator.java | 23 ++ src/java/io/bazel/rulesscala/scalac/BUILD | 1 + .../rulesscala/scalac/ScalaCInvoker.java | 80 ++++- 5 files changed, 271 insertions(+), 145 deletions(-) diff --git a/scala/scala.bzl b/scala/scala.bzl index 91658a432..70b9aec0b 100644 --- a/scala/scala.bzl +++ b/scala/scala.bzl @@ -21,35 +21,43 @@ _srcjar_filetype = FileType([".srcjar"]) # TODO is there a way to derive this from the above? _scala_srcjar_filetype = FileType([".scala", ".srcjar", ".java"]) + def _adjust_resources_path(path): - dir_1, dir_2, rel_path = path.partition("resources") - if rel_path: - return dir_1 + dir_2, rel_path - (dir_1,dir_2,rel_path) = path.partition("java") - if rel_path: - return dir_1 + dir_2, rel_path - return "", path + # Here we are looking to find out the offset of this resource inside + # any resources folder. We want to return the root to the resources folder + # and then the sub path inside it + dir_1, dir_2, rel_path = path.partition("resources") + if rel_path: + return dir_1 + dir_2, rel_path + + # The same as the above but just looking for java + (dir_1, dir_2, rel_path) = path.partition("java") + if rel_path: + return dir_1 + dir_2, rel_path + return "", path + def _add_resources_cmd(ctx, dest): - res_cmd = "" - for f in ctx.files.resources: - c_dir, res_path = _adjust_resources_path(f.path) - target_path = res_path - if res_path[0] != "/": - target_path = "/" + res_path - res_cmd += "\nmkdir -p $(dirname {out_dir}{target_path})\ncp {c_dir}{res_path} {out_dir}{target_path}".format( - out_dir=dest, - res_path=res_path, - target_path=target_path, - c_dir=c_dir) - return res_cmd + res_cmd = "" + for f in ctx.files.resources: + c_dir, res_path = _adjust_resources_path(f.path) + target_path = res_path + if res_path[0] != "/": + target_path = "/" + res_path + res_cmd += "\nmkdir -p $(dirname {out_dir}{target_path})\ncp {c_dir}{res_path} {out_dir}{target_path}".format( + out_dir=dest, + res_path=res_path, + target_path=target_path, + c_dir=c_dir) + return res_cmd + def _get_jar_path(paths): - for p in paths: - path = p.path - if path.endswith("/jar_deploy.jar"): - return path - return None + for p in paths: + path = p.path + if path.endswith("/jar_deploy.jar"): + return path + return None def _get_scalac_jar_path(paths): for p in paths: @@ -107,124 +115,140 @@ def _collect_plugin_paths(plugins): paths += [f.path for f in p.files] return paths + def _compile(ctx, _jars, dep_srcjars, buildijar): - jars = _jars - cp_resources = _add_resources_cmd(ctx, "{out}_tmp".format(out=ctx.outputs.jar.path)) - ijar_cmd = "" - if buildijar: - ijar_cmd = "\n{ijar} {out} {ijar_out}".format( - ijar=ctx.file._ijar.path, - out=ctx.outputs.jar.path, - ijar_out=ctx.outputs.ijar.path) + jars = _jars + cp_resources = _add_resources_cmd(ctx, "{out}_tmp".format(out=ctx.outputs.jar.path)) + ijar_cmd = "" + if buildijar: + ijar_cmd = "\n{ijar} {out} {ijar_out}".format( + ijar=ctx.file._ijar.path, + out=ctx.outputs.jar.path, + ijar_out=ctx.outputs.ijar.path) + + java_srcs = _java_filetype.filter(ctx.files.srcs) + sources = _scala_filetype.filter(ctx.files.srcs) + java_srcs + srcjars = _srcjar_filetype.filter(ctx.files.srcs) + all_srcjars = set(srcjars + list(dep_srcjars)) + # look for any plugins: + plugins = _collect_plugin_paths(ctx.attr.plugins) + plugin_arg = ",".join(list(plugins)) + + # Set up the args to pass to scalac because they can be too long for bash + scalac_args_file = ctx.new_file(ctx.outputs.jar, ctx.label.name + "_scalac_args") # noqa + + compiler_classpath = '{scalalib}:{scalacompiler}:{scalareflect}:{jars}'.format( # noqa + scalalib=ctx.file._scalalib.path, + scalacompiler=ctx.file._scalacompiler.path, + scalareflect=ctx.file._scalareflect.path, + jars=":".join([j.path for j in jars]), + ) - java_srcs = _java_filetype.filter(ctx.files.srcs) - sources = _scala_filetype.filter(ctx.files.srcs) + java_srcs - srcjars = _srcjar_filetype.filter(ctx.files.srcs) - all_srcjars = set(srcjars + list(dep_srcjars)) - # look for any plugins: - plugins = _collect_plugin_paths(ctx.attr.plugins) - plugin_arg = "" - if (len(plugins) > 0): - plugin_arg = " ".join(["-Xplugin:%s" % p for p in plugins]) - - # Set up the args to pass to scalac because they can be too long for bash - scalac_args_file = ctx.new_file(ctx.outputs.jar, ctx.label.name + "_scalac_args") - scalac_args = """{scala_opts} {plugin_arg} -classpath {scalalib}:{scalacompiler}:{scalareflect}:{jars} -d {out}_tmp {files}""".format( - scala_opts=" ".join(ctx.attr.scalacopts), - scalalib=ctx.file._scalalib.path, - scalacompiler=ctx.file._scalacompiler.path, - scalareflect=ctx.file._scalareflect.path, - plugin_arg = plugin_arg, - jars=":".join([j.path for j in jars]), - files=" ".join([f.path for f in sources]), - out=ctx.outputs.jar.path + # these are args that our launcher app is going to consume + # and will not be passed to scalac + pre_pargs = '{out} {manifest}'.format( + out=ctx.outputs.jar.path, # 0 + manifest=ctx.outputs.manifest.path, # 1 + ) + scalac_args = """{pre_pargs} {scala_opts} {plugin_arg} {cp} {files}""".format( # noqa + pre_pargs=pre_pargs, + scala_opts=",".join(ctx.attr.scalacopts), # 2 + plugin_arg=plugin_arg, # 3 + cp=compiler_classpath, # 4 + files=",".join([f.path for f in sources]), # 5 + ) + ctx.file_action(output=scalac_args_file, content=scalac_args) + javac_sources_cmd = "" + compile_java_srcs = len(java_srcs) != 0 + if (compile_java_srcs): + # Set up the args to pass to javac because they can be + # too long for bash + javac_args_file = ctx.new_file( + ctx.outputs.jar, + ctx.label.name + "_javac_args") + + javac_args = """{javac_opts} -classpath "{jars}:{out}_tmp" -d {out}_tmp {files}""".format( # noqa + javac_opts=" ".join(ctx.attr.javacopts), + jars=":".join([j.path for j in jars]), + files=" ".join([f.path for f in java_srcs]), + out=ctx.outputs.jar.path + ) + ctx.file_action(output=javac_args_file, content=javac_args) + javac_sources_cmd = """ + cat {javac_args} {{out}}_args/files_from_jar > {{out}}_args/java_args + {javac} {{jvm_flags}} @{{out}}_args/java_args""".format( + javac_args=javac_args_file.path, + javac=ctx.file._javac.path + ) + + srcjar_cmd = "" + if len(all_srcjars) > 0: + srcjar_cmd = "\nmkdir -p {out}_tmp_expand_srcjars\n" + for srcjar in all_srcjars: + # Note: this is double escaped because we need to do one format call + # per each srcjar, but then we are going to include this in the bigger format + # call that is done to generate the full command + + #TODO would like to be able to switch >/dev/null, -v, etc based on the user's settings + srcjar_cmd += """ + unzip -o {srcjar} -d {{out}}_tmp_expand_srcjars >/dev/null + """.format(srcjar = srcjar.path) + srcjar_cmd += """find {out}_tmp_expand_srcjars -type f -name "*.scala" > {out}_args/files_from_jar\n""" + + cmd = """ + rm -rf {out}_args + rm -rf {out}_tmp + rm -rf {out}_tmp_expand_srcjars + set -e + mkdir -p {out}_args + touch {out}_args/files_from_jar + mkdir -p {out}_tmp""" + srcjar_cmd + """ + cat {scalac_args} {out}_args/files_from_jar > {out}_args/scala_args + {java} -jar {scalac} {jvm_flags} @{out}_args/scala_args""" + javac_sources_cmd + """ + # add any resources + {cp_resources} + """ + ijar_cmd + cmd = cmd.format( + cp_resources=cp_resources, + java=ctx.file._java.path, + jvm_flags=" ".join(["-J" + flag for flag in ctx.attr.jvm_flags]), + scalac=_get_scalac_jar_path(ctx.files._scalac), + scalac_args=scalac_args_file.path, + out=ctx.outputs.jar.path, + manifest=ctx.outputs.manifest.path, + jar=_get_jar_path(ctx.files.__deploy_jar), + ijar=ctx.file._ijar.path, ) - ctx.file_action(output = scalac_args_file, content = scalac_args) - javac_sources_cmd = "" - compile_java_srcs = len(java_srcs) != 0 - if (compile_java_srcs): - # Set up the args to pass to javac because they can be too long for bash - javac_args_file = ctx.new_file(ctx.outputs.jar, ctx.label.name + "_javac_args") - javac_args = """{javac_opts} -classpath "{jars}:{out}_tmp" -d {out}_tmp {files}""".format( - javac_opts=" ".join(ctx.attr.javacopts), - jars=":".join([j.path for j in jars]), - files=" ".join([f.path for f in java_srcs]), - out=ctx.outputs.jar.path + outs = [ctx.outputs.jar] + if buildijar: + outs.extend([ctx.outputs.ijar]) + ins = (list(jars) + + list(dep_srcjars) + + list(srcjars) + + list(sources) + + ctx.files.srcs + + ctx.files.plugins + + ctx.files.resources + + ctx.files._jdk + + ctx.files._scalac + + ctx.files.__deploy_jar + + ctx.files._scalasdk + + [ctx.outputs.manifest, + ctx.file._ijar, + ctx.file._java, + scalac_args_file]) + if compile_java_srcs: + ins.extend([javac_args_file]) + ctx.action( + inputs=ins, + outputs=outs, + command=cmd, + # executable=ctx.executable._scalac, + mnemonic="Scalac", + progress_message="scala %s" % ctx.label, + execution_requirements={"supports-workers": "1"}, + arguments=["@" + ctx.outputs.jar.path + "_args/scala_args"], ) - ctx.file_action(output = javac_args_file, content = javac_args) - javac_sources_cmd = """ - cat {javac_args} {{out}}_args/files_from_jar > {{out}}_args/java_args - {javac} {{jvm_flags}} @{{out}}_args/java_args""".format(javac_args = javac_args_file.path,javac=ctx.file._javac.path) - - srcjar_cmd = "" - if len(all_srcjars) > 0: - srcjar_cmd = "\nmkdir -p {out}_tmp_expand_srcjars\n" - for srcjar in all_srcjars: - # Note: this is double escaped because we need to do one format call - # per each srcjar, but then we are going to include this in the bigger format - # call that is done to generate the full command - - #TODO would like to be able to switch >/dev/null, -v, etc based on the user's settings - srcjar_cmd += """ -unzip -o {srcjar} -d {{out}}_tmp_expand_srcjars >/dev/null -""".format(srcjar = srcjar.path) - srcjar_cmd += """find {out}_tmp_expand_srcjars -type f -name "*.scala" > {out}_args/files_from_jar\n""" - - cmd = """ -rm -rf {out}_args -rm -rf {out}_tmp -rm -rf {out}_tmp_expand_srcjars -set -e -mkdir -p {out}_args -touch {out}_args/files_from_jar -mkdir -p {out}_tmp""" + srcjar_cmd + """ -cat {scalac_args} {out}_args/files_from_jar > {out}_args/scala_args -{java} -jar {scalac} {jvm_flags} @{out}_args/scala_args""" + javac_sources_cmd + """ -# add any resources -{cp_resources} -{java} -jar {jar} -m {manifest} {out} {out}_tmp -rm -rf {out}_args -rm -rf {out}_tmp -rm -rf {out}_tmp_expand_srcjars -""" + ijar_cmd - cmd = cmd.format( - cp_resources=cp_resources, - java=ctx.file._java.path, - jvm_flags=" ".join(["-J" + flag for flag in ctx.attr.jvm_flags]), - scalac=_get_scalac_jar_path(ctx.files._scalac), - scalac_args=scalac_args_file.path, - out=ctx.outputs.jar.path, - manifest=ctx.outputs.manifest.path, - jar=_get_jar_path(ctx.files.__deploy_jar), - ijar=ctx.file._ijar.path, - ) - outs = [ctx.outputs.jar] - if buildijar: - outs.extend([ctx.outputs.ijar]) - ins = (list(jars) + - list(dep_srcjars) + - list(srcjars) + - list(sources) + - ctx.files.srcs + - ctx.files.plugins + - ctx.files.resources + - ctx.files._jdk + - ctx.files._scalac + - ctx.files.__deploy_jar + - ctx.files._scalasdk + - [ctx.outputs.manifest, - ctx.file._ijar, - ctx.file._java, - scalac_args_file]) - if compile_java_srcs: - ins.extend([javac_args_file]) - ctx.action( - inputs=ins, - outputs=outs, - command=cmd, - mnemonic="Scalac", - progress_message="scala %s" % ctx.label, - arguments=[]) def _compile_or_empty(ctx, jars, srcjars, buildijar): # We assume that if a srcjar is present, it is not empty @@ -482,7 +506,7 @@ _implicit_deps = { "_scalareflect": attr.label(default=Label("@scala//:lib/scala-reflect.jar"), single_file=True, allow_files=True), "_java": attr.label(executable=True, default=Label("@bazel_tools//tools/jdk:java"), single_file=True, allow_files=True), "_javac": attr.label(executable=True, default=Label("@bazel_tools//tools/jdk:javac"), single_file=True, allow_files=True), - "__deploy_jar": attr.label(executable=True, default=Label("//src/java/io/bazel/rulesscala/jar:jar_deploy.jar"), allow_files=True), + "__deploy_jar": attr.label(executable=True, default=Label("//src/java/io/bazel/rulesscala/jar:binary_deploy.jar"), allow_files=True), "_jdk": attr.label(default=Label("//tools/defaults:jdk"), allow_files=True), } diff --git a/src/java/io/bazel/rulesscala/jar/BUILD b/src/java/io/bazel/rulesscala/jar/BUILD index 009207740..f8e6c90ed 100644 --- a/src/java/io/bazel/rulesscala/jar/BUILD +++ b/src/java/io/bazel/rulesscala/jar/BUILD @@ -1,5 +1,13 @@ -java_binary(name = "jar", +java_library(name = "jar", srcs = ["JarCreator.java", "JarHelper.java"], + visibility = ["//visibility:public"], +) + + +java_binary(name = "binary", + runtime_deps = [ + ":jar", + ], main_class = "io.bazel.rulesscala.jar.JarCreator", visibility = ["//visibility:public"], ) diff --git a/src/java/io/bazel/rulesscala/jar/JarCreator.java b/src/java/io/bazel/rulesscala/jar/JarCreator.java index 231404561..d69b83d40 100644 --- a/src/java/io/bazel/rulesscala/jar/JarCreator.java +++ b/src/java/io/bazel/rulesscala/jar/JarCreator.java @@ -170,6 +170,29 @@ public void execute() throws IOException { } } + public static void buildJar(String[] args) throws IOException { + if (args.length < 1) { + System.err.println("usage: CreateJar [-m manifest] output [root directories]"); + System.exit(1); + } + + int idx = 0; + String manifestFile = null; + if (args[0].equals("-m")) { + manifestFile = args[1]; + idx = 2; + } + String output = args[idx]; + JarCreator createJar = new JarCreator(output); + createJar.setManifestFile(manifestFile); + for (int i = (idx+1); i < args.length; i++) { + createJar.addDirectory(args[i]); + } + createJar.setCompression(true); + createJar.setNormalize(true); + createJar.execute(); + } + /** * A simple way to create Jar file using the JarCreator class. */ diff --git a/src/java/io/bazel/rulesscala/scalac/BUILD b/src/java/io/bazel/rulesscala/scalac/BUILD index 5ea7f4447..95fafcb47 100644 --- a/src/java/io/bazel/rulesscala/scalac/BUILD +++ b/src/java/io/bazel/rulesscala/scalac/BUILD @@ -7,6 +7,7 @@ java_binary(name = "scalac", "@scala//:lib/scala-reflect.jar", "@scala//:lib/scala-compiler.jar", "@scala//:lib/scala-xml_2.11-1.0.4.jar", + "//src/java/io/bazel/rulesscala/jar", ], visibility = ["//visibility:public"], ) diff --git a/src/java/io/bazel/rulesscala/scalac/ScalaCInvoker.java b/src/java/io/bazel/rulesscala/scalac/ScalaCInvoker.java index a9fc529b9..9dd3bbb1c 100644 --- a/src/java/io/bazel/rulesscala/scalac/ScalaCInvoker.java +++ b/src/java/io/bazel/rulesscala/scalac/ScalaCInvoker.java @@ -30,6 +30,11 @@ import java.io.*; import java.lang.reflect.Field; import scala.tools.nsc.reporters.ConsoleReporter; +import java.util.Arrays; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.FileSystems; +import io.bazel.rulesscala.jar.JarCreator; /** * A class for creating Jar files. Allows normalization of Jar entries by setting their timestamp to @@ -37,9 +42,35 @@ */ public class ScalaCInvoker { + public static String[] buildPluginArgs(String packedPlugins) { + // plugin_arg = "" + // if (len(plugins) > 0): + // plugin_arg = " ".join(["-Xplugin:%s" % p for p in plugins]) + + + String[] result = {}; + + return result; // new String[](0); + } + + public static String[] merge(String[]... arrays) { + int totalLength = 0; + for(String[] arr:arrays){ + totalLength += arr.length; + } + + String[] result = new String[totalLength]; + int offset = 0; + for(String[] arr:arrays){ + System.arraycopy(arr, 0, result, offset, arr.length); + offset += arr.length; + } + return result; + } + public static void main(String[] args) { try { - // System.out.println("\n\n\n___ARGS_START____\n"); + System.out.println("\n\n\n___ARGS_START____\n"); if(args.length == 1 && args[0].indexOf("@") == 0) { String line; @@ -48,9 +79,40 @@ public static void main(String[] args) { line = in.readLine(); args = line.split(" "); } - // for (int i = 0; i < args.length; i++) { - // System.out.println(args[i]); - // } + + String outputName = args[0]; + String manifestPath = args[1]; + String[] scalaOpts = args[2].split(","); + String[] pluginArgs = buildPluginArgs(args[3]); + String classpath = args[4]; + String[] files = args[5].split(","); + + args = Arrays.copyOfRange(args, 2, args.length); + + Path outputPath = FileSystems.getDefault().getPath(outputName); + Path tmpPath = Files.createTempDirectory(outputPath.getParent(),"tmp"); + + System.out.println("Output path will be: " + outputName); + System.out.println("Manifest path will be: " + manifestPath); + System.out.println("tmpPath path will be: " + tmpPath); + + String[] constParams = { + "-classpath", + classpath, + "-d", + tmpPath.toString() + }; + + String[] compilerArgs = merge( + scalaOpts, + pluginArgs, + constParams, + files); + + + for (int i = 0; i < compilerArgs.length; i++) { + System.out.println(compilerArgs[i]); + } @@ -67,7 +129,7 @@ public static void main(String[] args) { // System.out.println("SASDF"); // System.out.println("SASDF"); MainClass comp = new MainClass(); - comp.process(args); + comp.process(compilerArgs); // System.out.println("SASDF"); @@ -81,6 +143,14 @@ public static void main(String[] args) { reporter.flush(); } else { // reportSuccess(); + String[] jarCreatorArgs = { + "-m", + manifestPath, + outputPath.toString(), + tmpPath.toString() + }; + JarCreator.buildJar(jarCreatorArgs); + System.out.println("Success"); } }