Merge branch 'master' into dmcclanahan/python-dist-c++-sources

.travis.yml

@@ -3,6 +3,7 @@ env:
     - PANTS_CONFIG_FILES="${TRAVIS_BUILD_DIR}/pants.travis-ci.ini"
     - ANDROID_SDK_INSTALL_LOCATION="${HOME}/opt/android-sdk-install"
     - ANDROID_HOME="$ANDROID_SDK_INSTALL_LOCATION/android-sdk-linux"
+    - PYTEST_PASSTHRU_ARGS="-v --duration=3"
 
 before_cache:
   # Ensure permissions to do the below removals, which happen with or without caching enabled.
@@ -33,6 +34,7 @@ before_cache:
   # individually resolved artifacts.
   - rm -rf ${HOME}/.cache/pants/bin
   - rm -rf ${HOME}/.cache/pants/rust/rustup
+  - rm -rf ${HOME}/.cache/pants/lmdb_store
   # Render a summary of what is left in the home directory, to assist with further pruning of
   # the cache.
   - du -d2 ${HOME} | sort -r -n

3rdparty/python/requirements.txt

@@ -2,7 +2,7 @@ ansicolors==1.0.2
 beautifulsoup4>=4.3.2,<4.4
 cffi==1.11.1
 contextlib2==0.5.5
-coverage>=4.3.4,<4.4
+coverage>=4.5,<4.6
 docutils>=0.12,<0.13
 fasteners==0.14.1
 faulthandler==2.6
@@ -20,8 +20,8 @@ pyflakes==1.1.0
 Pygments==1.4
 pyopenssl==17.3.0
 pystache==0.5.3
-pytest-cov>=2.4,<2.5
-pytest>=3.0.7,<4.0
+pytest-cov>=2.5,<2.6
+pytest>=3.4,<4.0
 pywatchman==1.4.1
 requests[security]>=2.5.0,<2.19
 scandir==1.2

build-support/bin/ci.sh

@@ -185,7 +185,7 @@ if [[ "${skip_internal_backends:-false}" == "false" ]]; then
   start_travis_section "BackendTests" "Running internal backend python tests"
   (
     ./pants.pex ${PANTS_ARGS[@]} test.pytest \
-    pants-plugins/tests/python::
+    pants-plugins/tests/python:: -- ${PYTEST_PASSTHRU_ARGS}
   ) || die "Internal backend python test failure"
   end_travis_section
 fi
@@ -198,7 +198,7 @@ if [[ "${skip_python:-false}" == "false" ]]; then
   (
     ./pants.pex --tag='-integration' ${PANTS_ARGS[@]} test.pytest --chroot \
       --test-pytest-test-shard=${python_unit_shard} \
-      tests/python::
+      tests/python:: -- ${PYTEST_PASSTHRU_ARGS}
   ) || die "Core python test failure"
   end_travis_section
 fi
@@ -212,15 +212,18 @@ if [[ "${skip_contrib:-false}" == "false" ]]; then
     ./pants.pex ${PANTS_ARGS[@]} --exclude-target-regexp='.*/testprojects/.*' \
     --build-ignore=$SKIP_ANDROID_PATTERN test.pytest \
     --test-pytest-test-shard=${python_contrib_shard} \
-    contrib:: \
+    contrib:: -- ${PYTEST_PASSTHRU_ARGS}
   ) || die "Contrib python test failure"
   end_travis_section
 fi
 
 if [[ "${skip_rust_tests:-false}" == "false" ]]; then
   start_travis_section "RustTests" "Running Pants rust tests"
   (
-    "${REPO_ROOT}/src/rust/engine/run-all-tests.sh"
+    source "${REPO_ROOT}/build-support/pants_venv"
+    source "${REPO_ROOT}/build-support/bin/native/bootstrap.sh"
+    activate_pants_venv
+    RUST_BACKTRACE=1 PANTS_SRCPATH="${REPO_ROOT}/src/python" ensure_cffi_sources=1 run_cargo test "${MODE_FLAG}" --all --manifest-path="${REPO_ROOT}/src/rust/engine/Cargo.toml"
   ) || die "Pants rust test failure"
   end_travis_section
 fi
@@ -234,7 +237,7 @@ if [[ "${skip_integration:-false}" == "false" ]]; then
   (
     ./pants.pex ${PANTS_ARGS[@]} --tag='+integration' test.pytest \
       --test-pytest-test-shard=${python_intg_shard} \
-      tests/python::
+      tests/python:: -- ${PYTEST_PASSTHRU_ARGS}
   ) || die "Pants Integration test failure"
   end_travis_section
 fi

build-support/bin/native/bootstrap.sh

@@ -60,6 +60,7 @@ function calculate_current_hash() {
      "${REPO_ROOT}/src/python/pants/engine/native.py" \
      "${REPO_ROOT}/build-support/bin/native" \
      "${REPO_ROOT}/3rdparty/python/requirements.txt" \
+   | grep -v -E -e "/BUILD$" -e "/[^/]*\.md$" \
    | git hash-object -t blob --stdin-paths) | fingerprint_data
   )
 }
@@ -91,6 +92,7 @@ function ensure_native_build_prerequisites() {
   # Make sure rust is pinned at the correct version.
   # We sincerely hope that no one ever runs `rustup override set` in a subdirectory of the working directory.
   "${CARGO_HOME}/bin/rustup" override set "${RUST_TOOLCHAIN}" >&2
+  "${CARGO_HOME}/bin/rustup" component add rustfmt-preview >&2
 
   # Sometimes fetching a large git repo dependency can take more than 10 minutes.
   # This times out on travis, because nothing is printed to stdout/stderr in that time.
@@ -107,7 +109,6 @@ function ensure_native_build_prerequisites() {
   "${CARGO_HOME}/bin/cargo" ensure-installed --package=cargo-ensure-installed --version=0.1.0 >&2
   "${CARGO_HOME}/bin/cargo" ensure-installed --package=protobuf --version=1.4.2 >&2
   "${CARGO_HOME}/bin/cargo" ensure-installed --package=grpcio-compiler --version=0.2.0 >&2
-  "${CARGO_HOME}/bin/cargo" ensure-installed --package=rustfmt --version=0.9.0 >&2
 
   local download_binary="${REPO_ROOT}/build-support/bin/download_binary.sh"
   local readonly cmakeroot="$("${download_binary}" "binaries.pantsbuild.org" "cmake" "3.9.5" "cmake.tar.gz")" || die "Failed to fetch cmake"

build-support/bin/release.sh

@@ -2,6 +2,8 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+set -e
+
 ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && cd "$(git rev-parse --show-toplevel)" && pwd)
 source ${ROOT}/build-support/common.sh
 
@@ -234,7 +236,7 @@ function build_pants_packages() {
     RUST_BACKTRACE=1 PANTS_SRCPATH="${ROOT}/src/python" run_cargo build --release --manifest-path="${ROOT}/src/rust/engine/fs/fs_util/Cargo.toml"
     dst_dir="${DEPLOY_DIR}/bin/fs_util/$("${ROOT}/build-support/bin/get_os.sh")/${version}"
     mkdir -p "${dst_dir}"
-    cp "${ROOT}/src/rust/engine/fs/fs_util/target/release/fs_util" "${dst_dir}/"
+    cp "${ROOT}/src/rust/engine/target/release/fs_util" "${dst_dir}/"
   ) || die "Failed to build fs_util"
   end_travis_section
 
@@ -275,7 +277,7 @@ EOM
 
 function install_and_test_packages() {
   local VERSION=$1
-  shift 2
+  shift
   local PIP_ARGS=(
     "${VERSION}"
     "$@"
@@ -672,21 +674,62 @@ function activate_twine() {
 
 function build_pex() {
   # Builds a pex from the current UNSTABLE version.
+  # If $1 == "build", builds a pex just for this platform, from source.
+  # If $1 == "fetch", fetches the linux and OSX wheels which were built on travis.
+  local mode="$1"
 
   local linux_platform="linux_x86_64"
+  local osx_platform="macosx_10.10_x86_64"
+
+  case "${mode}" in
+    build)
+      case "$(uname)" in
+	Darwin)
+	  local platform="${osx_platform}"
+	  ;;
+	Linux)
+	  local platform="${linux_platform}"
+	  ;;
+	*)
+	  echo >&2 "Unknown uname"
+	  exit 1
+	  ;;
+      esac
+      local platforms=("${platform}")
+      local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.${platform}.pex"
+      ;;
+    fetch)
+      local platforms=("${linux_platform}" "macosx_10.10_x86_64")
+      local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.pex"
+      ;;
+    *)
+      echo >&2 "Bad build_pex mode ${mode}"
+      exit 1
+      ;;
+  esac
 
   rm -rf "${DEPLOY_DIR}"
   mkdir -p "${DEPLOY_DIR}"
-  fetch_and_check_prebuilt_wheels "${DEPLOY_DIR}"
-  adjust_wheel_platform "manylinux1_x86_64" "${linux_platform}" "${DEPLOY_DIR}"
 
-  local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.pex"
+  if [[ "${mode}" == "fetch" ]]; then
+    fetch_and_check_prebuilt_wheels "${DEPLOY_DIR}"
+  else
+    build_pants_packages "${PANTS_UNSTABLE_VERSION}"
+    build_3rdparty_packages "${PANTS_UNSTABLE_VERSION}"
+  fi
+
+  adjust_wheel_platform "manylinux1_x86_64" "${linux_platform}" "${DEPLOY_DIR}"
 
   activate_tmp_venv && trap deactivate RETURN && pip install "pex==1.3.1" || die "Failed to install pex."
 
-  local requirements_string=""
+  local requirements=()
   for pkg_name in $PANTS_PEX_PACKAGES; do
-    requirements_string="${requirements_string} ${pkg_name}==${PANTS_UNSTABLE_VERSION}"
+    requirements=("${requirements[@]}" "${pkg_name}==${PANTS_UNSTABLE_VERSION}")
+  done
+
+  local platform_flags=()
+  for platform in "${platforms[@]}"; do
+    platform_flags=("${platform_flags[@]}" "--platform=${platform}")
   done
 
   pex \
@@ -695,11 +738,10 @@ function build_pex() {
     --no-build \
     --no-pypi \
     --disable-cache \
-    --platform="macosx_10.10_x86_64" \
-    --platform="${linux_platform}" \
+    "${platform_flags[@]}" \
     -f "${DEPLOY_PANTS_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
     -f "${DEPLOY_3RDPARTY_WHEEL_DIR}/${PANTS_UNSTABLE_VERSION}" \
-    ${requirements_string}
+    "${requirements[@]}"
 
   banner "Successfully built ${dest}"
 }
@@ -754,6 +796,7 @@ function usage() {
   echo " -o  Lists all pantsbuild package owners."
   echo " -e  Check that wheels are prebuilt for this release."
   echo " -p  Build a pex from prebuilt wheels for this release."
+  echo " -q  Build a pex which only works on the host platform, using the code as exists on disk."
   echo
   echo "All options (except for '-d') are mutually exclusive."
 
@@ -764,7 +807,7 @@ function usage() {
   fi
 }
 
-while getopts "hdntcloepw" opt; do
+while getopts "hdntcloepqw" opt; do
   case ${opt} in
     h) usage ;;
     d) debug="true" ;;
@@ -773,7 +816,8 @@ while getopts "hdntcloepw" opt; do
     l) list_packages ; exit $? ;;
     o) list_owners ; exit $? ;;
     e) fetch_and_check_prebuilt_wheels ; exit $? ;;
-    p) build_pex ; exit $? ;;
+    p) build_pex fetch ; exit $? ;;
+    q) build_pex build ; exit $? ;;
     w) list_prebuilt_wheels ; exit $? ;;
     *) usage "Invalid option: -${OPTARG}" ;;
   esac

contrib/buildrefactor/src/python/pants/contrib/buildrefactor/buildozer_binary.py

@@ -21,7 +21,7 @@ class BuildozerBinary(NativeTool):
   # that runs buildozer.
   options_scope = 'buildozer-binary'
   name = 'buildozer'
-  default_version = '0.6.0.dce8b3c287652cbcaf43c8dd076b3f48c92ab44c'
+  default_version = '0.6.0-80c7f0d45d7e40fa1f7362852697d4a03df557b3'
 
   replaces_scope = 'buildozer'
   replaces_name = 'version'

contrib/googlejavaformat/src/python/pants/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

contrib/googlejavaformat/src/python/pants/contrib/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

contrib/googlejavaformat/src/python/pants/contrib/googlejavaformat/BUILD

@@ -0,0 +1,18 @@
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(sources=['googlejavaformat.py'])
+
+contrib_plugin(
+  name='plugin',
+  dependencies=[
+    ':googlejavaformat',
+    'src/python/pants/goal:task_registrar',
+  ],
+  distribution_name='pantsbuild.pants.contrib.googlejavaformat',
+  description='Google Java Format code formatter pants plugins.',
+  additional_classifiers=[
+    'Topic :: Software Development :: Pre-processors'
+  ],
+  register_goals=True,
+)

contrib/googlejavaformat/src/python/pants/contrib/googlejavaformat/googlejavaformat.py

@@ -0,0 +1,80 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from abc import abstractproperty
+
+from pants.backend.jvm.tasks.rewrite_base import RewriteBase
+from pants.base.exceptions import TaskError
+from pants.java.jar.jar_dependency import JarDependency
+from pants.task.fmt_task_mixin import FmtTaskMixin
+from pants.task.lint_task_mixin import LintTaskMixin
+
+
+class GoogleJavaFormatBase(RewriteBase):
+
+  @classmethod
+  def register_options(cls, register):
+    super(GoogleJavaFormatBase, cls).register_options(register)
+    cls.register_jvm_tool(register,
+                          'google-java-format',
+                          classpath=[
+                          JarDependency(org='com.google.googlejavaformat',
+                                        name='google-java-format',
+                                        rev='1.5')
+                          ])
+
+  @classmethod
+  def implementation_version(cls):
+    return super(GoogleJavaFormatBase, cls).implementation_version() + [('GoogleJavaFormatBase', 1)]
+
+  @classmethod
+  def target_types(cls):
+    return ['java_library', 'junit_tests', 'java_tests']
+
+  @classmethod
+  def source_extension(cls):
+    return '.java'
+
+  def invoke_tool(self, _, target_sources):
+    args = list(self.additional_args)
+    args.extend([source for target, source in target_sources])
+    return self.runjava(classpath=self.tool_classpath('google-java-format'),
+                        main='com.google.googlejavaformat.java.Main',
+                        args=args,
+                        workunit_name='google-java-format',
+                        jvm_options=self.get_options().jvm_options)
+
+  @abstractproperty
+  def additional_args(self):
+    """List of additional args to supply on the tool command-line."""
+
+
+class GoogleJavaFormatCheckFormat(LintTaskMixin, GoogleJavaFormatBase):
+  """Check if Java source code complies with Google Java Style.
+
+  If the files are not formatted correctly an error is raised
+  including the command to run to format the files correctly
+  """
+
+  sideeffecting = False
+  additional_args = ['--set-exit-if-changed']
+
+  def process_result(self, result):
+    if result != 0:
+      raise TaskError('google-java-format failed with exit code {}; to fix run: '
+                      '`./pants fmt <targets>`'.format(result), exit_code=result)
+
+
+class GoogleJavaFormat(FmtTaskMixin, GoogleJavaFormatBase):
+  """Reformat Java source code to comply with Google Java Style."""
+
+  sideeffecting = True
+  additional_args = ['-i']
+
+  def process_result(self, result):
+    if result != 0:
+      raise TaskError('google-java-format failed to format files', exit_code=result)

contrib/googlejavaformat/src/python/pants/contrib/googlejavaformat/register.py

@@ -0,0 +1,16 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+from pants.goal.task_registrar import TaskRegistrar as task
+
+from pants.contrib.googlejavaformat.googlejavaformat import (GoogleJavaFormat,
+                                                             GoogleJavaFormatCheckFormat)
+
+
+def register_goals():
+  task(name='google-java-format', action=GoogleJavaFormat).install('fmt')
+  task(name='google-java-format', action=GoogleJavaFormatCheckFormat).install('lint')

contrib/googlejavaformat/tests/python/pants_test/contrib/googlejavaformat/BUILD

@@ -0,0 +1,10 @@
+python_tests(
+  dependencies = [
+    'contrib/googlejavaformat/src/python/pants/contrib/googlejavaformat',
+    'src/python/pants/backend/jvm:plugin',
+    'src/python/pants/backend/jvm/targets:java',
+    'src/python/pants/base:exceptions',
+    'src/python/pants/build_graph',
+    'tests/python/pants_test/jvm:nailgun_task_test_base',
+  ],
+)