From 9322a294e7b67d53f61447960effb5281bd0ee81 Mon Sep 17 00:00:00 2001 From: Kyle Weaver Date: Tue, 5 Nov 2019 11:59:29 -0800 Subject: [PATCH] [BEAM-8512] Add integration tests for flink_runner.py --- build.gradle | 4 ++++ .../org/apache/beam/gradle/BeamModulePlugin.groovy | 12 +++++++----- sdks/python/test-suites/portable/py2/build.gradle | 7 +++++++ sdks/python/test-suites/portable/py35/build.gradle | 7 +++++++ sdks/python/test-suites/portable/py36/build.gradle | 7 +++++++ sdks/python/test-suites/portable/py37/build.gradle | 7 +++++++ 6 files changed, 39 insertions(+), 5 deletions(-) diff --git a/build.gradle b/build.gradle index 133543ff3feca..abc0cb515a85f 100644 --- a/build.gradle +++ b/build.gradle @@ -227,22 +227,26 @@ task python2PostCommit() { dependsOn ":sdks:python:test-suites:direct:py2:directRunnerIT" dependsOn ":sdks:python:test-suites:direct:py2:hdfsIntegrationTest" dependsOn ":sdks:python:test-suites:direct:py2:mongodbioIT" + dependsOn ":sdks:python:test-suites:portable:py2:postCommitPy2" } task python35PostCommit() { dependsOn ":sdks:python:test-suites:dataflow:py35:postCommitIT" dependsOn ":sdks:python:test-suites:direct:py35:postCommitIT" + dependsOn ":sdks:python:test-suites:portable:py35:postCommitPy35" } task python36PostCommit() { dependsOn ":sdks:python:test-suites:dataflow:py36:postCommitIT" dependsOn ":sdks:python:test-suites:direct:py36:postCommitIT" + dependsOn ":sdks:python:test-suites:portable:py36:postCommitPy36" } task python37PostCommit() { dependsOn ":sdks:python:test-suites:dataflow:py37:postCommitIT" dependsOn ":sdks:python:test-suites:direct:py37:postCommitIT" dependsOn ":sdks:python:test-suites:direct:py37:hdfsIntegrationTest" + dependsOn ":sdks:python:test-suites:portable:py37:postCommitPy37" } task portablePythonPreCommit() { diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index 04f96eb39e690..0ae4c4681bbbf 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1899,8 +1899,8 @@ class BeamModulePlugin implements Plugin { } } - def addPortableWordCountTask = { boolean isStreaming -> - project.task('portableWordCount' + (isStreaming ? 'Streaming' : 'Batch')) { + def addPortableWordCountTask = { boolean isStreaming, String runner -> + project.task('portableWordCount' + (runner.equals("PortableRunner") ? "" : runner) + (isStreaming ? 'Streaming' : 'Batch')) { dependsOn = ['installGcpTest'] mustRunAfter = [ ':runners:flink:1.9:job-server-container:docker', @@ -1914,7 +1914,7 @@ class BeamModulePlugin implements Plugin { def options = [ "--input=/etc/profile", "--output=/tmp/py-wordcount-direct", - "--runner=PortableRunner", + "--runner=${runner}", "--experiments=worker_threads=100", "--parallelism=2", "--shutdown_sources_on_final_watermark", @@ -1953,8 +1953,10 @@ class BeamModulePlugin implements Plugin { } project.ext.addPortableWordCountTasks = { -> - addPortableWordCountTask(false) - addPortableWordCountTask(true) + addPortableWordCountTask(false, "PortableRunner") + addPortableWordCountTask(true, "PortableRunner") + addPortableWordCountTask(false, "FlinkRunner") + addPortableWordCountTask(true, "FlinkRunner") } } } diff --git a/sdks/python/test-suites/portable/py2/build.gradle b/sdks/python/test-suites/portable/py2/build.gradle index 2b9529628c3dd..3c1548dff52a4 100644 --- a/sdks/python/test-suites/portable/py2/build.gradle +++ b/sdks/python/test-suites/portable/py2/build.gradle @@ -34,6 +34,13 @@ task preCommitPy2() { dependsOn portableWordCountStreaming } +task postCommitPy2() { + dependsOn 'setupVirtualenv' + dependsOn ':runners:flink:1.9:job-server:shadowJar' + dependsOn portableWordCountFlinkRunnerBatch + dependsOn portableWordCountFlinkRunnerStreaming +} + // TODO: Move the rest of this file into ../common.gradle. // Before running this, you need to: diff --git a/sdks/python/test-suites/portable/py35/build.gradle b/sdks/python/test-suites/portable/py35/build.gradle index 42667c7cfd5ee..1b2cb4f90687a 100644 --- a/sdks/python/test-suites/portable/py35/build.gradle +++ b/sdks/python/test-suites/portable/py35/build.gradle @@ -30,3 +30,10 @@ task preCommitPy35() { dependsOn portableWordCountBatch dependsOn portableWordCountStreaming } + +task postCommitPy35() { + dependsOn 'setupVirtualenv' + dependsOn ':runners:flink:1.9:job-server:shadowJar' + dependsOn portableWordCountFlinkRunnerBatch + dependsOn portableWordCountFlinkRunnerStreaming +} diff --git a/sdks/python/test-suites/portable/py36/build.gradle b/sdks/python/test-suites/portable/py36/build.gradle index d536d14497aca..475e1105d125e 100644 --- a/sdks/python/test-suites/portable/py36/build.gradle +++ b/sdks/python/test-suites/portable/py36/build.gradle @@ -30,3 +30,10 @@ task preCommitPy36() { dependsOn portableWordCountBatch dependsOn portableWordCountStreaming } + +task postCommitPy36() { + dependsOn 'setupVirtualenv' + dependsOn ':runners:flink:1.9:job-server:shadowJar' + dependsOn portableWordCountFlinkRunnerBatch + dependsOn portableWordCountFlinkRunnerStreaming +} diff --git a/sdks/python/test-suites/portable/py37/build.gradle b/sdks/python/test-suites/portable/py37/build.gradle index da57c9333eec4..912b3169faa67 100644 --- a/sdks/python/test-suites/portable/py37/build.gradle +++ b/sdks/python/test-suites/portable/py37/build.gradle @@ -30,3 +30,10 @@ task preCommitPy37() { dependsOn portableWordCountBatch dependsOn portableWordCountStreaming } + +task postCommitPy37() { + dependsOn 'setupVirtualenv' + dependsOn ':runners:flink:1.9:job-server:shadowJar' + dependsOn portableWordCountFlinkRunnerBatch + dependsOn portableWordCountFlinkRunnerStreaming +}