forked from alteryx/spark
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request alteryx#7 from markhamstra/master-csd
Rebased to AMPLab master
- Loading branch information
Showing
59 changed files
with
1,389 additions
and
940 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -36,4 +36,5 @@ streaming-tests.log | |
dependency-reduced-pom.xml | ||
.ensime | ||
.ensime_lucene | ||
checkpoint | ||
derby.log |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
@echo off | ||
|
||
rem This script computes Spark's classpath and prints it to stdout; it's used by both the "run" | ||
rem script and the ExecutorRunner in standalone cluster mode. | ||
|
||
set SCALA_VERSION=2.9.3 | ||
|
||
rem Figure out where the Spark framework is installed | ||
set FWDIR=%~dp0..\ | ||
|
||
rem Load environment variables from conf\spark-env.cmd, if it exists | ||
if exist "%FWDIR%conf\spark-env.cmd" call "%FWDIR%conf\spark-env.cmd" | ||
|
||
set CORE_DIR=%FWDIR%core | ||
set REPL_DIR=%FWDIR%repl | ||
set EXAMPLES_DIR=%FWDIR%examples | ||
set BAGEL_DIR=%FWDIR%bagel | ||
set STREAMING_DIR=%FWDIR%streaming | ||
set PYSPARK_DIR=%FWDIR%python | ||
|
||
rem Build up classpath | ||
set CLASSPATH=%SPARK_CLASSPATH%;%MESOS_CLASSPATH%;%FWDIR%conf;%CORE_DIR%\target\scala-%SCALA_VERSION%\classes | ||
set CLASSPATH=%CLASSPATH%;%CORE_DIR%\target\scala-%SCALA_VERSION%\test-classes;%CORE_DIR%\src\main\resources | ||
set CLASSPATH=%CLASSPATH%;%STREAMING_DIR%\target\scala-%SCALA_VERSION%\classes;%STREAMING_DIR%\target\scala-%SCALA_VERSION%\test-classes | ||
set CLASSPATH=%CLASSPATH%;%STREAMING_DIR%\lib\org\apache\kafka\kafka\0.7.2-spark\* | ||
set CLASSPATH=%CLASSPATH%;%REPL_DIR%\target\scala-%SCALA_VERSION%\classes;%EXAMPLES_DIR%\target\scala-%SCALA_VERSION%\classes | ||
set CLASSPATH=%CLASSPATH%;%FWDIR%lib_managed\jars\* | ||
set CLASSPATH=%CLASSPATH%;%FWDIR%lib_managed\bundles\* | ||
set CLASSPATH=%CLASSPATH%;%FWDIR%repl\lib\* | ||
set CLASSPATH=%CLASSPATH%;%FWDIR%python\lib\* | ||
set CLASSPATH=%CLASSPATH%;%BAGEL_DIR%\target\scala-%SCALA_VERSION%\classes | ||
|
||
rem Add hadoop conf dir - else FileSystem.*, etc fail | ||
rem Note, this assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts | ||
rem the configurtion files. | ||
if "x%HADOOP_CONF_DIR%"=="x" goto no_hadoop_conf_dir | ||
set CLASSPATH=%CLASSPATH%;%HADOOP_CONF_DIR% | ||
:no_hadoop_conf_dir | ||
|
||
if "x%YARN_CONF_DIR%"=="x" goto no_yarn_conf_dir | ||
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR% | ||
:no_yarn_conf_dir | ||
|
||
rem Add Scala standard library | ||
set CLASSPATH=%CLASSPATH%;%SCALA_HOME%\lib\scala-library.jar;%SCALA_HOME%\lib\scala-compiler.jar;%SCALA_HOME%\lib\jline.jar | ||
|
||
rem A bit of a hack to allow calling this script within run2.cmd without seeing output | ||
if "%DONT_PRINT_CLASSPATH%"=="1" goto exit | ||
|
||
echo %CLASSPATH% | ||
|
||
:exit |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,89 @@ | ||
#!/bin/bash | ||
|
||
# This script computes Spark's classpath and prints it to stdout; it's used by both the "run" | ||
# script and the ExecutorRunner in standalone cluster mode. | ||
|
||
SCALA_VERSION=2.9.3 | ||
|
||
# Figure out where Spark is installed | ||
FWDIR="$(cd `dirname $0`/..; pwd)" | ||
|
||
# Load environment variables from conf/spark-env.sh, if it exists | ||
if [ -e $FWDIR/conf/spark-env.sh ] ; then | ||
. $FWDIR/conf/spark-env.sh | ||
fi | ||
|
||
CORE_DIR="$FWDIR/core" | ||
REPL_DIR="$FWDIR/repl" | ||
REPL_BIN_DIR="$FWDIR/repl-bin" | ||
EXAMPLES_DIR="$FWDIR/examples" | ||
BAGEL_DIR="$FWDIR/bagel" | ||
STREAMING_DIR="$FWDIR/streaming" | ||
PYSPARK_DIR="$FWDIR/python" | ||
|
||
# Build up classpath | ||
CLASSPATH="$SPARK_CLASSPATH" | ||
CLASSPATH="$CLASSPATH:$FWDIR/conf" | ||
CLASSPATH="$CLASSPATH:$CORE_DIR/target/scala-$SCALA_VERSION/classes" | ||
if [ -n "$SPARK_TESTING" ] ; then | ||
CLASSPATH="$CLASSPATH:$CORE_DIR/target/scala-$SCALA_VERSION/test-classes" | ||
CLASSPATH="$CLASSPATH:$STREAMING_DIR/target/scala-$SCALA_VERSION/test-classes" | ||
fi | ||
CLASSPATH="$CLASSPATH:$CORE_DIR/src/main/resources" | ||
CLASSPATH="$CLASSPATH:$REPL_DIR/target/scala-$SCALA_VERSION/classes" | ||
CLASSPATH="$CLASSPATH:$EXAMPLES_DIR/target/scala-$SCALA_VERSION/classes" | ||
CLASSPATH="$CLASSPATH:$STREAMING_DIR/target/scala-$SCALA_VERSION/classes" | ||
CLASSPATH="$CLASSPATH:$STREAMING_DIR/lib/org/apache/kafka/kafka/0.7.2-spark/*" # <-- our in-project Kafka Jar | ||
if [ -e "$FWDIR/lib_managed" ]; then | ||
CLASSPATH="$CLASSPATH:$FWDIR/lib_managed/jars/*" | ||
CLASSPATH="$CLASSPATH:$FWDIR/lib_managed/bundles/*" | ||
fi | ||
CLASSPATH="$CLASSPATH:$REPL_DIR/lib/*" | ||
# Add the shaded JAR for Maven builds | ||
if [ -e $REPL_BIN_DIR/target ]; then | ||
for jar in `find "$REPL_BIN_DIR/target" -name 'spark-repl-*-shaded-hadoop*.jar'`; do | ||
CLASSPATH="$CLASSPATH:$jar" | ||
done | ||
# The shaded JAR doesn't contain examples, so include those separately | ||
EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar` | ||
CLASSPATH+=":$EXAMPLES_JAR" | ||
fi | ||
CLASSPATH="$CLASSPATH:$BAGEL_DIR/target/scala-$SCALA_VERSION/classes" | ||
for jar in `find $PYSPARK_DIR/lib -name '*jar'`; do | ||
CLASSPATH="$CLASSPATH:$jar" | ||
done | ||
|
||
# Figure out the JAR file that our examples were packaged into. This includes a bit of a hack | ||
# to avoid the -sources and -doc packages that are built by publish-local. | ||
if [ -e "$EXAMPLES_DIR/target/scala-$SCALA_VERSION/spark-examples"*[0-9T].jar ]; then | ||
# Use the JAR from the SBT build | ||
export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/scala-$SCALA_VERSION/spark-examples"*[0-9T].jar` | ||
fi | ||
if [ -e "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar ]; then | ||
# Use the JAR from the Maven build | ||
export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar` | ||
fi | ||
|
||
# Add hadoop conf dir - else FileSystem.*, etc fail ! | ||
# Note, this assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts | ||
# the configurtion files. | ||
if [ "x" != "x$HADOOP_CONF_DIR" ]; then | ||
CLASSPATH="$CLASSPATH:$HADOOP_CONF_DIR" | ||
fi | ||
if [ "x" != "x$YARN_CONF_DIR" ]; then | ||
CLASSPATH="$CLASSPATH:$YARN_CONF_DIR" | ||
fi | ||
|
||
# Add Scala standard library | ||
if [ -z "$SCALA_LIBRARY_PATH" ]; then | ||
if [ -z "$SCALA_HOME" ]; then | ||
echo "SCALA_HOME is not set" >&2 | ||
exit 1 | ||
fi | ||
SCALA_LIBRARY_PATH="$SCALA_HOME/lib" | ||
fi | ||
CLASSPATH="$CLASSPATH:$SCALA_LIBRARY_PATH/scala-library.jar" | ||
CLASSPATH="$CLASSPATH:$SCALA_LIBRARY_PATH/scala-compiler.jar" | ||
CLASSPATH="$CLASSPATH:$SCALA_LIBRARY_PATH/jline.jar" | ||
|
||
echo "$CLASSPATH" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.