Skip to content

Commit

Permalink
[SYSTEMDS-31] Windows shell script, Windows MKL DLL, fixed license he…
Browse files Browse the repository at this point in the history
…aders of modified files
  • Loading branch information
corepointer committed Aug 29, 2019
1 parent 07debfc commit ceaa6ea
Show file tree
Hide file tree
Showing 16 changed files with 330 additions and 146 deletions.
73 changes: 43 additions & 30 deletions bin/systemds-standalone.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ EOF

# Script internally invokes 'java -Xmx4g -Xms4g -Xmn400m [Custom-Java-Options] -jar StandaloneSystemDS.jar -f <dml-filename> -exec singlenode -config=SystemDS-config.xml [Optional-Arguments]'

# This path can stay in *nix separator style even when using cygwin/msys.
export HADOOP_HOME=/tmp/systemds
mkdir -p $HADOOP_HOME

if [ -z "$1" ] ; then
echo "Wrong Usage.";
printSimpleUsage
Expand All @@ -39,17 +43,31 @@ if [ ! -z $SYSTEMDS_ROOT ]; then
PROJECT_ROOT_DIR="$SYSTEMDS_ROOT"
echo "SYTEMDS_ROOT is set to:" $SYSTEMDS_ROOT
else
# Paths that go to the java executable need to be Windows style in cygwin/msys.
DIR_SEP=/
# find the systemDS root path which contains the bin folder, the script folder and the target folder
# tolerate path with spaces
SCRIPT_DIR=$( dirname "$0" )
PROJECT_ROOT_DIR=$( cd "${SCRIPT_DIR}/.." ; pwd -P )

if [ "$OSTYPE" == "win32" ] || [ "$OSTYPE" == "msys" ] ; then
echo "Constructing Windows-style classpath"
DIR_SEP=\\
PROJECT_ROOT_DIR=`cygpath -w -p $( cd "${SCRIPT_DIR}/.." ; pwd -P )`

if [ ! -f ${HADOOP_HOME}${DIR_SEP}bin ]; then
cp -a ${PROJECT_ROOT_DIR}${DIR_SEP}target${DIR_SEP}lib${DIR_SEP}hadoop${DIR_SEP}bin ${HADOOP_HOME}
fi
USER_DIR=`cygpath -w -p ${PWD}`
else
PROJECT_ROOT_DIR=$( cd "${SCRIPT_DIR}/.." ; pwd -P )
USER_DIR=$PWD
fi
fi

USER_DIR=$PWD

BUILD_DIR=${PROJECT_ROOT_DIR}/target
HADOOP_LIB_DIR=${BUILD_DIR}/lib
DML_SCRIPT_CLASS=${BUILD_DIR}/classes/org/tugraz/sysds/api/DMLScript.class
BUILD_DIR=${PROJECT_ROOT_DIR}${DIR_SEP}target
HADOOP_LIB_DIR=${BUILD_DIR}${DIR_SEP}lib
DML_SCRIPT_CLASS=${BUILD_DIR}${DIR_SEP}classes${DIR_SEP}org${DIR_SEP}tugraz${DIR_SEP}sysds${DIR_SEP}api${DIR_SEP}DMLScript.class

BUILD_ERR_MSG="You must build the project before running this script."
BUILD_DIR_ERR_MSG="Could not find target directory \"${BUILD_DIR}\". ${BUILD_ERR_MSG}"
Expand All @@ -67,45 +85,41 @@ echo "==========================================================================
# if the present working directory is the project root or bin folder, then use the temp folder as user.dir
if [ "$USER_DIR" = "$PROJECT_ROOT_DIR" ] || [ "$USER_DIR" = "$PROJECT_ROOT_DIR/bin" ]
then
USER_DIR=${PROJECT_ROOT_DIR}/temp
USER_DIR=${PROJECT_ROOT_DIR}${DIR_SEP}temp
echo "Output dir: $USER_DIR"
fi


# if the SystemDS-config.xml does not exist, create it from the template
if [ ! -f "${PROJECT_ROOT_DIR}/conf/SystemDS-config.xml" ]
if [ ! -f "${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}SystemDS-config.xml" ]
then
cp "${PROJECT_ROOT_DIR}/conf/SystemDS-config.xml.template" \
"${PROJECT_ROOT_DIR}/conf/SystemDS-config.xml"
echo "... created ${PROJECT_ROOT_DIR}/conf/SystemDS-config.xml"
cp "${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}SystemDS-config.xml.template" \
"${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}SystemDS-config.xml"
echo "... created ${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}SystemDS-config.xml"
fi

# if the log4j.properties do not exis, create them from the template
if [ ! -f "${PROJECT_ROOT_DIR}/conf/log4j.properties" ]
if [ ! -f "${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}log4j.properties" ]
then
cp "${PROJECT_ROOT_DIR}/conf/log4j.properties.template" \
"${PROJECT_ROOT_DIR}/conf/log4j.properties"
echo "... created ${PROJECT_ROOT_DIR}/conf/log4j.properties"
cp "${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}log4j.properties.template" \
"${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}log4j.properties"
echo "... created ${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}log4j.properties"
fi




# add hadoop libraries which were generated by the build to the classpath
CLASSPATH=\"${BUILD_DIR}/lib/*\"

#SYSTEM_DS_JAR=$( find $PROJECT_ROOT_DIR/target/system-ds-*-SNAPSHOT.jar )
SYSTEM_DS_JAR=\"${BUILD_DIR}/classes\"

CLASSPATH=${CLASSPATH}:${SYSTEM_DS_JAR}
SYSTEM_DS_JAR=${BUILD_DIR}${DIR_SEP}classes
# add hadoop libraries which were generated by the build to the classpath
CLASSPATH=${BUILD_DIR}${DIR_SEP}lib${DIR_SEP}*
CLASSPATH=\"${CLASSPATH}\;${SYSTEM_DS_JAR}\"
#echo ${CLASSPATH}

echo "================================================================================"

# Set default Java options
SYSTEMDS_DEFAULT_JAVA_OPTS="\
-Xmx8g -Xms4g -Xmn1g \
-cp $CLASSPATH \
-Dlog4j.configuration=file:'$PROJECT_ROOT_DIR/conf/log4j.properties' \
-Dlog4j.configuration=file:'$PROJECT_ROOT_DIR${DIR_SEP}conf${DIR_SEP}log4j.properties' \
-Duser.dir='$USER_DIR'"

# Add any custom Java options set by the user at command line, overriding defaults as necessary.
Expand All @@ -115,8 +129,8 @@ if [ ! -z "${SYSTEMDS_JAVA_OPTS}" ]; then
fi

# Add any custom Java options set by the user in the environment variables file, overriding defaults as necessary.
if [ -f "${PROJECT_ROOT_DIR}/conf/systemds-env.sh" ]; then
. "${PROJECT_ROOT_DIR}/conf/systemds-env.sh"
if [ -f "${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}systemds-env.sh" ]; then
. "${PROJECT_ROOT_DIR}${DIR_SEP}conf${DIR_SEP}systemds-env.sh"
if [ ! -z "${SYSTEMDS_JAVA_OPTS}" ]; then
SYSTEMDS_DEFAULT_JAVA_OPTS+=" ${SYSTEMDS_JAVA_OPTS}"
fi
Expand All @@ -129,7 +143,6 @@ CMD="\
java ${SYSTEMDS_DEFAULT_JAVA_OPTS} \
org.tugraz.sysds.api.DMLScript \
-help"
# echo ${CMD}
eval ${CMD}
exit 0
}
Expand Down Expand Up @@ -157,7 +170,7 @@ shift
if [ ! -f "$SCRIPT_FILE" ]
then
SCRIPT_FILE_NAME=$(basename $SCRIPT_FILE)
SCRIPT_FILE_FOUND=$(find "$PROJECT_ROOT_DIR/scripts" -name "$SCRIPT_FILE_NAME")
SCRIPT_FILE_FOUND=$(find "$PROJECT_ROOT_DIR${DIR_SEP}scripts" -name "$SCRIPT_FILE_NAME")
if [ ! "$SCRIPT_FILE_FOUND" ]
then
echo "Could not find DML script: $SCRIPT_FILE"
Expand All @@ -175,10 +188,10 @@ java ${SYSTEMDS_DEFAULT_JAVA_OPTS} \
org.tugraz.sysds.api.DMLScript \
-f '$SCRIPT_FILE' \
-exec singlenode \
-config '$PROJECT_ROOT_DIR/conf/SystemDS-config.xml' \
-config '$PROJECT_ROOT_DIR${DIR_SEP}conf${DIR_SEP}SystemDS-config.xml' \
$@"

export HADOOP_HOME=/tmp/systemds
echo "Executing "${CMD}
eval ${CMD}

RETURN_CODE=$?
Expand Down
149 changes: 149 additions & 0 deletions bin/systemds.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
@ECHO OFF
::-------------------------------------------------------------
::
:: Copyright 2019 Graz University of Technology
::
:: Licensed under the Apache License, Version 2.0 (the "License");
:: you may not use this file except in compliance with the License.
:: You may obtain a copy of the License at
::
:: http://www.apache.org/licenses/LICENSE-2.0
::
:: Unless required by applicable law or agreed to in writing, software
:: distributed under the License is distributed on an "AS IS" BASIS,
:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
:: See the License for the specific language governing permissions and
:: limitations under the License.
::
::-------------------------------------------------------------

echo ================================================================================

IF "%~1" == "" GOTO Err
IF "%~1" == "-help" GOTO Msg
IF "%~1" == "-h" GOTO Msg

setLocal EnableDelayedExpansion

SET USER_DIR=%CD%

pushd %~dp0..
SET PROJECT_ROOT_DIR=%CD%
popd

SET BUILD_DIR=%PROJECT_ROOT_DIR%\target
SET HADOOP_LIB_DIR=%BUILD_DIR%\lib
SET DML_SCRIPT_CLASS=%BUILD_DIR%\classes\org\tugraz\sysds\api\DMLScript.class

SET BUILD_ERR_MSG=You must build the project before running this script.
SET BUILD_DIR_ERR_MSG=Could not find target directory "%BUILD_DIR%". %BUILD_ERR_MSG%
SET HADOOP_LIB_ERR_MSG=Could not find required libraries "%HADOOP_LIB_DIR%\*". %BUILD_ERR_MSG%
SET DML_SCRIPT_ERR_MSG=Could not find "%DML_SCRIPT_CLASS%". %BUILD_ERR_MSG%

:: check if the project had been built and the jar files exist
IF NOT EXIST "%BUILD_DIR%" ( echo %BUILD_DIR_ERR_MSG% & GOTO ExitErr )
IF NOT EXIST "%HADOOP_LIB_DIR%" ( echo %HADOOP_LIB_ERR_MSG% & GOTO ExitErr )
IF NOT EXIST "%DML_SCRIPT_CLASS%" ( echo %DML_SCRIPT_ERR_MSG% & GOTO ExitErr )


:: if the present working directory is the project root or the bin folder, then use the temp folder as user.dir
IF "%USER_DIR%" == "%PROJECT_ROOT_DIR%" (
SET USER_DIR=%PROJECT_ROOT_DIR%\temp
ECHO Output dir: "!USER_DIR!"
)
IF "%USER_DIR%" == "%PROJECT_ROOT_DIR%\bin" (
SET USER_DIR=%PROJECT_ROOT_DIR%\temp
ECHO Output dir: "!USER_DIR!"
)


:: if the SystemDS-config.xml does not exist, create it from the template
IF NOT EXIST "%PROJECT_ROOT_DIR%\conf\SystemDS-config.xml" (
copy "%PROJECT_ROOT_DIR%\conf\SystemDS-config.xml.template" ^
"%PROJECT_ROOT_DIR%\conf\SystemDS-config.xml" > nul
echo ...created "%PROJECT_ROOT_DIR%\conf\SystemDS-config.xml"
)

:: if the log4j.properties do not exis, create them from the template
IF NOT EXIST "%PROJECT_ROOT_DIR%\conf\log4j.properties" (
copy "%PROJECT_ROOT_DIR%\conf\log4j.properties.template" ^
"%PROJECT_ROOT_DIR%\conf\log4j.properties" > nul
echo ...created "%PROJECT_ROOT_DIR%\conf\log4j.properties"
)

SET SCRIPT_FILE=%1

:: if the script file path was omitted, try to complete the script path
IF NOT EXIST %SCRIPT_FILE% (
FOR /R "%PROJECT_ROOT_DIR%" %%f IN (%SCRIPT_FILE%) DO IF EXIST %%f ( SET "SCRIPT_FILE_FOUND=%%f" )
)

IF NOT EXIST %SCRIPT_FILE% IF NOT DEFINED SCRIPT_FILE_FOUND (
echo Could not find DML script: "%SCRIPT_FILE%"
GOTO Err
) ELSE (
SET SCRIPT_FILE="%SCRIPT_FILE_FOUND%"
echo DML script: "%SCRIPT_FILE_FOUND%"
)


:: the hadoop winutils
SET HADOOP_HOME=%PROJECT_ROOT_DIR%\target\lib\hadoop

:: add dependent libraries to classpath (since Java 1.6 we can use wildcards)
set CLASSPATH=%PROJECT_ROOT_DIR%\target\lib\*

:: add compiled SystemDS classes to classpath
set CLASSPATH=%CLASSPATH%;%PROJECT_ROOT_DIR%\target\classes

ECHO Classpath: "%CLASSPATH%"

:: remove the DML script file from the list of arguments
:: allow for dml script path with spaces, enclosed in quotes
rem for /f "tokens=1,* delims= " %%a in ("%*") do set DML_OPT_ARGS=%%b
rem for /f tokens^=1^,*^ delims^=^" %%a in ("%*") do set DML_OPT_ARGS=%%b
set ARGS=%*
set DML_OPT_ARGS=!ARGS:%1 =!

echo ================================================================================

:: construct the java command with options and arguments
set CMD=java -Xmx4g -Xms2g -Xmn400m ^
-cp "%CLASSPATH%" ^
-Dlog4j.configuration=file:"%PROJECT_ROOT_DIR%\conf\log4j.properties" ^
-Duser.dir="%USER_DIR%" ^
org.tugraz.sysds.api.DMLScript ^
-f %SCRIPT_FILE% ^
-exec singlenode ^
-config "%PROJECT_ROOT_DIR%\conf\SystemDS-config.xml" ^
%DML_OPT_ARGS%

:: execute the java command
%CMD%

:: if there was an error, display the full java command (in case some of the variable substitutions broke it)
IF ERRORLEVEL 1 (
ECHO Failed to run SystemDS. Exit code: %ERRORLEVEL%
SET LF=^


:: keep empty lines above for the line breaks
ECHO %CMD: =!LF! %
EXIT /B %ERRORLEVEL%
)
GOTO End

:Err
ECHO Wrong Usage. Please provide DML filename to be executed.
GOTO Msg

:Msg
ECHO Usage: systemds.bat ^<dml-filename^> [arguments] [-help]
ECHO Script internally invokes 'java -Xmx4g -Xms4g -Xmn400m -jar SystemDS.jar -f ^<dml-filename^> -exec singlenode -config SystemDS-config.xml [Optional-Arguments]'
GOTO ExitErr

:ExitErr
EXIT /B 1

:End
EXIT /B 0
30 changes: 16 additions & 14 deletions conf/SystemDS-config.xml
Original file line number Diff line number Diff line change
@@ -1,21 +1,23 @@
<!--
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
********************************************************************
*
* Copyright 2019 Graz University of Technology
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
-->
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
********************************************************************
-->


<root>
<!-- local fs tmp working directory-->
Expand Down
30 changes: 15 additions & 15 deletions conf/SystemDS-config.xml.template
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
<!--
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
<!---------------------------------------------------------------
*
* Copyright 2019 Graz University of Technology
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
-->
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* --------------------------------------------------------------->


<root>
<!-- local fs tmp working directory-->
Expand Down
Loading

0 comments on commit ceaa6ea

Please sign in to comment.