Skip to content

Historial de cambios (preliminar)

Jump to bottom
Miguel Ángel Prosper edited this page Feb 17, 2024 · 1 revision

Versión 0.0.0

Características notables:

  • Soporte para proyectos Java genéricos multi-fichero con código fuente codificado en ASCII.
  • Almenaba los meta-datos en una variable para empotrar los en los generados dinámicos.
  • Generaba dinámicamente una plantilla PBS para Java que almacenaba en el proyecto.
  • Generaba diametralmente un ejecutable con el proyecto empotrado.
  • Comprobaba en que maquina estaba para determinar en que fase del procedimiento estaba.
  • Entrada/salida no interactiva, esperaba que terminara para mostrar los registros de salida.
  • Limpiaba el rastro de ejecución en el servidor para no ocupar espacio.

Código fuente:

#!/bin/bash
# Easy remote execution of Java projects in patan.act.uji.es
read -r -d '' METADATA <<EOF
# Source: gist.github.com/MAProsper/a7387ea3b16c491fe987301bfcbea16e
# Author: Miguel Ángel Prosper Quíros (al386125@uji.es)
# License: BSD 3-Clause
EOF

# argument verification
NAME="patan-run"
HELP="run $NAME for help"
if [ $# -lt 3 ]; then
    echo "$NAME user project main [args...]" >&2
    echo "> user: al386125" >&2
    echo "> project: path/to/project_directory" >&2
    echo "> main: package.javaFile.classWithMain" >&2
    exit 1
elif [[ ! "$1" =~ al[0-9]+ ]]; then
    echo "invalid user name: $1 ($HELP)" >&2
    exit 1
elif [ ! -d "$2" ]; then
    echo "invalid project directory: $2 ($HELP)" >&2
    exit 1
fi

# setup variables
USER="$1"; shift
SRC="$1"; shift
TMP="$NAME.$RANDOM"
LYNX="lynx.uji.es"
PATAN="patan.act.uji.es"

# helper functions
escape() { printf ' %q' "$@"; }

# prepare metadata
echo '> Generating archive'
PROJECT="$(escape "$(basename "$SRC")") ($USER)"
EXECUTE="$(escape "$@")"
read -r -d '' METADATA <<EOF
$METADATA

# Project:$PROJECT
# Execute:$EXECUTE
EOF

# prepare project
rm -r "$SRC/target" 2> /dev/null
mkdir -p "$SRC/target/classes"
mkdir -p "$SRC/target/patan"

# dynamically generate PBS embedding arguments
cat <<EOF >"$SRC/target/patan/pbs"
#!/bin/bash
#PBS -q epyc -l nodes=1:ppn=16 -N pbs -j oe
# PBS with Java project compilation and execution
$METADATA

# preapare project
cd "\$PBS_O_WORKDIR/../.."
find 'src/main/java' -name '*.java' | xargs javac -d 'target/classes'

# execute with arguments
java -cp 'target/classes'$EXECUTE
echo "> Exit code: \$?"
EOF

# dynamically generate command embedding project
cat <<EOF >"$TMP"
#!/bin/bash
# Self-extracting PBS Java project for $PATAN
$METADATA

# setup variables
TMP="$NAME.\$RANDOM"
SOURCES=\${#BASH_SOURCE[@]}

# helper functions
source_found() { test \$SOURCES -ne 0; }
source_verify() {
    source_found && return
    echo '$NAME archive: not found' >&2
    exit 1
}

# manage archive
if source_found; then
    chmod u+x "\$0"
    trap 'rm "\$0"' EXIT
fi

# host-dependant actions
case \$(hostname) in
$PATAN)
echo '> Project$PROJECT'
echo '> Executing$EXECUTE'

# extract project
mkdir "\$TMP"
cat <<EOF.BASE64 | base64 -d | tar -zxf - -C "\$TMP"
$(tar -C "$SRC" -zcf - '.' | base64 -w 0; echo)
EOF.BASE64

# queue execution job
cd "\$TMP/target/patan"
qsub 'pbs' >/dev/null

# wait for results
until [ -f pbs.o* ]; do sleep 1; done
cat pbs.o*

# cleanup project
cd - >/dev/null
rm -r "\$TMP" ;;

$LYNX) source_verify
echo '> Transfering to $PATAN'
ssh $USER@$PATAN bash < "\$0" ;;

*) source_verify
echo '> Transfering to $LYNX'
scp -q "\$0" "$USER@$LYNX:\$TMP"
echo "> Execute command ./\$TMP"
ssh $USER@$LYNX ;;
esac
EOF

chmod u+x "$TMP"
exec "./$TMP"

Versión 1.0.0

Características notables:

  • Soporte para proyectos no ASCII mediante conversión previa a la compilación.
  • Ya no almenaba los meta-datos para empotrar los, ya que los generados se limpiaban al terminar.
  • Definía todas las constantes al principio para facilitar el mantenimiento.
  • Sustituía la definición en linea de un ejecutable dinámico por empotrar el mismo ejecutable.
  • Ya no modificaba el proyecto para almenar ficheros auxiliares.
  • Funcionalidad comprobado por otra persona mas.

Código fuente:

#!/bin/bash
# Easy remote execution of Java projects in patan.act.uji.es
# Source: gist.github.com/MAProsper/50d5335ce3cbece0a7c7522572c8f71a
# Author: Miguel Ángel Prosper Quíros (al386125@uji.es)
# License: BSD 3-Clause

# constants
NAME='patan-run'
NETWORK='lynx.uji.es'
SERVER='patan.act.uji.es'
JOB=(-q 'epyc' -l 'nodes=1:ppn=16')

# independant functions
notice() { echo "> $*" >&2; }
escape() { printf ' %q' "$@" | cut -c2-; }

# source check
if [ -z "$BASH_SOURCE" ]; then
	notify 'invalid state: must be on disk'
	exit 1
fi

# variables
HOST="$(hostname)"
TMP_NAME="$NAME.$RANDOM"
SELF="$(readlink -e "$0")"
TMP="$(readlink -f "$TMP_NAME")"

# dependant functions
line() { grep -anxm1 "$*" "$SELF" | cut -d: -f1; }

# dependant variables
EOF=$(line 'exit')

trap 'rm -rf "$SELF" "$TMP".*' EXIT

# generate archive
if [ -z $EOF ]; then
	trap '-' EXIT

	# verify arguments
	HELP="run $NAME for help"
	if [ $# -lt 3 ]; then
		echo "$NAME user project main [args...]" >&2
		notice 'user: al386125'
		notice 'project: path/project_directory'
		notice 'main: package.classWithMain'
		exit 1
	elif [[ ! "$1" =~ al[0-9]+ ]]; then
		notice "invalid user name: $1 ($HELP)"
		exit 1
	elif [ ! -d "$2" ]; then
		notice "invalid project directory: $2 ($HELP)"
		exit 1
	fi

	# preapare varibales
	USER="$1"; shift
	SRC="$1"; shift
	SOF=$(line '')

	{ notice 'Generating archive'
		head -$SOF "$SELF"
		cat <<-EOF
		USER=$(escape "$USER")
		EXECUTE=($(escape "$@"))
		EOF
		tail +$SOF "$SELF"
		cat <<-EOF
		
		exit
		EOF
		tar -C "$SRC" -czf - '.' 2>/dev/null
	} >"$TMP"

	# execute archive
	chmod u+x "$TMP"
	exec "$TMP"

# compile and execute project
elif [ "$PBS_O_HOST" = "$SERVER" ]; then

	# preapare workspace
	trap 'rm -r "$TMP"' EXIT
	mkdir -p "$TMP/"{src,bin}
	cd "$TMP/src"

	# extract and compile project
	tail -n +$(($EOF+1)) "$SELF" | tar -xzf -
	find '.' -name '*.java' -exec native2ascii '{}' '{}' ';' -exec javac -d '../bin' '{}' '+'

	# execute with arguments
	java -cp '../bin' "${EXECUTE[@]}"
	notice "Exit code: $?"

# submit and collect job
elif [ "$HOST" = "$SERVER" ]; then
	notice "Executing ${EXECUTE[@]}"

	# preapare job
	TMP_SELF="$(escape "$SELF")"
	trap 'rm "SELF" "$TMP".*' EXIT
	echo "chmod u+x $TMP_SELF; exec $TMP_SELF" >"$TMP"

	# execute job
	qsub -z "${JOB[@]}" "$TMP"
	until [ -f "$TMP".o* ]; do sleep 1; done
	cat "$TMP".o*; cat "$TMP".e* >&2

# execute on job server
elif [ "$HOST" = "$NETWORK" ]; then
	notice "Transfering to $SERVER"
	trap 'rm "$SELF"' EXIT

	# transfer and execute
	TMP_SSH="$(escape "$TMP_NAME")"
	CMD="cat >$TMP_SSH; chmod u+x $TMP_SSH; exec ./$TMP_SSH"
	ssh "$USER@$SERVER" "$CMD" < "$SELF"

# transfer to network
else
	trap 'rm "$SELF"' EXIT
	chmod u+x "$SELF"
	
	notice "Transfering to $NETWORK"
	scp -q "$SELF" "$USER@$NETWORK:$TMP_NAME"
	
	notice "Execute command ./$TMP_NAME"
	ssh "$USER@$NETWORK"
fi

Versión 2.0.0

Características notables:

  • Conexión directa a patan.act.uji.es mediante el uso de ProxyJump en SSH.
  • Paso de argumentos directo, sin empotrado, debido a la conexión directa.
  • Funcionalidad comprobado por otra persona mas.

Código fuente:

#!/bin/bash
# Easy remote execution of Java projects in patan.act.uji.es
# Source: gist.github.com/MAProsper
# Author: Miguel Ángel Prosper Quíros (al386125@uji.es)
# License: BSD 3-Clause

# constants
NAME='patan-run'
PROXY='lynx.uji.es'
SERVER='patan.act.uji.es'
JOB=(-q 'epyc' -l 'nodes=1:ppn=16')

# helper functions
notice() { echo "> $*" >&2; }
escape() { printf ' %q' "$@" | cut -c2-; }
invalid() { notice "invalid argument: $*"; exit 1; }

# source check
if [ -z "$BASH_SOURCE" ]; then
	notify 'invalid state: must be on disk'
	exit 1
fi

# prepare enviroment
TMP="$NAME.$RANDOM"
EOF=$(grep -anxm1 'exit' "$0" | cut -d: -f1)

# transfer archive
if [ -z $EOF ]; then

	# verify arguments
	if [ $# -lt 3 ]; then
		echo "$NAME user project main [args...]" >&2
		notice 'user: al386125'
		notice 'project: path/project_directory'
		notice 'main: package.classWithMain'
		exit 1
	elif [[ ! "$1" =~ ^al[0-9]+$ ]]; then
		invalid 'user'
	elif [ ! -d "$2" ]; then
		invalid 'project'
	elif [ -z "$3" ]; then
		invalid 'main'
	fi

	# prepare variables
	TMP="$(escape "$TMP")"
	USER="$1"; shift
	SRC="$1"; shift

	# generate and transfer
	notice "Transfering: $(basename "$SRC")"
	CMD="cat >$TMP; chmod u+x $TMP; exec ./$TMP $(escape "$@")"
	ssh -J "$USER@$PROXY" "$USER@$SERVER" "$CMD" < <(
		cat "$0"; echo 'exit'
		tar -C "$SRC" -czf - '.' 2>/dev/null
	)

# submit and collect job
elif [ -z $PBS_O_HOST ]; then

	# prepare job
	notice "Executing: $1"
	trap 'rm "$0" "$TMP"*' EXIT
	echo "exec $(escape "$0" "$@")" >"$TMP"

	# execute job
	qsub -z "${JOB[@]}" "$TMP"
	until [ -f "$TMP".o* ]; do sleep 1; done
	cat "$TMP".o*; cat "$TMP".e* >&2

# compile and execute project
else
	# prepare workspace
	TMP="$(readlink -f "$TMP")"
	trap 'rm -r "$TMP"' EXIT
	mkdir -p "$TMP/"{src,bin}
	cd "$TMP/src"

	# extract and compile project
	tail -n +$((EOF+1)) "$0" | tar -xzf -
	find '.' -name '*.java' -exec javac -d '../bin' '{}' '+'

	# execute with arguments
	java -cp '../bin' "$@"
	notice "Exit code: $?"
fi

Versión 3.0.0

Características notables:

  • Sustituye el código de patan.act.uji.es por un argumento pasado a SSH.
  • Concatena la plantilla PBS, sustituyendo el condicional de maquina por un recorte del propio ejecutable.
  • Soporte para código fuente codificado en UTF8.
  • Entrada/salida interactiva.

Código fuente:

#!/bin/bash
# Easy remote execution of projects in patan.act.uji.es
# Source: gist.github.com/MAProsper/340c14bfebcd8e124fe32303d31c06a3
# Author: Miguel Ángel Prosper Quíros (al386125@uji.es)
# License: BSD 3-Clause

# constants
NAME='patan-run'
PROXY='lynx.uji.es'
SERVER='patan.act.uji.es'

# helper functions
notice() { echo "$NAME: $*"; }
error() { notice "error $*"; exit 1; }
escape() { printf ' %q' "$@" | cut -c2-; }
template() { sed -n "/^#PBS -N $1\>/,/^#PBS\>/p" "$0" | head -n -1; }

# verify state
if [ -z "$BASH_SOURCE" ]; then
	error 'source'
elif [ $# -lt 3 ]; then
	notice 'user type project [args...]'
	echo -e '\tuser\tal386125'
	echo -e '\ttype\tjava, mpi, ...'
	echo -e '\tproject\tpath/project_directory'
	exit 1
elif [[ ! "$1" =~ ^al[0-9]+$ ]]; then
	error 'user'
elif [ -z "$(template "$2")" ]; then
	error 'types'
elif [ ! -d "$3" ]; then
	error 'project'
fi

# prepare variables
USER="$1"; shift
TYPE="$1"; shift
PROJECT="$1"; shift
TMP="$NAME.$RANDOM"
SRC="$TMP/src"
JOB="$TMP/job"
ARGS="$(escape "$@")"
ROOT="$(basename "$PROJECT")"
MSG="$(notice "transfering $TYPE project $ROOT")"

# prepare command
read -rd '' CMD <<EOF
echo $(escape "$MSG")
trap "rm -r $TMP" EXIT; tar -xzf -
PBS_ARGS=$(escape "$ARGS") qsub -I -d $SRC -V -x \$(readlink -e $JOB) | cat
EOF

# generate and transfer
notice "authenticating $PROXY and $SERVER"
exec ssh -J "$USER@$PROXY" "$USER@$SERVER" "$CMD" < <(
	trap "rm $TMP" EXIT; template "$TYPE" >$TMP; chmod u+x $TMP
	tar -czO $TMP --xform="s|^$TMP$|$JOB|" -C "$PROJECT" --xform="s|^\.|$SRC|" '.'
)

#PBS -N java -q epyc -l nodes=1:ppn=16
mkdir '../bin'
find '.' -name '*.java' -exec javac -encoding 'UTF-8' -d '../bin' '{}' '+'
java -cp '../bin' $PBS_ARGS

Versión 4.0.0

Características notables:

  • Soporte para proyectos MPI mono-fichero.
  • Soporte para proyectos gráficos.
  • Extendido concepto de concatenado a todos los fragmentos.
  • Mejora de interfaz y ayuda dinámica por subcomando.
  • Soporte para parámetros PBS dinámicos.
  • Soporte de cancelación con Ctrl + C.

Código fuente:

#!/bin/bash
# Easy remote execution of projects in patan.act.uji.es
# Author: Miguel Ángel Prosper Quíros (al386125@uji.es)
# Source: gist.github.com/MAProsper
# License: BSD 3-Clause

# variables
NAME='patan-run'; ID="$NAME.$RANDOM"
EB=$'\e[1m'; ED=$'\e[2m'; ER=$'\e[0m'
PROXY='lynx.uji.es'; SERVER='patan.act.uji.es'
HF='(\t+[^\t]+)'; HF="s/$HF$HF$HF/$EB\1$ER\2$ED\3$ER/"
SELF="$(readlink -e "$BASH_SOURCE")"; TMP="$(readlink -f "$ID")"

# functions
mktd() { trap 'rm -r "$TMP"' EXIT; mkdir "$TMP"; }
notice() { echo "$EB$NAME$ER: ${*:-$(cat)}" >&2; }
error() { notice "$*"; exit 1; }
embed() { sed '\|^#!/'"${2:-$TYPE}/$1"'$|,/^#!/!d' "$SELF" | head -n -1; }
help() { error < <(embed 'help' "$1" | sed -E "1d; s/# //; $HF" | column -ts $'\t' | sed 's/  \+/  /'); }
script() { embed 'bash' 'bin'; embed "$1" 'bin'; embed "$1"; echo exit; }

#!/bin/help
# user type project [args...]
# 	user	al386125		(any valid student account)
# 	type	java, ...		(project's specific language)
# 	project	path/directory	(project's root directory)
# 	args	...				(type specific arguments)

#!/bin/local 
[ -z "$BASH_SOURCE" ]		&& error 'source must be local'
[ $# -lt 3 ]				&& help 'bin'; TYPE="$2"
[[ ! "$1" =~ ^al[0-9]+$ ]]	&& error 'user not a student account'
[ -z "$(embed 'local')" ]	&& error 'type not supported'
[ ! -d "$3" ]				&& error 'project not found'
source <(embed 'local')

# prepare command and transfer
notice 'authenticating servers'
mkdir -p "$ID/"{src,bin,run}
echo "${@@A}" >"$ID/run/id"
script 'job' >"$ID/run/job"; chmod u+x "$ID/run/job"
script 'remote' >"$ID/run/remote"; chmod u+x "$ID/run/remote"
SSH="ssh -XS $ID/run/ssh -l $1 -J $1@$PROXY"
$SSH -Mfo 'ControlPersist=yes' "$SERVER" 'sleep infinity'
notice 'transfering project'
rsync -ae "$SSH" "$ID/" "$SERVER:$ID"
rsync -ae "$SSH" "$3/" "$SERVER:$ID/src"
$SSH "$SERVER" "$ID/run/remote"
$SSH -qO 'exit' "$SERVER"
rm -r "$ID"
exit

#!/bin/remote
cd "$(dirname "$SELF")/../src"; source '../run/id'
TMP="$(readlink -e '..')"; trap 'rm -r "$TMP"' EXIT

job() { notice 'waiting queue'; qsub "$@" -XI -x "$TMP/run/job" | sed -u '/\r$/!d'; }

#!/bin/job
cd "$PBS_O_WORKDIR"; source '../run/id'
trap 'notice "exit code $?"' EXIT
notice 'executing job'

#!/java/help
# main [args...]
# 	main	package.class	(class path to main method)
# 	args	...				(program arguments)

#!/java/local
[ $# -lt 4 ]	&& help
[ -z "$4" ]		&& error 'main class can not be empty'

#!/java/remote
job -N 'PruebaJAVA' -q 'epyc' -l 'nodes=1:ppn=16'

#!/java/job
find '.' -name '*.java' -exec javac -encoding 'UTF-8' -d '../bin' '{}' '+'
java -cp '../bin' "${@:4}"

#!/mpi/help
# main nodes [args...]
# 	main	main.c	(main file realative to project)
# 	nodes	16		(number of nodes to utilize)
# 	args	...		(program arguments)

#!/mpi/local
[ $# -lt 5 ]					&& help
[ ! -f "$3/$4" ]				&& error 'main file not found'
[[ ! "$5" =~ ^[1-9][0-9]*$ ]]	&& error 'nodes must be positive integer'

#!/mpi/remote
job -N 'PruebaMPI' -q 'bi' -l "nodes=$5:ppn=4"

#!/mpi/job
mpicc "$4" -o '../bin/exe'
uniq "$PBS_NODEFILE" >'../run/node'
mpirun -mca 'btl_tcp_if_include' 'eth0' -np "$5" -machinefile '../run/node' '../bin/exe' "${@:6}"

#!/bin/end

Versión 5.0.0

Características notables:

  • Simplificación y minificación de código.
  • Soporte para instalación y desinstalación, eliminado la necesidad de especificar el usuario.
  • Uso del directorio de trabajo como raíz de proyecto, eliminado la necesidad de especificarlo.
  • Añadido subcomando para solo conectar, útil para depuración manual del proyecto.

Código fuente:

#!/bin/bash -e
# Easy remote execution of projects in patan.act.uji.es
# Author: Miguel Ángel Prosper Quíros (al386125@uji.es)
# Source: gist.github.com/MAProsper
# License: BSD 3-Clause

NAME='patan-run'; TMP="${TMP-.$RANDOM}"; JOB="$TMP/$NAME"
info() { echo "$NAME: $*" >&2; }; err() { info "$*"; exit 1; }

#!/man/patan-run
# user type [args...]	(cd to project's root first)
# 	user	al386125	(skip username if installed)
# 	type	java, ...	(type of action or project)
# 	args	...		(type specific arguments)

#!/bin/patan-run
SERVER="$1@patan.act.uji.es"; PROXY="$1@lynx.uji.es"
TYPE="$2"; ARGS="${@@A}; ${TMP@A}"; SSH="ssh -XS $TMP/ssh -J $PROXY"
EXE="$HOME/.local/bin/$NAME"; LNS="$HOME/.bash_aliases"; LN="alias $NAME='$NAME $1'"
exe() { echo "$ARGS"; tuc 'bin' 'bash'; tuc "$1" "$NAME"; tuc "$1"; }
tuc() { sed "\|^#!/$1/${2:-$TYPE}\>|,/^#!/!d;//d" "$0"; }
man() { err "$(tuc 'man' "$1" | cut -c3-)"; }
cpx() { install -TD "$0" "$1"; }

# validate arguments
[ "$BASH_SOURCE" = "$0" ]	|| err 'must execute directly locally'
[ $# -ge 2 ]			|| man "$NAME"
[[ "$1" =~ ^[a-z0-9]+$ ]]	|| err 'username not valid'
[ "$(tuc bin; tuc srv)" ]	|| err 'type not supported'
. <(tuc 'bin')			## type specific validation

# main fragment
info "$PWD"; trap 'rm -r "$TMP"' EXIT; cpx "$JOB"; exe 'job' >"$_"
info 'authenticating servers'; $SSH -Mo 'ControlPersist=3' "$SERVER" ':'
info 'transferring workspace'; rsync -ae "$SSH" './' "$SERVER:$TMP"
$SSH -t "$SERVER" "$(exe srv)"; exit

#!/srv/patan-run
cd "$TMP"; trap 'rm -r "$PWD"' EXIT
job() { qsub "$@" -IXx "$PWD/$JOB"; }

#!/job/patan-run
cd "$PBS_O_WORKDIR"; trap 'info "exit code $?"' EXIT

#!/bin/install
grep -qxs "$LN" "$LNS" || echo "$LN" >>"$LNS"
cpx "$EXE"; err 'restart system to finish'

#!/bin/remove
[ -f "$LNS" ] && sed -i "/^$LN$/d" "$LNS"
rm -f "$EXE"; err 'removed from system'

#!/srv/shell
[ $# -ge 3 ]	|| $SHELL
$SHELL -c "${*:3}"

#!/man/java
# main [args...]
# 	main	package.class	(main method's class path)
# 	args	...		(program arguments)

#!/bin/java
[ $# -ge 3 ]	|| man
[ "$3" ]	|| err 'main class cannot be empty'

#!/srv/java
job -q 'epyc' -l 'nodes=1:ppn=16'

#!/job/java
find '.' -name '*.java' -exec javac -encoding 'UTF-8' -d "$TMP" '{}' '+'
java -cp "$TMP" "${@:3}"

#!/man/mpi
# main nodes [args...]
# 	main	main.c	(main file's relative path)
# 	nodes	16	(number of nodes to utilize)
# 	args	...	(program arguments)

#!/bin/mpi
[ $# -ge 4 ]		|| man
[ -f "$3" ]		|| err 'main file not found'
[[ "$4" =~ ^[0-9]+$ ]]	|| err 'nodes must be positive integer'

#!/srv/mpi
job -q 'bi' -l "nodes=$4:ppn=4"

#!/job/mpi
EXE="$TMP/exe"; NODE="$TMP/node"
uniq "$PBS_NODEFILE" >"$NODE"; mpicc "$3" -o "$EXE"
mpirun -mca 'btl_tcp_if_include' 'eth0' -np "$4" -machinefile "$NODE" "$EXE" "${@:5}"
Clone this wiki locally