start_jupyter_nb.sh 22.8 KB
Newer Older
sfux's avatar
sfux committed
1
2
#!/bin/bash

sfux's avatar
sfux committed
3
4
###############################################################################
#                                                                             #
sfux's avatar
sfux committed
5
#  Script to start a jupyter notebook/lab on Euler from a local computer      #
sfux's avatar
sfux committed
6
#                                                                             #
sfux's avatar
sfux committed
7
8
#  Main author    : Samuel Fux                                                #
#  Contributions  : Jarunan Panyasantisuk, Andrei Plamada, Swen Vermeul,      #
9
#                   Urban Borsnik, Steven Armstrong, Henry Lütcke,            #
sfux's avatar
sfux committed
10
#                   Gül Sena Altıntaş, Mikolaj Rybinski, Gerhard Bräunlich    #
sfux's avatar
sfux committed
11
12
13
14
15
#  Date           : December 2018                                             #
#  Location       : ETH Zurich                                                #
#  Version        : 1.0                                                       #
#  Change history :                                                           #
#                                                                             #
brgerhar's avatar
brgerhar committed
16
17
#  19.07.2022    Add option --extra-modules                                   #
#                                                                             #
sfux's avatar
sfux committed
18
19
#  14.02.2022    Adding option -j for julia kernel to the script              #
#                                                                             #
sfux's avatar
sfux committed
20
21
#  14.02.2022    Bringing back bash and R kernels                             #
#                                                                             #
sfux's avatar
sfux committed
22
23
#  04.11.2021    Added support for Jupyter lab                                # 
#                                                                             #
24
25
#  02.11.2021    Added virtual environment support                            #
#                                                                             #
sfux's avatar
sfux committed
26
27
28
29
30
31
32
33
34
#  26.10.2021    The script was rewritten:                                    #
#                * clean up of the code (naming scheme for variables)         #
#                * replaced multiline echo's with heredoc's                   #
#                * added command line parser for options                      #
#                * added option for config file                               #
#                * added option for working directory of the notebook         #
#                * added choice for software stack (old/new)                  #
#                * added more bsub options                                    #
#                                                                             #
sfux's avatar
sfux committed
35
#  23.07.2021    Added partial support for windows 10 with git-bash           #
sfux's avatar
sfux committed
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
#                                                                             #
#  17.08.2020    Added a section with configuration options to specify        #
#                non-standard locations for keys                              #
#                                                                             #
#  02.04.2020    Added reconnect_info file that contains all information to   #
#                reconnect to a notebook                                      #
#                                                                             #
#  01.10.2019    Added bash and R kernels for jupyter notebooks               #
#                                                                             #
#  24.01.2019    Added option to specify cluster on which the notebook is     #
#                executed                                                     #
#                                                                             #
###############################################################################

###############################################################################
# Configuration options, initalising variables and setting default values     #
###############################################################################

# Version
sfux's avatar
sfux committed
55
JNB_VERSION="1.2"
sfux's avatar
sfux committed
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

# Script directory
JNB_SCRIPTDIR=$(pwd)

# hostname of the cluster to connect to
JNB_HOSTNAME="euler.ethz.ch"

# order for initializing configuration options
# 1. Defaults values set inside this script
# 2. Command line options overwrite defaults
# 3. Config file options  overwrite command line options

# Configuration file default    : $HOME/.jnb_config
JNB_CONFIG_FILE="$HOME/.jnb_config"

# Username default              : no default
JNB_USERNAME=""

# Number of CPU cores default   : 1 CPU core
JNB_NUM_CPU=1

# Runtime limit default         : 1:00 hour
JNB_RUN_TIME="01:00"

# Memory default                : 1024 MB per core
JNB_MEM_PER_CPU_CORE=1024

# Number of GPUs default        : 0 GPUs
JNB_NUM_GPU=0

# Waiting interval default      : 60 seconds
JNB_WAITING_INTERVAL=60

# SSH key location default      : no default
JNB_SSH_KEY_PATH=""

# Software stack default        : new
JNB_SOFTWARE_STACK="new"

sfux's avatar
sfux committed
95
96
# Workdir default               : no default
JNB_WORKING_DIR=""
sfux's avatar
sfux committed
97

98
99
100
# Virtual env default           : no default
JNB_ENV=""

sfux's avatar
sfux committed
101
102
103
# jupyter lab default           : empty string (will start a notebook instead of lab)
JNB_JLAB=""

sfux's avatar
sfux committed
104
105
106
# julia kernels                 : FALSE -> no julia kernel; TRUE -> julia kernel
JNB_JKERNEL="FALSE"

brgerhar's avatar
brgerhar committed
107
108
109
# Extra cluster modules         : no additional modules
JNB_EXTRA_MODULES=()

sfux's avatar
sfux committed
110
111
112
113
114
115
###############################################################################
# Usage instructions                                                          #
###############################################################################

function display_help {
cat <<-EOF
sfux's avatar
sfux committed
116
$0: Script to start jupyter notebook/lab on Euler from a local computer
sfux's avatar
sfux committed
117
118
119

Usage: start_jupyter_nb.sh [options]

sfux's avatar
sfux committed
120
Required options:
sfux's avatar
sfux committed
121
122
123
124
125
126

        -u | --username       USERNAME         ETH username for SSH connection to Euler

Optional arguments:

        -c | --config         CONFIG_FILE      Configuration file for specifying options
127
        -e | --environment    ENV              Python virtual environment
sfux's avatar
sfux committed
128
129
130
        -g | --numgpu         NUM_GPU          Number of GPUs to be used on the cluster
        -h | --help                            Display help for this script and quit
        -i | --interval       INTERVAL         Time interval for checking if the job on the cluster already started
sfux's avatar
sfux committed
131
        -j | --julia          BOOL             Start jupyter notebook with (BOOL=TRUE) or without (BOOL=FALSE) julia kernel enabled
sfux's avatar
sfux committed
132
        -k | --key            SSH_KEY_PATH     Path to SSH key with non-standard name
sfux's avatar
sfux committed
133
        -l | --lab                             Start jupyter lab instead of a jupyter notebook
sfux's avatar
sfux committed
134
135
        -m | --memory         MEM_PER_CORE     Memory limit in MB per core
        -n | --numcores       NUM_CPU          Number of CPU cores to be used on the cluster
sfux's avatar
sfux committed
136
137
        -s | --softwarestack  SOFTWARE_STACK   Software stack to be used (old, new)
        -v | --version                         Display version of the script and exit
sfux's avatar
sfux committed
138
        -w | --workdir        WORKING_DIR      Working directory for the jupyter notebook/lab
brgerhar's avatar
brgerhar committed
139
             --extra-modules  EXTRA_MODULES    Load additional cluster modules before starting
sfux's avatar
sfux committed
140
        -W | --runtime        RUN_TIME         Run time limit for jupyter notebook/lab in hours and minutes HH:MM
sfux's avatar
sfux committed
141

brgerhar's avatar
brgerhar committed
142
Examples:
sfux's avatar
sfux committed
143
144
145

        ./start_jupyter_nb.sh -u sfux -n 4 -W 04:00 -m 2048 -w /cluster/scratch/sfux

sfux's avatar
sfux committed
146
147
        ./start_jupyter_nb.sh -u sfxu -n 1 -W 01:00 -m 1024 -j TRUE

sfux's avatar
sfux committed
148
149
150
151
152
153
154
155
156
        ./start_jupyter_nb.sh --username sfux --numcores 2 --runtime 01:30 --memory 2048 --softwarestack new

        ./start_jupyter_nb.sh -c $HOME/.jnb_config

Format of configuration file:

JNB_USERNAME=""             # ETH username for SSH connection to Euler
JNB_NUM_CPU=1               # Number of CPU cores to be used on the cluster
JNB_NUM_GPU=0               # Number of GPUs to be used on the cluster
sfux's avatar
sfux committed
157
JNB_RUN_TIME="01:00"        # Run time limit for jupyter notebook/lab in hours and minutes HH:MM
sfux's avatar
sfux committed
158
159
160
161
JNB_MEM_PER_CPU_CORE=1024   # Memory limit in MB per core
JNB_WAITING_INTERVAL=60     # Time interval to check if the job on the cluster already started
JNB_SSH_KEY_PATH=""         # Path to SSH key with non-standard name
JNB_SOFTWARE_STACK="new"    # Software stack to be used (old, new)
sfux's avatar
sfux committed
162
JNB_WORKING_DIR=""          # Working directory for jupyter notebook/lab
163
JNB_ENV=""                  # Path to virtual environment
sfux's avatar
sfux committed
164
JNB_JLAB=""                 # "lab" -> start jupyter lab; "" -> start jupyter notebook
sfux's avatar
sfux committed
165
JNB_JKERNEL="FALSE"         # "FALSE" -> no Julia kernel; "TRUE" -> Julia kernel
sfux's avatar
sfux committed
166
167
168

EOF
exit 1
sfux's avatar
sfux committed
169
170
}

sfux's avatar
sfux committed
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
###############################################################################
# Parse configuration options                                                 #
###############################################################################

while [[ $# -gt 0 ]]
do
        case $1 in
                -h|--help)
                display_help
                ;;
                -v|--version)
                echo -e "start_jupyter_nb.sh version: $JNB_VERSION\n"
                exit
                ;;
                -u|--username)
                JNB_USERNAME=$2
                shift
                shift
                ;;
                -n|--numcores)
                JNB_NUM_CPU=$2
                shift
                shift
                ;;
                -W|--runtime)
                JNB_RUN_TIME=$2
                shift
                shift
                ;;
                -m|--memory)
                JNB_MEM_PER_CPU_CORE=$2
                shift
                shift
                ;;
                -c|--config)
                JNB_CONFIG_FILE=$2
                shift
                shift
                ;;
210
211
212
213
214
                -e |--environment)
                JNB_ENV=$2
                shift
                shift
                ;;
sfux's avatar
sfux committed
215
216
217
218
219
220
221
222
223
224
                -g|--numgpu)
                JNB_NUM_GPU=$2
                shift
                shift
                ;;
                -i|--interval)
                JNB_WAITING_INTERVAL=$2
                shift
                shift
                ;;
sfux's avatar
sfux committed
225
                -j|--julia)
sfux's avatar
sfux committed
226
227
228
                JNB_JKERNEL=$2
                shift
                shift
sfux's avatar
sfux committed
229
                ;;
sfux's avatar
sfux committed
230
231
232
233
234
                -k|--key)
                JNB_SSH_KEY_PATH=$2
                shift
                shift
                ;;
sfux's avatar
sfux committed
235
236
237
238
                -l|--lab)
                JNB_JLAB="lab"
                shift
                ;;
sfux's avatar
sfux committed
239
                -s|--softwarestack)
sfux's avatar
sfux committed
240
241
242
243
244
245
246
247
248
                JNB_SOFTWARE_STACK=$2
                shift
                shift
                ;;
                -w|--workdir)
                JNB_WORKING_DIR=$2
                shift
                shift
                ;;
brgerhar's avatar
brgerhar committed
249
250
251
252
253
                --extra-modules)
                JNB_EXTRA_MODULES=( $2 )
                shift
                shift
                ;;
sfux's avatar
sfux committed
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
                *)
                echo -e "Warning: ignoring unknown option $1 \n"
                shift
                ;;
        esac
done

###############################################################################
# Check configuration options                                                 #
###############################################################################

# check if user has a configuration file and source it to initialize options
if [ -f "$JNB_CONFIG_FILE" ]; then
        echo -e "Found configuration file $JNB_CONFIG_FILE"
        echo -e "Initializing configuration from file ${JNB_CONFIG_FILE}:"
        cat "$JNB_CONFIG_FILE"
        source "$JNB_CONFIG_FILE"
sfux's avatar
sfux committed
271
272
fi

273
# check that JNB_USERNAME is not an empty string when there is no entry in ~/.ssh/config
sfux's avatar
sfux committed
274
if [ -z "$JNB_USERNAME" ]; then
275
276
277
278
        if ! grep -q "^[Hh][Oo][Ss][Tt] \(.* \)\?euler.ethz.ch\( .*\)\?$" "$HOME/.ssh/config" ; then
            echo -e "Error: No ETH username is specified, terminating script\n"
            display_help
        fi
sfux's avatar
sfux committed
279
280
281
else
        echo -e "ETH username: $JNB_USERNAME"
fi
sfux's avatar
sfux committed
282

sfux's avatar
sfux committed
283
# check if JNB_JLAB is empty
sfux's avatar
sfux committed
284
if [ "$JNB_JLAB" == "lab" ]; then
sfux's avatar
sfux committed
285
286
287
288
289
290
        JNB_START_OPTION="lab"
        echo -e "Using jupyter lab instead of jupyter notebook"
else
        JNB_START_OPTION="notebook"
fi

sfux's avatar
sfux committed
291
# check number of CPU cores
sfux's avatar
sfux committed
292

sfux's avatar
sfux committed
293
# check if JNB_NUM_CPU an integer
sfux's avatar
sfux committed
294
295
296
297
if ! [[ "$JNB_NUM_CPU" =~ ^[0-9]+$ ]]; then
        echo -e "Error: $JNB_NUM_CPU -> Incorrect format. Please specify number of CPU cores as an integer and try again\n"
        display_help
fi
298

sfux's avatar
sfux committed
299
# check if JNB_NUM_CPU is <= 128
sfux's avatar
sfux committed
300
301
302
if [ "$JNB_NUM_CPU" -gt "128" ]; then
        echo -e "Error: $JNB_NUM_CPU -> Larger than 128. No distributed memory supported, therefore the number of CPU cores needs to be smaller or equal to 128\n"
        display_help
303
304
fi

sfux's avatar
sfux committed
305
if [ "$JNB_NUM_CPU" -gt "0" ]; then
sfux's avatar
sfux committed
306
        echo -e "Requesting $JNB_NUM_CPU CPU cores for running the jupyter $JNB_START_OPTION"
sfux's avatar
sfux committed
307
308
309
fi

# check number of GPUs
sfux's avatar
sfux committed
310

sfux's avatar
sfux committed
311
# check if JNB_NUM_GPU an integer
sfux's avatar
sfux committed
312
313
314
315
if ! [[ "$JNB_NUM_GPU" =~ ^[0-9]+$ ]]; then
        echo -e "Error: $JNB_NUM_GPU -> Incorrect format. Please specify the number of GPU as an integer and try again\n"
        display_help
fi
sfux's avatar
sfux committed
316

sfux's avatar
sfux committed
317
# check if JNB_NUM_GPU is <= 8
sfux's avatar
sfux committed
318
if [ "$JNB_NUM_GPU" -gt "8" ]; then
sfux's avatar
sfux committed
319
        echo -e "Error: No distributed memory supported, therefore number of GPUs needs to be smaller or equal to 8\n"
sfux's avatar
sfux committed
320
321
        display_help
fi
sfux's avatar
sfux committed
322

sfux's avatar
sfux committed
323
if [ "$JNB_NUM_GPU" -gt "0" ]; then
sfux's avatar
sfux committed
324
        echo -e "Requesting $JNB_NUM_GPU GPUs for running the jupyter $JNB_START_OPTION"
sfux's avatar
sfux committed
325
326
327
        JNB_SNUM_GPU="-R \"rusage[ngpus_excl_p=$JNB_NUM_GPU]\""
else
        JNB_SNUM_GPU=""
sfux's avatar
sfux committed
328
fi
sfux's avatar
sfux committed
329

sfux's avatar
sfux committed
330
if [ ! "$JNB_NUM_CPU" -gt "0" -a ! "$JNB_NUM_GPU" -gt "0" ]; then
sfux's avatar
sfux committed
331
        echo -e "Error: No CPU and no GPU resources requested, terminating script"
sfux's avatar
sfux committed
332
        display_help
sfux's avatar
sfux committed
333
fi
sfux's avatar
sfux committed
334

sfux's avatar
sfux committed
335
# check if JNB_RUN_TIME is provided in HH:MM format
sfux's avatar
sfux committed
336
337
338
339
340
341
if ! [[ "$JNB_RUN_TIME" =~ ^[0-9][0-9]:[0-9][0-9]$ ]]; then
        echo -e "Error: $JNB_RUN_TIME -> Incorrect format. Please specify runtime limit in the format HH:MM and try again\n"
        display_help
else
    echo -e "Run time limit set to $JNB_RUN_TIME"
fi
sfux's avatar
sfux committed
342

sfux's avatar
sfux committed
343
# check if JNB_MEM_PER_CPU_CORE is an integer
sfux's avatar
sfux committed
344
345
346
if ! [[ "$JNB_MEM_PER_CPU_CORE" =~ ^[0-9]+$ ]]; then
        echo -e "Error: $JNB_MEM_PER_CPU_CORE -> Memory limit must be an integer, please try again\n"
        display_help
sfux's avatar
sfux committed
347
else
sfux's avatar
sfux committed
348
    echo -e "Memory per core set to $JNB_MEM_PER_CPU_CORE MB"
sfux's avatar
sfux committed
349
fi
sfux's avatar
sfux committed
350

sfux's avatar
sfux committed
351
# check if JNB_WAITING_INTERVAL is an integer
sfux's avatar
sfux committed
352
353
354
355
356
357
if ! [[ "$JNB_WAITING_INTERVAL" =~ ^[0-9]+$ ]]; then
        echo -e "Error: $JNB_WAITING_INTERVAL -> Waiting time interval [seconds] must be an integer, please try again\n"
        display_help
else
    echo -e "Setting waiting time interval for checking the start of the job to $JNB_WAITING_INTERVAL seconds"
fi
sfux's avatar
sfux committed
358

sfux's avatar
sfux committed
359
360
# check if JNB_JKERNEL is TRUE or FALSE and if the new software stack is used (julia kernel not supported with the old software stack)
if [ "$JNB_JKERNEL" == "TRUE" ]; then
sfux's avatar
sfux committed
361
        JNB_JULIA="julia/1.6.5"
sfux's avatar
sfux committed
362
363
364
365
366
367
368
369
370
371
372
373
        if [ "$JNB_SOFTWARE_STACK" = "old" ]; then
                echo -e "Error: The Julia kernel is only supported when using the new software stack. Please change the software stack and try again\n"
                display_help
        fi
        echo -e "Enabling Julia kernel"
elif [ "$JNB_JKERNEL" == "FALSE" ]; then
        JNB_JULIA=""
else
        echo -e "Error: \$JNB_JKERNEL can only have the values TRUE or FALSE. Please specify a correct value for the -j/--julia parameter and try again\n"
        display_help
fi

sfux's avatar
sfux committed
374
375
376
377
378
379
380
# check which software stack to use
case $JNB_SOFTWARE_STACK in
        old)
        JNB_MODULE_COMMAND="new gcc/4.8.2 r/3.6.0 python/3.6.1 eth_proxy"
        echo -e "Using old software stack (new gcc/4.8.2 r/3.6.0 python/3.6.1 eth_proxy)"
        ;;
        new)
jarunanp's avatar
jarunanp committed
381
        if [ "$JNB_NUM_GPU" -gt "0" ]; then
sfux's avatar
sfux committed
382
383
            JNB_MODULE_COMMAND="gcc/6.3.0 r/4.0.2 python_gpu/3.8.5 eth_proxy $JNB_JULIA"
            echo -e "Using new software stack (gcc/6.3.0 python_gpu/3.8.5 eth_proxy $JNB_JULIA)"
jarunanp's avatar
jarunanp committed
384
        else
sfux's avatar
sfux committed
385
386
            JNB_MODULE_COMMAND="gcc/6.3.0 r/4.0.2 python/3.8.5 eth_proxy $JNB_JULIA"
            echo -e "Using new software stack (gcc/6.3.0 python/3.8.5 eth_proxy $JNB_JULIA)"
jarunanp's avatar
jarunanp committed
387
        fi  
sfux's avatar
sfux committed
388
389
390
391
392
393
        ;;
        *)
        echo -e "Error: $JNB_SOFTWARE_STACK -> Unknown software stack. Software stack either needs to be set to 'new' or 'old'\n"
        display_help
        ;;
esac
brgerhar's avatar
brgerhar committed
394
JNB_MODULE_COMMAND+=" ${JNB_EXTRA_MODULES[@]}"
sfux's avatar
sfux committed
395

sfux's avatar
sfux committed
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
# check if JNB_SSH_KEY_PATH is empty or contains a valid path
if [ -z "$JNB_SSH_KEY_PATH" ]; then
        JNB_SKPATH=""
else
        JNB_SKPATH="-i $JNB_SSH_KEY_PATH"
        echo -e "Using SSH key $JNB_SSH_KEY_PATH"
fi

# check if JNB_WORKING_DIR is empty
if [ -z "$JNB_WORKING_DIR" ]; then
        JNB_SWORK_DIR=""
else
        JNB_SWORK_DIR="--notebook-dir $JNB_WORKING_DIR"
        echo -e "Using $JNB_WORKING_DIR as working directory"
fi

412
413
# check if JNB_ENV is empty
if [ "$JNB_ENV" != "" ]; then
sfux's avatar
sfux committed
414
        echo "Using $JNB_ENV as python environment"
415
416
fi

sfux's avatar
sfux committed
417
# put together string for SSH options
418
419
420
421
422
if [ -z "$JNB_USERNAME" ]; then
    JNB_SSH_OPT="$JNB_SKPATH $JNB_HOSTNAME"
else
    JNB_SSH_OPT="$JNB_SKPATH $JNB_USERNAME@$JNB_HOSTNAME"
fi
sfux's avatar
sfux committed
423
424
425
###############################################################################
# Check for leftover files                                                    #
###############################################################################
sfux's avatar
sfux committed
426
427

# check if some old files are left from a previous session and delete them
sfux's avatar
sfux committed
428
429

# check for reconnect_info in the current directory on the local computer
sfux's avatar
sfux committed
430
echo -e "Checking for left over files from previous sessions"
sfux's avatar
sfux committed
431
if [ -f "$JNB_SCRIPTDIR/reconnect_info" ]; then
sfux's avatar
sfux committed
432
        echo -e "Found old reconnect_info file, deleting it ..."
sfux's avatar
sfux committed
433
        rm "$JNB_SCRIPTDIR/reconnect_info"
sfux's avatar
sfux committed
434
fi
sfux's avatar
sfux committed
435
436
437

# check for log files from a previous session in the home directory of the cluster
ssh -T $JNB_SSH_OPT <<ENDSSH
438
if [ -f "\$HOME/jnbinfo" ]; then
sfux's avatar
sfux committed
439
        echo -e "Found old jnbinfo file, deleting it ..."
440
        rm "\$HOME/jnbinfo"
sfux's avatar
sfux committed
441
fi
442
if [ -f "\$HOME/jnbip" ]; then
sfux's avatar
sfux committed
443
	echo -e "Found old jnbip file, deleting it ..."
444
        rm "\$HOME/jnbip"
sfux's avatar
sfux committed
445
fi 
sfux's avatar
sfux committed
446
447
ENDSSH

sfux's avatar
sfux committed
448
###############################################################################
sfux's avatar
sfux committed
449
# Start jupyter notebook/lab on the cluster                                   #
sfux's avatar
sfux committed
450
451
###############################################################################

sfux's avatar
sfux committed
452
453
# run the jupyter notebook/lab job on Euler and save ip, port and the token in the files jnbip and jninfo in the home directory of the user on Euler
echo -e "Connecting to $JNB_HOSTNAME to start jupyter $JNB_START_OPTION in a batch job"
sfux's avatar
sfux committed
454
455
# FIXME: save jobid in a variable, that the script can kill the batch job at the end
ssh $JNB_SSH_OPT bsub -n $JNB_NUM_CPU -W $JNB_RUN_TIME -R "rusage[mem=$JNB_MEM_PER_CPU_CORE]" $JNB_SNUM_GPU  <<ENDBSUB
sfux's avatar
sfux committed
456
module load $JNB_MODULE_COMMAND
457
if [ "$JNB_ENV" != "" ]; then echo -e "Activating the $JNB_ENV"; source $JNB_ENV/bin/activate; fi
sfux's avatar
sfux committed
458
export XDG_RUNTIME_DIR=
459
JNB_IP_REMOTE="\$(hostname -i)"
460
echo "Remote IP:\$JNB_IP_REMOTE" >> \$HOME/jnbip
sfux's avatar
sfux committed
461
462
export JNB_RUN_TIME=$JNB_RUN_TIME
export JNB_START_TIME=`date +"%Y-%m-%dT%H:%M:%S%z"`
463
jupyter $JNB_START_OPTION --no-browser --ip "\$JNB_IP_REMOTE" $JNB_SWORK_DIR &> \$HOME/jnbinfo
sfux's avatar
sfux committed
464
465
ENDBSUB

466
# wait until jupyter notebook/lab has started, poll every $JNB_WAITING_INTERVAL seconds to check if $HOME/jnbinfo exists
sfux's avatar
sfux committed
467
# once the file exists and is not empty, the notebook/lab has been startet and is listening
sfux's avatar
sfux committed
468
ssh $JNB_SSH_OPT <<ENDSSH
469
while ! [ -e \$HOME/jnbinfo -a -s \$HOME/jnbinfo ]; do
sfux's avatar
sfux committed
470
        echo 'Waiting for jupyter $JNB_START_OPTION to start, sleep for $JNB_WAITING_INTERVAL sec'
sfux's avatar
sfux committed
471
472
473
        sleep $JNB_WAITING_INTERVAL
done
ENDSSH
sfux's avatar
sfux committed
474

sfux's avatar
sfux committed
475
# get remote ip, port and token from files stored on Euler
sfux's avatar
sfux committed
476
echo -e "Receiving ip, port and token from jupyter $JNB_START_OPTION"
477
478
479
JNB_REMOTE_IP=$(ssh $JNB_SSH_OPT "cat \$HOME/jnbip | grep -m1 'Remote IP' | cut -d ':' -f 2")
JNB_REMOTE_PORT=$(ssh $JNB_SSH_OPT "cat \$HOME/jnbinfo | grep -m1 token | cut -d '/' -f 3 | cut -d ':' -f 2")
JNB_TOKEN=$(ssh $JNB_SSH_OPT "cat \$HOME/jnbinfo | grep -m1 token | cut -d '=' -f 2")
sfux's avatar
sfux committed
480
481

# check if the IP, the port and the token are defined
sfux's avatar
sfux committed
482
if  [[ "$JNB_REMOTE_IP" == "" ]]; then
sfux's avatar
sfux committed
483
484
485
cat <<EOF
Error: remote ip is not defined. Terminating script.
* Please check login to the cluster and check with bjobs if the batch job on the cluster is running and terminate it with bkill.
486
* Please check the $HOME/jnbinfo for logs regarding the failure to identify the remote ip on the cluster
sfux's avatar
sfux committed
487
488
EOF
exit 1
sfux's avatar
sfux committed
489
490
fi

sfux's avatar
sfux committed
491
if  [[ "$JNB_REMOTE_PORT" == "" ]]; then
sfux's avatar
sfux committed
492
493
494
cat <<EOF
Error: remote port is not defined. Terminating script.
* Please check login to the cluster and check with bjobs if the batch job on the cluster is running and terminate it with bkill.
495
* Please check the $HOME/jnbinfo for logs regarding the failure to identify the remote ip on the cluster
sfux's avatar
sfux committed
496
497
EOF
exit 1
sfux's avatar
sfux committed
498
499
fi

sfux's avatar
sfux committed
500
if  [[ "$JNB_TOKEN" == "" ]]; then
sfux's avatar
sfux committed
501
cat <<EOF
sfux's avatar
sfux committed
502
Error: token for the jupyter $JNB_START_OPTION session is not defined. Terminating script.
sfux's avatar
sfux committed
503
* Please check login to the cluster and check with bjobs if the batch job on the cluster is running and terminate it with bkill.
504
* Please check the $HOME/jnbinfo for logs regarding the failure to identify the remote ip on the cluster
sfux's avatar
sfux committed
505
506
EOF
exit 1
sfux's avatar
sfux committed
507
508
fi

sfux's avatar
sfux committed
509
510
511
512
# print information about IP, port and token
echo -e "Remote IP address: $JNB_REMOTE_IP"
echo -e "Remote port: $JNB_REMOTE_PORT"
echo -e "Jupyter token: $JNB_TOKEN"
sfux's avatar
sfux committed
513
514
515

# get a free port on local computer
echo -e "Determining free port on local computer"
sfux's avatar
sfux committed
516
# JNB_LOCAL_PORT=$(python -c 'import socket; s=socket.socket(); s.bind(("",0)); print(s.getsockname()[1]); s.close()')
sfux's avatar
sfux committed
517
# FIXME: check if there is a solution that does not require python (as some Windows computers don't have a usable Python installed by default)
sfux's avatar
sfux committed
518
# if python is not available, one could use
sfux's avatar
sfux committed
519
JNB_LOCAL_PORT=$((3 * 2**14 + RANDOM % 2**14))
sfux's avatar
sfux committed
520
521
# as a replacement. No guarantee that the port is unused, but so far best non-Python solution

sfux's avatar
sfux committed
522
echo -e "Using local port: $JNB_LOCAL_PORT"
sfux's avatar
sfux committed
523

sfux's avatar
sfux committed
524
525
526
527
528
529
530
# put together URL
if [ "$JNB_START_OPTION" == "notebook" ]; then
        JNB_URL=http://localhost:$JNB_LOCAL_PORT/?token=$JNB_TOKEN
else
        JNB_URL=http://localhost:$JNB_LOCAL_PORT/lab?token=$JNB_TOKEN
fi

sfux's avatar
sfux committed
531
# write reconnect_info file
sfux's avatar
sfux committed
532
533
cat <<EOF > $JNB_SCRIPTDIR/reconnect_info
Restart file
sfux's avatar
sfux committed
534
Username          : $JNB_USERNAME
sfux's avatar
sfux committed
535
536
537
538
539
540
541
Remote IP address : $JNB_REMOTE_IP
Remote port       : $JNB_REMOTE_PORT
Local port        : $JNB_LOCAL_PORT
Jupyter token     : $JNB_TOKEN
SSH tunnel        : ssh $JNB_SSH_OPT -L $JNB_LOCAL_PORT:$JNB_REMOTE_IP:$JNB_REMOTE_PORT -N &
URL               : http://localhost:$JNB_LOCAL_PORT/?token=$JNB_TOKEN
EOF
sfux's avatar
sfux committed
542

543
# setup SSH tunnel from local computer to compute node via login node
sfux's avatar
sfux committed
544
# FIXME: check if the tunnel can be managed via this script (opening, closing) by using a control socket from SSH
sfux's avatar
sfux committed
545
echo -e "Setting up SSH tunnel for connecting the browser to the jupyter $JNB_START_OPTION"
sfux's avatar
sfux committed
546
ssh $JNB_SSH_OPT -L $JNB_LOCAL_PORT:$JNB_REMOTE_IP:$JNB_REMOTE_PORT -N &
sfux's avatar
sfux committed
547

548
549
# SSH tunnel is started in the background, pause 5 seconds to make sure
# it is established before starting the browser
sfux's avatar
sfux committed
550
sleep 5
551

sfux's avatar
sfux committed
552
echo -e "Starting browser and connecting it to jupyter notebook"
sfux's avatar
sfux committed
553
echo -e "Connecting to url $JNB_URL"
sfux's avatar
sfux committed
554

sfux's avatar
sfux committed
555
# start local browser if possible
Andrei Plamada's avatar
Andrei Plamada committed
556
if [[ "$OSTYPE" == "linux-gnu" ]]; then
sfux's avatar
sfux committed
557
        xdg-open $JNB_URL
Andrei Plamada's avatar
Andrei Plamada committed
558
elif [[ "$OSTYPE" == "darwin"* ]]; then
sfux's avatar
sfux committed
559
        open $JNB_URL
sfux's avatar
sfux committed
560
elif [[ "$OSTYPE" == "msys" ]]; then # Git Bash on Windows 10
sfux's avatar
sfux committed
561
        start $JNB_URL
sfux's avatar
sfux committed
562
else
sfux's avatar
sfux committed
563
        echo -e "Your operating system does not allow to start the browser automatically."
sfux's avatar
sfux committed
564
        echo -e "Please open $JNB_URL in your browser."
sfux's avatar
sfux committed
565
fi