diff --git a/docs/programmers/tutorials/index.rst b/docs/programmers/tutorials/index.rst index 1677aa46..1dfa1d9f 100644 --- a/docs/programmers/tutorials/index.rst +++ b/docs/programmers/tutorials/index.rst @@ -50,11 +50,18 @@ to extend and adapt the examples provided. `Customizing command-line processing `_ - How to set up command-line argument and option processing in GC3Pie's `SessionBasedScript`:class: + How to set up command-line argument and option processing in + GC3Pie's `SessionBasedScript`:class: + +`Application requirements `_ + + How to specify running requirements for `Application`:class: tasks, + e.g., how much memory is needed to run. `Application control and post-processing `_ - How to check and react on the termination status of a GC3Pie Task/Application. + How to check and react on the termination status of a GC3Pie + Task/Application. A bottom-up introduction to programming with GC3Pie diff --git a/docs/programmers/tutorials/workflows/part05.pdf b/docs/programmers/tutorials/workflows/part05.pdf new file mode 100644 index 00000000..95252628 Binary files /dev/null and b/docs/programmers/tutorials/workflows/part05.pdf differ diff --git a/docs/programmers/tutorials/workflows/part05.tex b/docs/programmers/tutorials/workflows/part05.tex new file mode 100644 index 00000000..1c13d255 --- /dev/null +++ b/docs/programmers/tutorials/workflows/part05.tex @@ -0,0 +1,263 @@ +\documentclass[english,serif,mathserif,xcolor=pdftex,dvipsnames,table]{beamer} +\usetheme{gc3} + +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{babel} + + +\usepackage{gc3} + +\title[Requirements]{% + Application requirements +} +\author[R. Murri, S3IT UZH]{% + Riccardo Murri \texttt{} + \\[1ex] + \emph{S3IT: Services and Support for Science IT} + \\[1ex] + University of Zurich +} +\date{July~11--14, 2016} + + +\begin{document} + +% title frame +\maketitle + + +\begin{frame} + Applications need to allocate computing resources. + For instance, request 4 processors for 8 hours. + + \+ + GC3Pie allows requesting: + \begin{itemize} + \item the number of processors that a job can use, + \item the architecture (32-bit or 64-bit) of these processors, + \item the guaranteed duration of a job, + \item the amount of memory that a job can use (per processor). + \end{itemize} + + \+ + More fine-grained matching is possible, but outside the scope of + this introductory training. +\end{frame} + + +\begin{frame}[fragile] + Resources are requested using additional constructor parameters for + \texttt{Application} objects. + + \+ + The allowed parameters are: + \lstinline|requested_cores|, + \lstinline|requested_architecture|, + \lstinline|requested_walltime|, + \lstinline|requested_memory|. +\end{frame} + + +\begin{frame}[fragile] + \frametitle{Running parallel jobs} + + You request allocation of a certain number of processors using the + \lstinline|requested_cores| parameter: set it to the number of CPU + cores that you want. + + \+ + For example, the following runs the command \texttt{mpixexec + simulator} on 4 processors: + \begin{python} + class ZodsApplication(Application): + # ... + Application.__init__(self, + arguments=['mpiexec', '-n', '4', 'simulator'], + # ... + ~\HL{requested\_cores=4}~) + \end{python} + + \+ + {\small Note that GC3Pie only guarantees the availability of a certain + number of processors; it is your application's responsibility to use + them, e.g., by starting a command using MPI or any other parallel + processing mechanism.} +\end{frame} + + +\begin{frame}[fragile] + \frametitle{Requesting processor architecture} + + If you send the compiled executable along with your application, you + need to select only resources that can run that binary file. + + \+ + The \lstinline|requested_architecture| parameter provides the + choice between \lstinline|gc3libs.Run.Arch.X86_64| (for 64-bit + Intel/AMD computers) and \lstinline|gc3libs.Run.Arch.X86_32| (for + 32-bit ones): + \begin{python} + from gc3libs import Run + class CodemlApplication(Application): + # ... + Application.__init__(self, + arguments=['./codeml.bin'], + inputs = ['/usr/local/bin/codeml', ...] + # ... + ~\HL{requested\_architecture=Run.Arch.X86\_64}~) + \end{python} +\end{frame} + + +\begin{frame}[fragile] + \frametitle{Requesting running time} + + In order to ensure that your job is allotted enough time to run on + the remote computing system, use the \lstinline|requested_walltime| + parameter. + + \+ + \begin{python} + ~\HL{\textbf{from} gc3libs.quantity}~ \ + ~\HL{\textbf{import} minutes, hours, days}~ + + class CodemlApplication(Application): + # ... + Application.__init__(self, + # ... + ~\HL{requested\_walltime=8*hours}~) + \end{python} + + \+ + You \textbf{must} use a \texttt{gc3libs.quantity} multiple for the + \lstinline|requested_walltime| parameter; any other value will be + rejected with an error. +\end{frame} + +\begin{frame}[fragile] + \frametitle{Units of time} + The Python module \texttt{gc3libs.quantity} provides units for + expressing time requirements in days, hours, minutes, seconds. + + \+ + Just multiply the unit by the amount you need: + \begin{python} + >>> an_hour = 1*hours + \end{python} + Or sum the amounts: + \begin{python} + >>> two_days = 1*days + 24*hours + \end{python} + + \+ + GC3Pie will automatically perform the conversions: + \begin{python} + >>> two_hours = 2*hours + >>> another_two_hours = 7200*seconds + >>> two_hours == another_two_hours + True + \end{python} +\end{frame} + + +% \begin{frame} +% \frametitle{CPU time vs wall-clock time} + +% ``CPU time'' is the total time spent by all CPUs in the system +% actually executing code from our job. + +% \+ +% The ``wall-clock time'' (abbr. ``walltime'') is the time that +% passes on a clock from the moment the system starts executing a job +% until the end of that job. +% \end{frame} + + +\begin{frame}[fragile] + \frametitle{Requesting memory} + In order to secure a certain amount of memory for a job, use the + \lstinline|requested_memory| parameter. + + \+ + Example: +\begin{python} + ~\HL{\textbf{from} gc3libs.quantity \textbf{import} GB, MB, kB}~ + class CodemlApplication(Application): + # ... + Application.__init__(self, + # ... + ~\HL{requested\_memory=8*GB}~) +\end{python} + + \+ + Note that \lstinline|requested_memory| expresses the total + memory used by the job! +\end{frame} + +\begin{frame}[fragile] + \frametitle{Units of memory} + The Python module \texttt{gc3libs.quantity} provides units for + expressing memory requirements in kilo-, Mega- and Giga-bytes. + + \+ + Just multiply the unit by the amount you need: +\begin{python} + >>> a_gigabyte = 1*GB + >>> two_megabytes = 2*MB +\end{python} + + \+ + GC3Pie will automatically perform the conversions: + \begin{python} + >>> two_gigabytes = 2*GB + >>> another_two_gbs = 2000*MB + >>> two_gigabytes == another_two_gbs + True + \end{python} +\end{frame} + + +\begin{frame}[fragile] + \frametitle{All together now} + +\begin{python} +from gc3libs.quantity import GB, MB, kB +from gc3libs.quantity import days, hours, minutes + +class CodemlApplication(Application): + # ... + Application.__init__(self, + # ... + requested_cores=1, + requested_memory=2*GB, + requested_walltime=8*hours) +\end{python} + + \+ When several resource requirements are specified, GC3Pie tries to + satisfy \emph{all} of them. \textbf{If this is not possible, task + submission fails and the task stays in state \emph{NEW}.} + +\end{frame} + + +% \begin{frame} +% \begin{exercise}[5.A] +% Modify the \texttt{GrayscaleApp} in +% \href{https://github.com/uzh/gc3pie/blob/master/docs/programmers/tutorials/workflows/solutions/ex2c.py}{ex2c.py} +% so that it requests an impossibly high amount of memory. Re-run +% the script and watch it fail: the application should not be +% submitted and remain in state \emph{NEW}. + +% \+ +% Can you achieve the same result by using other requirement specifiers? +% \end{exercise} +% \end{frame} + + +\end{document} + +%%% Local Variables: +%%% mode: latex +%%% TeX-master: t +%%% End: