custom_tests_presentation/slides.tex

\pdfminorversion=4

\documentclass{beamer}

\usepackage[T1]{ fontenc }
\usepackage[utf8]{ inputenc }
\usepackage{ lmodern }
\usepackage{ color }
\usepackage{ graphicx }
\graphicspath{ {./pix/} }

\usepackage{ drawstack }

\usetheme{metropolis}

\title{Customs tests in Free Electrons's lab}
\subtitle{}
\date{\today}
\author{Florent (Skia) Jacquet}
\institute{Free Electrons}

\begin{document}

\maketitle

\begin{frame}[fragile]\frametitle{Why custom tests}
    \begin{itemize}
        \item Just booting is not enough.
        \item Need to test some specific components such as USB, SATA,
            network...
        \item Also need to test with custom kernel images, or custom rootfs, or
            both.
        \item Need to be able to send jobs, on demand, easilly, on the specified
            boards, while being notified at the end.
    \end{itemize}
\end{frame}

\section{Overall architecture}
\begin{frame}[fragile]\frametitle{Many tools}
    \begin{itemize}
        \item LAVA
        \item Buildroot
        \item The custom test tool
        \item The test suite
    \end{itemize}
\end{frame}

\begin{frame}[fragile]\frametitle{LAVA}
    \begin{itemize}
        \item Runs jobs on devices.
        \item Runs KernelCI jobs: only boot.
        \item Provides an API to do many thing. Among them: sending jobs and
            querying results.
    \end{itemize}
\end{frame}

\begin{frame}[fragile]{Buildroot}
    \begin{itemize}
        \item Used to make only rootfs.
        \item Is configured to provided the needed tools such as iperf,
            bonnie++, etc...
        \item Builds fresh rootfs every night for all the needed architectures:
            ARMv4, ARMv5, ARMv7, ARMv8.
        \item Stores them in a LAVA-accessible folder.
    \end{itemize}
\end{frame}

\begin{frame}[fragile]{The custom test tool}
    \begin{itemize}
        \item Has jobs templates and knows how to send them.
        \item Can upload custom artifacts useable for tests.
        \item Can ask LAVA to notify given emails addresses when the job ends.
        \item Needs an access to the LAVA API, an SSH access to the LAVA server
            to upload artifacts, and a KernelCI API token to go fetch the latest
            builds if no custom artifacts are provided.
        \item Keeps a \verb$boards.py$ file to store the boards configuration.
    \end{itemize}
\end{frame}

\begin{frame}[fragile]{The test suite}
    \begin{itemize}
        \item Is a bunch of shell scripts that will be run on the devices.
        \item Also needs YAML files to interface with LAVA.
        \item Leaves in one of our git repo on Github \footnote{\url{https://github.com/free-electrons/test_suite}}.
    \end{itemize}
\end{frame}

\begin{frame}[fragile]{Workflow}
    \begin{itemize}
        \item Daily jobs:
            \begin{itemize}
                \item A cron job runs at 23:23 to:
                    \begin{itemize}
                        \item Build fresh rootfs.
                        \item Launch custom tests on every boards with there
                            default configuration.
                    \end{itemize}
                \item Another cron is run at 11:00 to get the results of the
                    last 24 hours and send mails given the failed tests and
                    which email address is subscribed to what board.
            \end{itemize}
        \item On demand jobs:
            \begin{itemize}
                \item Someone installs and configures the custom test tool on
                    his/her laptop.
                \item He/She can send jobs, specifying whether to use custom
                    boot artifacts from his/her laptop, or to use KernelCI's
                    one, and in that case, which branch on which tree to use.
                \item He/She gets notified once the job's done.
            \end{itemize}
    \end{itemize}
\end{frame}

\section{How do tests works?}

\begin{frame}[fragile]{LAVA v2 jobs}
    \begin{itemize}
        \item Jobs are YAML files that, combined with the device-types and
            devices dictionaries in the LAVA configuration, describe what to do
            with which device.
        \item It contains:
            \begin{itemize}
                \item Which device(s) to use
                \item The artifacts informations (URL, type, ...)
                \item How to boot them (Uboot, fastboot, ramdisk, nfs, shell
                    strings to expect, ...)
                \item The tests to run once booted.
                \item Some metadata and various job informations (priority,
                    notifications, timeouts, ...)
            \end{itemize}
    \end{itemize}
\end{frame}

\begin{frame}[fragile]{How do LAVA runs test?}
    \begin{itemize}
        \item Before bringing up the board, LAVA:
            \begin{itemize}
                \item Fetches the artifacts.
                \item Applies the needed modifications (append DTB, mkimage,
                    modules, ...)
                \item Also downloads the test suite and puts it in the rootfs.
                    This moment also makes some magic to provide some helpers
                    useable in the test scripts.
                \item Finally connects itself and powers up the board.
            \end{itemize}
        \item After the board is booted:
            \begin{itemize}
                \item Runs each test described in the job definition.
                \item For each call to the `lava-test-case` function, creates
                    the results objects.
            \end{itemize}
    \end{itemize}

\end{frame}


\begin{frame}{Questions}
    \begin{center}
        \alert{Question?}
    \end{center}
\end{frame}

\end{document}