diff --git a/HS22_Backups/LICENSE b/HS22_Backups/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..da6ab6cc8f333d7e89a99812866df8f24374d47c --- /dev/null +++ b/HS22_Backups/LICENSE @@ -0,0 +1,396 @@ +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.†The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. + diff --git a/HS22_Backups/README.md b/HS22_Backups/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dd4f01b68017043b10974f11effbcedf23e67b6a --- /dev/null +++ b/HS22_Backups/README.md @@ -0,0 +1,6 @@ +# Backups in GNU/Linux +## Planning and automating backups with Borg + +This folder contains the slides and scripts presented during the "Spotlight 2 - Backups" that took place at ETH on 18 November 2022. + +<a rel="license" href="http://creativecommons.org/licenses/by/4.0/"><img alt="Creative Commons License" style="border-width:0" src="https://i.creativecommons.org/l/by/4.0/88x31.png" /></a><br />This work is licensed under a <a rel="license" href="http://creativecommons.org/licenses/by/4.0/">Creative Commons Attribution 4.0 International License</a>. diff --git a/HS22_Backups/borgbackup.tex b/HS22_Backups/borgbackup.tex new file mode 100644 index 0000000000000000000000000000000000000000..92d9ab7d79aa342553ce486b8a79b5d25045189d --- /dev/null +++ b/HS22_Backups/borgbackup.tex @@ -0,0 +1,584 @@ +\documentclass[aspectratio=1610,handout]{beamer} % Remove handout to generate pdf with overlays +\usetheme[titleformat=regular,progressbar=head,background=light,block=fill]{metropolis} + +\usepackage{hyperref} +\usepackage{booktabs} +\usepackage[normalem]{ulem} +\usepackage{listings} +\usepackage{xcolor} + +\definecolor{codegreen}{rgb}{0,0.6,0} +\definecolor{codegray}{rgb}{0.5,0.5,0.5} +\definecolor{codepurple}{rgb}{0.58,0,0.82} +\definecolor{backcolour}{rgb}{0.95,0.95,0.92} + +\lstdefinestyle{mystyle}{ + backgroundcolor=\color{backcolour}, + commentstyle=\color{codegreen}, + keywordstyle=\color{magenta}, + numberstyle=\tiny\color{codegray}, + stringstyle=\color{codepurple}, + basicstyle=\fontsize{11}{13}\selectfont\ttfamily, + breakatwhitespace=false, + breaklines=true, + captionpos=b, + keepspaces=true, + numbers=left, + numbersep=5pt, + showspaces=false, + showstringspaces=false, + showtabs=false, + tabsize=2 +} +\lstset{style=mystyle} +\lstdefinelanguage{Ini} +{ + columns=fullflexible, + morecomment=[s][\color{codepurple}\bfseries]{[}{]}, + morecomment=[l]{\#}, + morecomment=[l]{;}, + commentstyle=\color{codegreen}\ttfamily, + morekeywords={}, + otherkeywords={=,:}, + keywordstyle={\color{magenta}\bfseries} +} + +%---------------------------------------------------------------------------------------- +% TITLE PAGE +%---------------------------------------------------------------------------------------- + +\title[short title]{Backups in GNU/Linux} +\subtitle{Planning and automating backups with Borg} +\author{Iyán Méndez Veiga} +\date{November 18, 2022} + +%---------------------------------------------------------------------------------------- +% PRESENTATION SLIDES +%---------------------------------------------------------------------------------------- + +\begin{document} + +\begin{frame} +\titlepage +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Plan for this spotlight} +\begin{enumerate} + \item<1-> Why backups? + \item<2-> Planning backups + \item<3-> Efficient backups with Borg + \item<4-> (Other tools) +\end{enumerate} +\end{frame} + +%------------------------------------------------ +\section{Why backups?} +%------------------------------------------------ + +\begin{frame}{Why backups?} +\begin{itemize} + \item<1-> \alert{Information age}: we generate and consume huge amounts of digital data + \item<2-> This number continues to grow year after year ($\sim 60$ ZB in 2020) + \item<3-> We depend on this data in our daily life + \item<4-> We invest a lot of \alert{time and effort} creating (some of) this data + \item<5-> \alert{Murphy's law}: anything that can happen, will eventually happen\\ + \onslide<6->{$\rightarrow$ And loosing our valuable data \underline{can} indeed happen} +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{What can actually happen?} +\onslide<1->{Many things can go wrong.} + +\onslide<2->{From more to less likely (not rigorously)} +\begin{itemize} + \item<3-> \alert{Human error} + \item<4-> Hardware failure + \item<5-> Accidents + \item<6-> Theft + \item<7-> Malware + \item<8-> Natural disasters + \item<9-> \dots +\end{itemize} + +\onslide<10->{$\sim90\%$ of times backups will save us from our own mistakes!} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{However...} +\onslide<1->{\alert{Opportunity cost!} Backups are \underline{not} free. We need:} +\begin{itemize} + \item<2-> Time to plan and execute them + \item<3-> Time to test them + \item<4-> Additional resources (storage, computation time, bandwidth, ...) + \item<5-> \$\$\$ for third-party offsite backups +\end{itemize} +\onslide<6->{Two takeaways:} +\begin{enumerate} + \item<7-> Plan and automate your backups to minimize time, effort and costs + \item<8-> Don't back up \emph{rubbish} to minimize resources +\end{enumerate} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{What is a backup?} +\begin{itemize} + \item<1-> There are many valid definitions out there + \item<2-> But this is what I mean by \textbf{backup} from now on +\end{itemize} +\begin{exampleblock}<3->{Backup} +An \alert{explicit and exact copy} of some data, optionally compressed and/or encrypted, with \alert{several snapshots} corresponding to different points in time that are \alert{easily accessible} +\end{exampleblock} +\begin{itemize} + \item<4-> A simple mirror is not a backup \onslide<5->{{\tiny(no snapshots $\rightarrow$ mistakes will propagate)}} + \item<6-> Storage redundancy (e.g. RAID5) is also not a backup \onslide<7->{{\tiny(no explicit copy and probably no snapshots)}} + \item<8-> A read-only copy of our data with snapshots every day for the last 10 years that we don't know how to restore is definitely not a backup \onslide<9->{{\tiny(not easily accessible)}} + \item<10-> Most cloud providers don't offer you backups (e.g. GoPro cloud) +\end{itemize} +\onslide<11->{About last property: What is worse than not having a backup?}\\ +\onslide<12->{$\rightarrow$ Thinking you have one, only to learn when you need it that you cannot use it} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Types of backup} +\onslide<1->{There are different ways to categorize backups} +\begin{itemize} + \item<2-> Full system backups (system images) vs \alert{data backups} + \item<3-> Full vs incremental vs differential + \item<4-> Discrete vs continuous backups + \item<5-> Onsite vs offsite backups + \item<6-> \dots +\end{itemize} +\onslide<7->{Today we will focus on doing discrete data backups with Borg.}\\ +\onslide<8->{Technically we will do one full backup and later as many incremental/differential as we want} +\end{frame} + +%------------------------------------------------ +\section{Planning backups} +%------------------------------------------------ + +\begin{frame}{General ideas} +\begin{itemize} + \item<1-> If you care about cybersecurity, backups are a must + \item<2-> When you doubt if you are backing up often enough, probably you aren't + \item<3-> Don't think of backups as being there for you \alert{\emph{if}} you ever lose data\\ + $\rightarrow$ Think of them being there for you \alert{\emph{when}} you lose your data + \item<4-> Don't forget Murphy's law\\ + $\rightarrow$ The more you use electronic devices regularly, the more likely you will lose some data eventually +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Proposal to categorize your data} +\onslide<1->{Before using any backup tool, we need to \alert{understand our own data}.}\\ +\onslide<2->{The first step is to categorize the data based on how important it is to us} +\begin{itemize} + \item<3-> \textbf{Irreplaceable}: \onslide<4->{family photos \& videos, encryption keys...} + \item<5-> \alert{\textbf{Very important}}: \onslide<6->{\LaTeX\;source of your PhD thesis...} + \item<7-> \alert{Important}: \onslide<8->{contracts, bills, some emails...} + \item<9-> Dispensable: \onslide<10->{games, movies, music...} + \item<11-> \sout{Garbage}: \onslide<12->{caches, spam, screenshots...} +\end{itemize} +\onslide<13->{\textbf{Exercise}: make a table with all your data and assign each item a category.\\ +Add additional columns if you think it can help you (e.g. size, how often is modified, other ``backup'', device, \dots).}\\ +\onslide<14->{Useful commands: \texttt{du -sh}, \texttt{find -mtime -1}}\\ +\onslide<15->{Useful apps: Filelight} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Example table} +\begin{table} + \begin{tabular}{lccl} + \toprule + Data & Category & Size (GiB) & Modified\\ + \midrule + /home/iyan/.gnupg & Irreplaceable & 0 & Rarely\\ + /home/iyan/Documents & Very important & 132 & Daily\\ + /home/iyan/Pictures & Irreplaceable & 458 & Rarely\\ + /home/iyan/Music & Dispensable & 2538 & Very rarely\\ + Phone (2FA codes, Signal, etc.) & Important & 1 & Daily\\ + me@iyanmv.com & Very important & 2 & Daily\\ + \multicolumn{1}{c}{\vdots} & \vdots & \vdots & \vdots\\ + \bottomrule + \end{tabular} + \end{table} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Backup frequency} +\begin{itemize} + \item<1-> Different data is modified at different pace + \item<2-> With typical backup tools we should optimize this to avoid wasting storage\\ + $\rightarrow$ e.g. Increase time between differential backups + \item<3-> With Borg not that important because of the efficient \alert{deduplication} + \item<4-> For most people, \alert{one backup per day} should be enough + \item<5-> If you have data that is both irreplaceable and updated very often (e.g. your current work project), you can consider backing that folder every few hours +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{The 3-2-1 strategy} +\begin{itemize} + \item<2-> Similar to investing, \emph{diversify} your backups + \item<3-> At least \alert{3 copies} of your data + \item<4-> \alert{2 local} (on-site) but on different devices {\tiny(external HDD, NAS, ...)} + \item<5-> At least \alert{1 off-site} copy {\tiny(cloud provider, friend's computer, ...)} + \item<6-> Follow this rule at least for irreplaceable and very important data +\end{itemize} +\end{frame} + +%------------------------------------------------ +\section{Backups with Borg} +%------------------------------------------------ + +\begin{frame}{A little bit about Borg} +\begin{itemize} + \item<1-> \alert{\textbf{BorgBackup}} (or simply Borg) is a deduplicating backup program + \item<2-> Written in Python and C (Cython) + \item<3-> It is free software (BSD license) + \item<4-> Officially supports GNU/Linux, macOS and FreeBSD + \item<5-> Also experimental support for Cygwin and WSL + \item<6-> Space efficient storage: chunk-based \alert{deduplication} + \item<7-> Supports compression: lz4, zlib, lzma, \alert{zstd} + \item<8-> Supports (authenticated) encryption: AES256 + \item<9-> Supports efficient offsite backups: delta transfers via SSH + \item<10-> Big changes coming soon with Borg 2.x (crypto, repos, compression) +\end{itemize} +\begin{center} +\onslide<11->{\href{https://www.borgbackup.org/}{\alert{https://www.borgbackup.org}}\\ +\href{https://borgbackup.readthedocs.io}{\alert{https://borgbackup.readthedocs.io}}} + +\end{center} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Who uses Borg?} +\begin{itemize} + \item<1-> FSFE + \item<2-> Archlinux + \item<3-> ... + \item<4-> You? +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{How can I install Borg?} +\onslide<1->{Easy! Almost all distros have Borg in their repos now} +\begin{itemize} +\item<2-> Debian/Ubuntu: \lstinline|sudo apt-get install borgbackup| +\item<2-> Fedora: \lstinline|sudo dnf install borgbackup| +\item<2-> Arch Linux: \lstinline|sudo pacman -S borg| +\item<2-> Gentoo: \lstinline|emerge borgbackup| +\item<2-> OpenSUSE: \lstinline|sudo zypper in borgbackup| +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Attack model} +\begin{itemize} + \item<1-> Environment of the \alert{client process is trusted} + \item<2-> Repository (server) is not trusted + \item<3-> The attacker has any and all access to the repository, including interactive manipulation (man-in-the-middle) for remote repositories + \item<4-> Under this mode, Borg guarantees that an attacker cannot: + \begin{enumerate} + \item<5-> Modify the data of any archive without the client detecting the change + \item<6-> Rename, remove or add an archive without the client detecting the change + \item<7-> Recover plain-text data + \item<8-> Recover definite (heuristics based on access patterns are possible) structural information such as the object graph (which archives refer to what chunks) + \end{enumerate} + \item<9-> The attacker can always impose a denial of service + \item<10-> Borg fails to provide confidentiality if multiple clients independently update the same repository +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[fragile]{Commands we will see today} +\onslide<1->{We always run Borg in the same way} +\onslide<2->\begin{lstlisting} +borg [common options] command [options] ARCHIVE [PATH...] +\end{lstlisting} +\onslide<3->{With common options we can control how the output looks (e.g. enable debug messages), set a network upload rate limit, etc.} + +\onslide<4->{These are the commands we will see today:} +{\tiny +\begin{itemize} + \item<4-> \lstinline|init| + \item<4-> \lstinline|create| + \item<4-> \lstinline|info| + \item<4-> \lstinline|list| + \item<4-> \lstinline|prune| + \item<4-> \lstinline|compact| + \item<4-> \lstinline|check| + \item<4-> \lstinline|mount| +\end{itemize}} + +\onslide<5->{Once we see how to use these, you can check the docs and explore the remaining commnands} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[fragile]{Creating our first backup with Borg} +\onslide<1->{The first thing we need to do is initialize a repository.}\\ +\onslide<2->{This is the directory where Borg will store all the deduplicated data, some metadata and your (encrypted) encryption keys} +\onslide<3->\begin{lstlisting} +borg init --encryption MODE \ + --append-only \ + --storage-quota QUOTA \ + --make-parent-dirs /path/to/repo +\end{lstlisting} +\onslide<4->{The encryption MODE can be: none, authenticated, authenticated-blake2, repokey, keyfile, repokey-blake2 or keyfile-blake2.\\ \alert{Note}: The MODE can only be configured when creating a new repository.} + +\onslide<5->{The QUOTA (e.g. 1.5T, 500G) can be changed later. For example, to remove any limit} +\onslide<6->\begin{lstlisting} +borg config /path/to/repo storage_quota 0 +\end{lstlisting} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[fragile]{Creating our first backup with Borg} +\onslide<1->{Once we have initialized a repository, we can create our first backup} +\onslide<2->\begin{lstlisting} +borg create /path/to/repo::ARCHIVE-NAME path/to/files +\end{lstlisting} +\begin{itemize} + \item<3-> The archive name must be unique + \item<4-> Borg defines some useful variables (e.g. \{now\}, \{hostname\}, \{user\}, ...) + \item<5-> Some examples: +\onslide<6->\begin{lstlisting} +borg create /path/to/repo::my-documents ~/Documents +borg create --compression auto,lzma,9 \ + /path/to/repo::desktop ~/Desktop +borg create ssh://borg@backup.example.org:2222\ + /path/to/repo::{fqdn}-home-{now} /home +\end{lstlisting} +\item<7-> Many useful specific options $\rightarrow$ Check the docs! +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{And our second, third...} +\begin{itemize} + \item<1-> Try to run a few times the same create command + \item<2-> Change filenames, move files around, etc. + \item<3-> Modify binary files + \item<4-> etc. +\end{itemize} +\onslide<5->{New backups only include chunks that are not already in the repository.}\\ +\onslide<6->{$\rightarrow$ We avoid having duplicates $\Rightarrow$ \alert{Deduplication}} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Deduplication?} +\begin{itemize} + \item<1-> Deduplication is a method to save storage at the cost of some additional computation (e.g. computing and checking hashes, instead of just checking filenames and modification times) + \item<2-> Borg does a \alert{chunk-based deduplication} (way more efficient than file-based) + \item<3-> Each file is split into a number of variable length chunks + \item<4-> Only chunks that have never been seen before are added to the repository +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[fragile]{Querying information about our repo} +\begin{lstlisting} +borg info /path/to/repo +borg --last N /path/to/repo +borg info /path/to/repo::archive +\end{lstlisting} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[fragile]{Listing archives in a repo} +\begin{lstlisting} +borg list /path/to/repo +\end{lstlisting} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[fragile]{Pruning} +\begin{itemize} + \item<1-> If storage is limited, we will run out of space eventually + \item<2-> It's nice to have tons of snapshots, but do we really need them? + \item<3-> A good compromise is to define a \alert{retention policy} + \item<4-> For example, we can decide to keep one snapshot per day for the last two weeks, one snapshot per week for a month, and beyond that only one snapshot per month + \item<6-> All this can be achieved with \lstinline|prune| (enforce retencion policy) and \lstinline|compact| (free space) commands +\end{itemize} +\onslide<5->\begin{lstlisting} +borg prune --keep-daily 14 --keep-weekly 4 --keep-monthly -1 +\end{lstlisting} +\onslide<7->\begin{lstlisting} +borg --progress compact --cleanup-commits /path/to/repo +\end{lstlisting} +\onslide<8->{\alert{Note}: Compactation is only possible if the the (local) repository is not append only. Cloud providers may give you the option to compact repos automatically and connect with append-only users.} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[fragile]{Checking integrity} +\begin{itemize} + \item<1-> It is a good idea to verify the integrity of our repository once in a while (e.g. once per month) + \item<2-> Errors can happen due to unreliability hardware (e.g. bad memory) + \item<3-> Errors detected early can be fixed + \item<4-> If we never check/correct errors, these will propagate and may lead to data loss eventually + \item<5-> It is good practice to \alert{check and compact} repos before and after a \alert{major Borg update} +\end{itemize} +\onslide<6->\begin{lstlisting} +borg --progress --verbose check /path/to/repo +borg --progress --verbose check --verify-data /path/to/repo +borg --progress --verbose check --repair /path/to/repo +\end{lstlisting} +\onslide<7->{\alert{Note}: Repair flag can result in data loss if errors can't be fixed. Do a backup of your repo (\texttt{borg export-tar}) and ask for help (IRC, email list) before trying anything if you don't understand what you are doing.} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[fragile]{Restoring data from backups} +\begin{itemize} + \item<1-> Remember: it is not a backup if we cannot access the snapshots easily + \item<2-> Two commands to restore data from a Borg repository + \begin{enumerate} + \item<3-> \lstinline|mount|: convenient to explore your snapshots but slow + \item<5-> \lstinline|extract|: fast and efficient if you know exactly what you want + \end{enumerate} +\end{itemize} +\onslide<4->\begin{lstlisting} +borg mount /path/to/repo /mnt/borg_restore +cd /mnt/borg_restore +borg umount /mnt/borg_restore +\end{lstlisting} +\onslide<6->\begin{lstlisting} +mkdir borg_restore +cd borg_restore +borg list /path/to/repo +borg extract /path/to/repo::ARCHIVE path/to/extract +\end{lstlisting} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Final comments about Borg} +\begin{itemize} + \item<1-> Careful with running out of space\\ + $\rightarrow$ Either \alert{configure quotas} or use LVM logical volumes + \item<2-> To avoid permission issues, \alert{always access the repository using the same user account} + \item<3-> Run with an unprivileged user whenever is possible + \item<4-> Careful with files changing during the creation of an archive (e.g. databases, VMs, containers) + \item<5-> \alert{Block automatic updates} (at least major ones) from your distribution and read the release notes before manually updating and creating a new archive +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Cloud providers compatible with Borg} +\begin{itemize} + \item<2-> \alert{BorgBase}: 10 GiB free, \$24/year 100 GiB, \$80/year 1 TiB, \$150/year 2 TiB + \item<3-> rsync.net: \$18/year 100 GiB, \$96/year 1 TiB (up to 99 TiB: 0.8 ct / GiB / Month) + \item<4-> Hetzner: 46€/year 1TiB, 156€/year 5 TiB +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Automating everything} +The goal is to set up this once and then forget about it. +\begin{itemize} + \item<2-> Two scripts: one to create and prune backups, and one to check them + \item<3-> Two simple systemd services + \item<4-> Two systemd timers +\end{itemize} +\onslide<5->{I've been using this setup for almost three years now, and no issues so far.} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[allowframebreaks]{Script (create off-site backups)} +\lstinputlisting[language=Bash,basicstyle=\fontsize{7}{9}\selectfont\ttfamily]{files/borg-create.sh} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[allowframebreaks]{Script (check off-site backups)} +\lstinputlisting[language=Bash,basicstyle=\fontsize{7}{9}\selectfont\ttfamily]{files/borg-check.sh} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Systemd services and timers} +\lstinputlisting[language=Ini,basicstyle=\fontsize{7}{9}\selectfont\ttfamily]{files/borg.service} +\lstinputlisting[language=Ini,basicstyle=\fontsize{7}{9}\selectfont\ttfamily]{files/borg.timer} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Systemd services and timers} +\lstinputlisting[language=Ini,basicstyle=\fontsize{7}{9}\selectfont\ttfamily]{files/borg-check.service} +\lstinputlisting[language=Ini,basicstyle=\fontsize{7}{9}\selectfont\ttfamily]{files/borg-check.timer} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[fragile]{Installing and enabling systemd units} +\begin{enumerate} + \item<1-> Copy scripts + \item<2-> Adapt them to your needs + \item<3-> Copy systemd units to \texttt{~/.config/systemd/user} + \item<4-> Edit them + \item<5-> Enabled them with \texttt{s} + \onslide<6->\begin{lstlisting} +systemctl --user enable borg.timer --now +systemctl --user enable borg-check.timer --now +\end{lstlisting} +\end{enumerate} +\end{frame} + +%------------------------------------------------ + +\begin{frame}{Syncing for centralized backups} +\begin{itemize} + \item<1-> You can use borg on multiple devices all accessing the same repository + \item<2-> You can use the \texttt{\{hostname\}} or \texttt{\{fqdn\}} variables to create archives + \item<3-> But you can also synchronize (e.g. with syncthing) your devices and always run Borg on the same device +\end{itemize} +\end{frame} + +%------------------------------------------------ +\section{Other tools} +%------------------------------------------------ + +\begin{frame}{Other tools} +Borg is awesome but there are other great tools out there +\begin{itemize} + \item<1-> borgmatic + \item<2-> restic + \item<3-> bacula + \item<4-> bup + \item<5-> ... +\end{itemize} +\end{frame} + +%------------------------------------------------ + +\begin{frame}[standout] +Thank you! Question? +\end{frame} + +\end{document} diff --git a/HS22_Backups/borgbackup_handout.pdf b/HS22_Backups/borgbackup_handout.pdf new file mode 100644 index 0000000000000000000000000000000000000000..9dcb6a64e7ab574783f45a40c8fc3ac8e3f264a8 Binary files /dev/null and b/HS22_Backups/borgbackup_handout.pdf differ diff --git a/HS22_Backups/borgbackup_presentation.pdf b/HS22_Backups/borgbackup_presentation.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0ea7606968bfca3c5a8eac605db092946600d20f Binary files /dev/null and b/HS22_Backups/borgbackup_presentation.pdf differ diff --git a/HS22_Backups/files/borg-check.service b/HS22_Backups/files/borg-check.service new file mode 100644 index 0000000000000000000000000000000000000000..21b882e488562bdd1a4a538cb66a4bf59ef34049 --- /dev/null +++ b/HS22_Backups/files/borg-check.service @@ -0,0 +1,11 @@ +# borg-check.service +[Unit] +Description=Check backup on Borgbase repo +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +Restart=no +LogRateLimitIntervalSec=0 +ExecStart=/bin/bash /home/iyan/Documents/.backup_scripts/borgbase-check.sh diff --git a/HS22_Backups/files/borg-check.sh b/HS22_Backups/files/borg-check.sh new file mode 100755 index 0000000000000000000000000000000000000000..824e53622e99a7b55747d957b1badfc41cde21fe --- /dev/null +++ b/HS22_Backups/files/borg-check.sh @@ -0,0 +1,38 @@ +#!/bin/bash -ue + +# Colors +NC='\e[0m' +GR='\e[1;32m' +SP='\e[105m' + +# This is the location of the BorgBase repository +TARGET=xxx.repo.borgbase.com:repo + +# Options for borg create +BORG_OPTS="-v" + +# Repo key passphrase +export BORG_PASSPHRASE="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + +# It is better to just fail quickly instead of hanging. +export BORG_RELOCATED_REPO_ACCESS_IS_OK=no +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no + +# Log Borg version +echo -e "${SP}$(borg --version)${NC}" + +echo -e "${GR}$(date '+%Y-%m-%d %H:%M')${NC} Starting to check repo" + +borg check $BORG_OPTS $TARGET \ +2>&1 | tee ~/.backup_check_last.log + +echo -e "${GR}$(date '+%Y-%m-%d %H:%M')${NC} Check completed" + +cat <<EOF > ~/.backup_check_last.mail +To: iyanmv@gmail.com +From: Borg <me@iyanmv.com> +Subject: [BorgBase] Check report $(date '+%Y-%m-%d %H:%M') + +EOF +cat ~/.backup_check_last.log >> ~/.backup_check_last.mail +cat ~/.backup_check_last.mail | msmtp -a iyanmv iyanmv@gmail.com diff --git a/HS22_Backups/files/borg-check.timer b/HS22_Backups/files/borg-check.timer new file mode 100644 index 0000000000000000000000000000000000000000..3ee34b2c3c939ebcf4fef1be04ed965ef74187f5 --- /dev/null +++ b/HS22_Backups/files/borg-check.timer @@ -0,0 +1,11 @@ +# borg-check.timer +[Unit] +Description=Run check on Borgbase repo + +[Timer] +OnCalendar=*-*-15 11:00:00 +Persistent=true +RandomizedDelaySec=60 + +[Install] +WantedBy=timers.target diff --git a/HS22_Backups/files/borg-create.sh b/HS22_Backups/files/borg-create.sh new file mode 100755 index 0000000000000000000000000000000000000000..aefe5ae190a70cb44e2ed97a8184eb28bf313593 --- /dev/null +++ b/HS22_Backups/files/borg-create.sh @@ -0,0 +1,65 @@ +#!/bin/bash -ue + +# Colors +NC='\e[0m' +GR='\e[1;32m' +SP='\e[105m' + +# This is the location of the BorgBase repository +TARGET=xxx.repo.borgbase.com:repo + +# Options for borg create +BORG_OPTS="--stats --compression auto,zstd,6" + +# Repo key passphrase +export BORG_PASSPHRASE="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + +# It is better to just fail quickly instead of hanging. +export BORG_RELOCATED_REPO_ACCESS_IS_OK=no +export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no + +# Log Borg version +echo -e "${SP}$(borg --version)${NC}" + +echo -e "${GR}$(date '+%Y-%m-%d %H:%M')${NC} Starting backup" + +borg create $BORG_OPTS \ + $TARGET::{hostname}-{now} \ + ~/.gnupg \ + ~/.ssh \ + ~/.zshrc \ + ~/.zsh_custom \ + ~/.zshenv \ + ~/.bashrc \ + ~/.certs \ + ~/Documents \ + ~/Pictures \ +2>&1 | tee ~/.backup_last.log + +echo -e "${GR}$(date '+%Y-%m-%d %H:%M')${NC} Backup completed" + +echo -e "${GR}$(date '+%Y-%m-%d %H:%M')${NC} Starting to prune old archives" + +borg prune -v --list \ + --glob-archives='{hostname}-*' \ + --keep-daily=7 \ + --keep-weekly=4 \ + --keep-monthly=-1 $TARGET \ +2>&1 | tee -a ~/.backup_last.log + +echo -e "${GR}$(date '+%Y-%m-%d %H:%M')${NC} Prune completed" + +#echo -e "${GR}$(date '+%Y-%m-%d %H:%M')${NC} Compacting repository" +# +#borg -v compact --cleanup-commits $TARGET 2>&1 | tee -a ~/.backup_last.log +# +#echo -e "${GR}$(date '+%Y-%m-%d %H:%M')${NC} Compacting completed" + +cat <<EOF > ~/.backup_last.mail +To: iyanmv@gmail.com +From: Borg <me@iyanmv.com> +Subject: [BorgBase] Backup report $(date '+%Y-%m-%d %H:%M') + +EOF +cat ~/.backup_last.log >> ~/.backup_last.mail +cat ~/.backup_last.mail | msmtp -a iyanmv iyanmv@gmail.com diff --git a/HS22_Backups/files/borg.service b/HS22_Backups/files/borg.service new file mode 100644 index 0000000000000000000000000000000000000000..e510e7ccb3bd46ccc591728a3044b7a647c68df7 --- /dev/null +++ b/HS22_Backups/files/borg.service @@ -0,0 +1,11 @@ +# borg.service +[Unit] +Description=Backup to Borgbase repo +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +Restart=no +LogRateLimitIntervalSec=0 +ExecStart=/bin/bash /home/iyan/Documents/.backup_scripts/borgbase-create.sh diff --git a/HS22_Backups/files/borg.timer b/HS22_Backups/files/borg.timer new file mode 100644 index 0000000000000000000000000000000000000000..4ea2f2490af9bfd47452531c78f3d3da3364d797 --- /dev/null +++ b/HS22_Backups/files/borg.timer @@ -0,0 +1,10 @@ +# borg.timer +[Unit] +Description=Run Borgbase backup + +[Timer] +OnCalendar=*-*-* 22:00:00 +Persistent=false + +[Install] +WantedBy=timers.target diff --git a/HS22_ConsoleToolkit/Figures/horrorGui.png b/HS22_ConsoleToolkit/Figures/horrorGui.png new file mode 100644 index 0000000000000000000000000000000000000000..65272bc02f05b5cd9425db2fb2b73957a45f731c Binary files /dev/null and b/HS22_ConsoleToolkit/Figures/horrorGui.png differ diff --git a/HS22_ConsoleToolkit/code.sty b/HS22_ConsoleToolkit/code.sty new file mode 100644 index 0000000000000000000000000000000000000000..d20af0955136da4dce7f7054893b4bc4a3fd0992 --- /dev/null +++ b/HS22_ConsoleToolkit/code.sty @@ -0,0 +1,77 @@ +\ProvidesPackage{code} + +\usepackage{tcolorbox} +\usepackage{url} + + +\usepackage{tikz} +\usetikzlibrary{tikzmark,calc,,arrows,shapes,decorations.pathreplacing} +\tikzset{every picture/.style={remember picture}} + +\tcbuselibrary{skins,xparse,listings} + +% Props to https://tex.stackexchange.com/questions/517976/drawing-realistic-linux-command-shell-windows-with-tcolorbox + +\tcbset{common/.style={ + colback=violet!50!black, + colupper=white, + colframe=gray!65!black, + listing only, + listing options={ + style=tcblatex, + language=sh,escapeinside=``, + }, + every listing line={\MyMainPrompt}, +}} + +\newtcblisting{shell}[1][]{ + common, + title={\textcolor{orange}{\Large{$\bullet$}}{\textcolor{gray}{\Large{$\bullet\bullet$}}}}, + codewidth, + fontsize, + #1 +} + +\newtcblisting{shellcomp}[1][]{ + common, + codewidth, + fontsize, + #1 +} + +\tcbset{ + codewidth/.style={width=#1\textwidth}, + codewidth/.default=1, + fontsize/.style={% + listing options={ % TODO add only relevant stuff here (do not overwrite unrelated default) + style=tcblatex, + language=sh,escapeinside=``, + basicstyle=\ttfamily#1, + }, + }, + % fontsize/.default=\normalsize, +} + +% \BeforeBeginEnvironment{shell}{\centering} + +\pgfkeys{/shell/.cd, + user/.code={\gdef\MyMainUser{#1}}, + user={}, + host/.code={\gdef\MyMainHost{#1}}, + host={}, + color/.code={\gdef\MyMainColor{#1}}, + color=white, + pwd/.code={\gdef\MyMainPwd{#1}}, + pwd=\~, + prompt char/.code={\gdef\MyMainPromptChar{#1}}, + prompt char=\#, + root/.style={user=root,host=ubuntu,color=lime,pwd=/,prompt char=\#}, + bob/.style={user=bob,host=ubuntu,color=cyan,pwd=\~,prompt char=\$}, + bobremote/.style={user=bob,host=myRemote,color=green,pwd=\~,prompt char=>}, + alice/.style={user=alice,host=arch,color=green,pwd=\~,prompt char=>}, +} +\newcommand{\SU}[1]{ + \pgfkeys{/shell/.cd,#1}% + \gdef\MyMainPrompt{\textcolor{\MyMainColor}{\MyMainUser @\MyMainHost{\textcolor{white}:}\textcolor{cyan!60}{\url{\MyMainPwd}}{\textcolor{white}\MyMainPromptChar} }} +} +\newcommand{\StartConsole}{\gdef\MyMainPrompt{}} diff --git a/HS22_ConsoleToolkit/general.sty b/HS22_ConsoleToolkit/general.sty new file mode 100644 index 0000000000000000000000000000000000000000..e92a06823a2ad3c3a0112a71e02099664cc7e32c --- /dev/null +++ b/HS22_ConsoleToolkit/general.sty @@ -0,0 +1,22 @@ +\ProvidesPackage{general} + +\usepackage[T1]{fontenc} +\usepackage[utf8]{inputenc} +\usepackage{xcolor} +\usepackage{multicol} +\usepackage{hyperref} +\usepackage{lmodern} +\usepackage{listings} +\usepackage[os=win]{menukeys} +\usepackage{tcolorbox} + +\usefonttheme[onlymath]{serif} +\setcounter{MaxMatrixCols}{20} + +\newcommand{\todo}[1]{\textcolor{red}{TODO: #1}\PackageWarning{TODO:}{#1!}} + +% a red warning box +\definecolor{lred}{HTML}{ffd6dd} +\newtcolorbox{WarningBox}{% + colframe=red, + colback=lred} diff --git a/HS22_ConsoleToolkit/main.pdf b/HS22_ConsoleToolkit/main.pdf new file mode 100644 index 0000000000000000000000000000000000000000..6ba1f2d4628cebfa6ef567ef6b108209242802f4 Binary files /dev/null and b/HS22_ConsoleToolkit/main.pdf differ diff --git a/HS22_ConsoleToolkit/main.tex b/HS22_ConsoleToolkit/main.tex new file mode 100644 index 0000000000000000000000000000000000000000..8feabd9c521ba1cb9e30d9d2e63fac1c0e3f9ffc --- /dev/null +++ b/HS22_ConsoleToolkit/main.tex @@ -0,0 +1,1245 @@ +\documentclass[aspectratio=169]{beamer} + +% Local Imports +\usepackage{general} +\usepackage{slideStyle} +\usepackage{code} + +\usepackage{hyperref} + +% Global Imports +\usepackage{adjustbox} + +% List commands +\newcommand{\ipro}{\item[+]} +\newcommand{\icon}{\item[-]} +\newcommand{\ides}[1]{\item \textbf{#1}} +\newcommand{\iexa}[1]{\item \textbf{Example:} #1} + +% For permission list +\usepackage{array, threeparttable, booktabs} +\newcommand{\dir}{\textcolor{orange}d} +\newcommand{\syl}{\textcolor{red}l} +\newcommand{\rd}{\textcolor{purple}r} +\newcommand{\wri}{\textcolor{blue}w} +\newcommand{\exc}{\textcolor{teal}x} + +\title{The Console Toolkit} +\author{Jean-Claude Graf} +\institute{TheAlternative, ETH Zürich} +\date{17. October 2022} + +\begin{document} + \begin{frame} + \maketitle + \end{frame} + + \section{Introduction} + + \begin{frame}{Goals of This Course} + \begin{itemize} + \item Getting used to the console + \item Navigating the file system + \item Modifying files + \item Learn some basic commands + \item How to get help? + \item Learn to recognize patterns an apply them out in the wild + \end{itemize} + \end{frame} + + \begin{frame}{What is the Console?} + \begin{itemize} + \item ``Keyboard + Text'' interface to your computer + \item Related Terms + \begin{itemize} + % \item \todo{Select terms and describe them} + \ides{Terminal (Emulator):} Synonym to console (today) + \ides{Shell:} SW running in terminal, processing I/O + \ides{Bash, ZSH:} Instances of shells + \ides{Command Line (Interface (CLI)):} Text-only input + \ides{Command Prompt:} + \end{itemize} + \end{itemize} + \end{frame} + + \begin{frame}{Why the Console?} + \begin{itemize} + \item Most direct interface to your computer + \item Great for advanced/complex tasks + \item Can easily be automated + \begin{itemize} + \item I.e. batch work + \end{itemize} + \item Similar on all Unix systems + \item The way to interact with other systems + \begin{itemize} + \item Remote servers + \item Containers + \end{itemize} + \item Last thing alive when you break your system + \end{itemize} + \end{frame} + + \begin{frame}[fragile]{Look, a Console!} + \SU{bob} + \begin{shell} +`\StartConsole` + \end{shell} + + \begin{itemize} + \item Displays different information + \begin{itemize} + \item Username, hostname, current working directory, git status, exit code, time, etc. + \end{itemize} + \item Can be customized + \end{itemize} + \end{frame} + + \part{Beginning} + \section{File System Basics} + + \begin{frame}[fragile]{The 70s were interesting times, or so I'm told} + \textbf{Unix dogma: \emph{Everything is a file!}} \\ + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item Files + \item Directories (or ``folders'') + \item Storage devices + \item Keyboards + \end{itemize} + \column{0.5\textwidth} + \begin{itemize} + \item Monitor + \item Battery + \item \emph{But not network sockets \ldots} + \end{itemize} + \end{columns} + \bigskip + \begin{itemize} + \ides{File:} Abstraction created by the OS + \item $\implies$ Unified interface + \end{itemize} + \end{frame} + + \begin{frame}[fragile]{The Root} + \begin{columns}[T] + \column{0.4\textwidth} + \SU{pwd=/} + \begin{shell}[fontsize=\scriptsize] +ls `\StartConsole` +bin -> usr/bin +boot +dev +etc +home +lib -> usr/lib +lib64 -> usr/lib +mnt +proc +root +sys +tmp +usr +var + \end{shell} + \column{0.6\textwidth} + \begin{itemize} + \item File system organized as tree + \item Everything under \texttt{/}, the root directory + \item In the console, you will be at some point in the tree, + the \emph{working directory} + \end{itemize} + \end{columns} + \end{frame} + + \begin{frame}[fragile]{Working Directory} + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item Where am I? \textrightarrow \: \texttt{pwd} + \item \textbf{P}resent \textbf{W}orking \textbf{D}irectory + \item Also sometimes directly shown in the prompt + \end{itemize} + \column{0.5\textwidth} + \SU{alice,pwd=/} + \begin{shell} +pwd `\StartConsole` +/ + \end{shell} + \SU{bob,pwd=/home} + \begin{shell} +pwd `\StartConsole` +/home + \end{shell} + \end{columns} + \end{frame} + + \begin{frame}[fragile]{Listing Files} + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item What is in here? \textrightarrow \: \texttt{ls} + \item \textbf{L}i\textbf{s}t + \end{itemize} + \column{0.5\textwidth} + \SU{bob} + \begin{shell} +ls `\StartConsole` +Desktop +Documents +Downloads +Music +Pictures +Videos + \end{shell} + \end{columns} + \end{frame} + + \begin{frame}[fragile]{Changing Directory} + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item I want to go somewhere else \textrightarrow \: \texttt{cd} + \item \textbf{C}hange \textbf{D}irectory + \end{itemize} + \column{0.5\textwidth} + \SU{bob} + \begin{shell} +cd Documents `\SU{pwd=~/Documents}` +`\StartConsole` + + \end{shell} + \end{columns} + \end{frame} + + \begin{frame}[fragile]{Special Files and Paths} + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item Special Names: + \begin{itemize} + \ides{Root (/):} Link to root + \ides{Home ($\sim$):} Link to user's home directory + \begin{itemize} + \item I.e. \texttt{/home/alice} + \end{itemize} + \end{itemize} + \item Special Files: + \begin{itemize} + \ides{Parent (..):} Link to parent directory + \ides{Current (.):} Link to itself + \end{itemize} + \ides{Absolut Path:} Start with \texttt{/} + \ides{Relative Path:} Relative to CWD + \begin{itemize} + \item I.e. starts with \texttt{.} + \end{itemize} + \end{itemize} + \column{0.5\textwidth} + \SU{alice} + \begin{shell} +cd / `\SU{pwd=/}` +cd ~ `\SU{pwd=\~}` +cd .. `\SU{pwd=/home}` +cd . +`\StartConsole` + \end{shell} + \end{columns} + \end{frame} + + \section{Command Basics} + + \begin{frame}[fragile]{Command Structure} + \SU{bob} + \begin{shell}[fontsize=\small] +command -shorOption --longOption argument1 argument2 + \end{shell} + \begin{itemize} + \ides{Command:} Name of executable + \ides{Short Option/Flag:} Single letter option + \ides{Long Option:} More memorable name for option + \ides{Arguments:} Additional information passed to the command + \end{itemize} + \end{frame} + + \begin{frame}[fragile]{Advanced Listing} + \begin{itemize} + \item \texttt{ls} Options: + \begin{itemize} + \item \texttt{-a} (\texttt{---all}): show hidden files + \begin{itemize} + \ides{Hidden File:} File whose name start with a dot + \end{itemize} + \item \texttt{-h} (\texttt{---human-readable}): print numbers in human readable format + \item \texttt{-l} (\texttt{---list}): show the long output format + \end{itemize} + \item Combine short options: \texttt{ls -l -a -h} $\Rightarrow$ \texttt{ls -lah} + \end{itemize} + \end{frame} + + \begin{frame}[fragile]{Advanced Listing} + \SU{alice} + \begin{shell}[fontsize=\small] +ls -lah `\StartConsole` +total 52K +drwx------ 8 alice alice 4.0K Sep 3 23:27 . +drwxr-xr-x 4 root root 4.0K Sep 3 23:26 .. +-rw-r--r-- 1 alice alice 21 Jun 4 10:54 .bash_logout +-rw-r--r-- 1 alice alice 57 Jun 4 10:54 .bash_profile +-rw-r--r-- 1 alice alice 141 Jun 4 10:54 .bashrc +-rw-r--r-- 1 alice alice 0 Sep 3 23:27 bla.jpg +drwxr-xr-x 2 alice alice 4.0K Sep 3 23:27 Desktop +drwxr-xr-x 2 alice alice 4.0K Sep 3 23:27 Documents +drwxr-xr-x 2 alice alice 4.0K Sep 3 23:27 Downloads +-rw-r--r-- 1 alice alice 24 Sep 3 23:28 .hidden_file + \end{shell} + \end{frame} + + \begin{frame}[t,fragile]{Getting Help} + \begin{itemize} + \item Where can I find out what options are available? + \item Two Ways: + \begin{itemize} + \item \verb+--help+ option + \begin{itemize} + \item E.g. \verb+ls --help+ + \end{itemize} + \item \textbf{Man}ual pages + \begin{itemize} + \item E.g. \verb+man ls+ + \end{itemize} + \end{itemize} + \item (Bonus) Shell Completion + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Man Pages} + \begin{minipage}{0.6\textwidth} + % \begin{adjustbox}{height=0.95\textheight} + \begin{shellcomp}[fontsize=\tiny] +NAME + ls - list directory contents +SYNOPSIS + ls [OPTION]... [FILE]... +DESCRIPTION + List information about the FILEs (the current directory by default). Sort entries alphabeti- + cally if none of -cftuvSUX nor --sort is specified. + + Mandatory arguments to long options are mandatory for short options too. + + -a, --all + do not ignore entries starting with . + + -A, --almost-all + do not list implied . and .. + + --author + with -l, print the author of each file + + -b, --escape + print C-style escapes for nongraphic characters + \end{shellcomp} + \end{minipage} + \begin{minipage}{0.35\textwidth} + \begin{itemize} + \item{Search by typing \keys{/}} + \item{Quit by typing \keys{q}} + \item{Sometimes there are multiple manuals! \textrightarrow \: Choose the right section} + \begin{itemize} + \item[1:] Shell Commands + \item[2:] System Calls + \item[3:] Library Calls + \item etc. + \end{itemize} + \end{itemize} + \end{minipage} + \end{frame} + + \section{More on FS} + + \begin{frame}[t,fragile]{Copying Files} + \begin{columns}[T] + \column{0.4\textwidth} + \begin{itemize} + \item \textbf{C}o\textbf{p}y: \texttt{cp} + \item Syntax: \texttt{cp source destination} + \end{itemize} + \column{0.6\textwidth} + \SU{alice} + \SU{pwd=~/Downloads} + \begin{shell} +ls `\StartConsole` +dir1 dir2 file1 file2 `\SU{}` +ls dir1 `\StartConsole` +`\SU{}` +cp file1 dir1 +ls `\StartConsole` +dir1 dir2 file1 file2 `\SU{}` +ls dir1 `\StartConsole` +`\StartConsole` + \end{shell} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Moving Files} + \begin{columns}[T] + \column{0.4\textwidth} + \begin{itemize} + \item \textbf{M}o\textbf{v}e command: \texttt{mv} + \item Syntax: \texttt{mv source destination} + \item Use \texttt{mv} to rename files + \end{itemize} + \column{0.65\textwidth} + \vspace{-1cm} + \SU{alice} + \SU{pwd=~/Downloads} + \begin{shell} +ls `\StartConsole` +dir1 dir2 file1 file2 `\SU{}` +ls dir1 `\StartConsole` +`\SU{}` +mv file1 dir1 `\SU{}` +mv file2 file_new `\SU{}` +ls `\StartConsole` +dir1 dir2 file_new `\SU{}` +ls dir1 `\StartConsole` +file1 `\StartConsole` + \end{shell} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Creating and Deleting Directories} + \begin{columns}[T] + \column{0.4\textwidth} + \begin{itemize} + \item \texttt{mkdir} creates a new directory + \item \texttt{rmdir} removes a directory + \begin{itemize} + \item Only works for empty directories + \end{itemize} + \end{itemize} + \column{0.65\textwidth} + \SU{alice,pwd=~/Downloads} + \begin{shell} +mkdir new_dir +ls `\StartConsole` +new_dir `\SU{}` +rmdir new_dir +ls `\StartConsole` +`\StartConsole` + \end{shell} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Deleting Files} + \begin{columns}[T] + \column{0.4\textwidth} + \begin{itemize} + \item \texttt{rm} removes files and directories + \item \texttt{-r} removes a directory and everything in it \emph{(recurisive)} + \item Is \textcolor{red}{irreversible!} + \end{itemize} + \column{0.65\textwidth} + \SU{bob} + \begin{shell} +ls `\StartConsole` +cat1.jpg cat2.jpg `\SU{}` +rm cat1.jpg +ls `\StartConsole` +cat2.jpg + \end{shell} + \end{columns} + \vspace{0.5cm} + \begin{WarningBox} + \texttt{rm} is a shotgun without safety! There is no trashcan. + You can delete your entire file system with \texttt{sudo rm -rf /}, + or your entire home directory with \texttt{rm -rf \textasciitilde}\:!% + \end{WarningBox} + \end{frame} + + \begin{frame}[t,fragile]{Showing Text Files} + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item Output a file's contents to the console with \texttt{cat} + \item Used to stand for \emph{concatenate} + \end{itemize} + \column{0.5\textwidth} + \SU{alice} + \begin{shell} +cat diary.txt `\StartConsole` +Dear diary, today I learned +how to use the console! + \end{shell} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Reading Long Files} + \begin{itemize} + \item What if the text doesn't fit on the terminal? + \item Use the \texttt{less} file viewer + \begin{itemize} + \item We have seen it be for for the man pages + \end{itemize} + \item Scroll up and down with \keys{\arrowkeyup}, \keys{\arrowkeydown} + \item Exit with \keys{q} + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Editing Files} + \begin{itemize} + \item Need a \emph{text editor}! + \item \emph{nano, vim, emacs} + \item Simple, intuitive, no learning required? \textrightarrow\:\texttt{nano} + \item Powerful, efficient? \textrightarrow\:\texttt{vim} + \item Obscure, eccentric and even more powerful? \textrightarrow\:\texttt{emacs} + \item Has some advantages to using a big GUI tool + \begin{itemize} + \item Navigation and editing in the same interface + \item Quick and efficient + \item Very powerful tools available + \end{itemize} + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Nano} + \begin{itemize} + \item Syntax: \texttt{nano [filename]} + \item Key bindings shown on the bottom + \begin{itemize} + \item \texttt{\textasciicircum} stands for the \keys{\ctrl} key (universal) + \end{itemize} + \item Save: \keys{\ctrl} + \keys{o} + \item Close: \keys{\ctrl} + \keys{x} + \item Navigate with arrow keys \keys{\arrowkeyleft} + \keys{\arrowkeydown} \keys{\arrowkeyup} + \keys{\arrowkeyright} + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Users \& Permissions} + \begin{itemize} + \item Linux is a \emph{multi-user operating system} + \item There can be many user accounts + \item Different users can even use the computer at the same time! + \item You usually only use your personal user account + \item Users can be put into groups + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Users} + \begin{columns}[T] + \column{0.5\textwidth} + \textbf{Personal User} + \begin{itemize} + \item{Home directory in \texttt{/home/alice}} + \item{Can only access files in home directory} + \item{Can only stop processes started by itself} + \end{itemize} + \column{0.5\textwidth} + \textbf{Root User} + \begin{itemize} + \item{Also \emph{called the superuser}} + \item{''System administrator``} + \item{Can do anything on the system} + \item{Access to all files} + \item{Can kill any process} + \item{Home directory in \texttt{/root}} + \end{itemize} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Permissions} + \begin{table}[!htbp] + \centering\sffamily + \begin{threeparttable} + \begin{tabular}{*{4} {>{\centering\arraybackslash}p{1.2cm}}} + \toprule + type & users & group & others \\ + \cmidrule(lr){1-1}\cmidrule(lr){2-2}\cmidrule(lr){3-3}\cmidrule(lr){4-4} + \dir \enspace\syl & \rd\enspace\wri\enspace\exc & \rd\enspace\wri\enspace\exc& \rd\enspace\wri\enspace\exc \\ + \bottomrule + \end{tabular} + \smallskip + \begin{tablenotes}[flushleft, online]\small + \item[\dir:] Is a directory. + \item[\syl:] Is a symbolic link. + \item[\rd:] Can Read,\, Read files. + \item[\wri:] Can Write,\, Create files. + \item[\exc:] Can Execute,\, List files. + \end{tablenotes} + \end{threeparttable} + \end{table} + \end{frame} + + \begin{frame}[t,fragile]{Permissions} + \SU{alice,pwd=~/SomeDir} + \begin{shell} +ls -lah `\StartConsole` +drwxr-xr-x 115 root root 12K Apr 3 13:24 etc +drwxr-xr-x 4 alice alice 4.0K Nov 9 2019 home +lrwxrwxrwx 1 root root 7 Dec 7 03:41 lib -> usr/lib +-rwxr-xr-- 1 alice alice 64 Nov 29 12:34 .fehbg +-rw-r--r-- 1 alice alice 25M Jan 27 2020 script.sh + \end{shell} + \end{frame} + + \begin{frame}[t,fragile]{Changing permissions} + \begin{itemize} + \item Change permission: \verb$chmod {u,g,a}{+,-}{r,w,x} file$ + \item Change user/group: \verb$chown OWNER:GROUP file$ + \end{itemize} + \SU{alice} + \begin{shell} +chmod +x program.sh +chmod u-x program.sh +chmod g+rw file.txt +chown alice:alice file.txt + \end{shell} + \end{frame} + + \begin{frame}[t,fragile]{Octal Permissions} + \begin{itemize} + \item Change permission: \verb+chmod OCTALMODE file+ + \begin{itemize} + \item \texttt{4: read} + \item \texttt{2: write} + \item \texttt{1: execute} + \end{itemize} + \item Octal representation allows setting all permissions in one go + \item Desired permissions are added up + \iexa{} \texttt{7 = read + write + execute} + \end{itemize} + \SU{bob} + \begin{shell} +chmod 776 program.sh +chmod 774 program.sh + \end{shell} + \end{frame} + + \section{Efficiency} + + \begin{frame}[t,fragile]{Tab Completion} + \begin{itemize} + \item Hit \keys{\tab} to complete a word you are typing (Command, flag, file, \dots) + \item Hit \keys{\tab} twice to show all possible options + \begin{itemize} + \item Use \keys{\tab} to cycle through them + \end{itemize} + \item Extremely useful terminal feature! Use always! + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Command History} + \begin{itemize} + \item Scroll up in your command history by pressing the \keys{\arrowkeyup} key + \item Press \keys{\ctrl} + \keys{r} to search the history + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Globbing} + \begin{columns}[T] + \column{0.4\textwidth} + \begin{itemize} + \item Match zero or more characters: \texttt{*} + \item Match a single character: \texttt{?} + \item Define a range: \texttt{[start-end]} + \begin{itemize} + \item Combine with \texttt{*} or \texttt{?} + \end{itemize} + \item Define several globbings: \texttt{\{glob1, glob2, ...\}} + \end{itemize} + \bigskip + Different Shells have different features + \column{0.65\textwidth} + \SU{alice,pwd=~/Eth} + \begin{shell} +rm -rf ana* +mv ex-1?.pdf exs/ +mv ex-[0-9]*.pdf exs/ +mv ../{IML, IS} sem6 +rm {*.doc,*.docx} + \end{shell} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Chaining Commands} + \begin{itemize} + \item Run next command iff current succeeds: \texttt{\&\&} + \item Run next command in any case: \texttt{;} + \item Pass stdout to stdin: \texttt{|} + \begin{itemize} + \item More on that later + \end{itemize} + \end{itemize} + \SU{bob} + \begin{shell} +mkdir test && cd test +docker-compose down ; docker-compose up -d + \end{shell} + \end{frame} + + \begin{frame}[t,fragile]{Other} + \begin{itemize} + \item Kill currently running process: \keys{CTRL} + \keys{C} + \begin{itemize} + \item Also useful to discard typed command + \end{itemize} + \item Different terminals/shells provide different features + \item Fast movement + \begin{itemize} + \item Got to: end of line, beginning of line + \item Move: one word back, one word forward + \end{itemize} + % \item \todo{Move task to bg etc.} + \end{itemize} + \end{frame} + + \part{Advanced} + \section{Piping and Redirection} + + \begin{frame}[t, fragile]{Complicated Needs} + How would you design an interface that can... + \begin{itemize} + \item \dots delete files larger than 100MB? + \item \dots show the last 2 lines of a file? + \item \dots sort files by length? + \item \dots search calendar entries and create reminders? + \end{itemize} + \end{frame} + + \begin{frame}[t, fragile]{Complicated Needs} + What about this? + + \begin{center} + \includegraphics[height=0.7\textheight]{./Figures/horrorGui.png} + \end{center} + \footnotetext{\url{https://www.jensroesner.com/wgetgui/}} + \end{frame} + + \begin{frame}[t, fragile]{Complicated Needs} + Many possible answers: + \begin{itemize} + \item Big GUI that does everything + \item A simple tool that users can extend themselves + \item Domain specific language that users write queries with + \item Many simple and combinable tools + \end{itemize} + Unix chooses the last two approaches + \end{frame} + + \begin{frame}[t, fragile] + \frametitle{Piping and Redirection} + \begin{itemize} + \item Unix has small and orthogonal tools + \item Piping and redirection are how to combine them + \begin{itemize} + \ides{Pipe:} Sends output from one command to another command + \ides{Redirection:} Read/Write to files (/streams) + \end{itemize} + \end{itemize} + \end{frame} + + \begin{frame}[t, fragile]{Piping} + \begin{minipage}{0.38\textwidth} + \begin{itemize} + \item Sends output from one command to another command + \item Uses the pipe symbol: \texttt{|} + \item Useful for sequential composition + \item Only works in "one direction" + \item Internally connects output of one process to input of other process + \end{itemize} + \end{minipage} + \begin{minipage}{0.60\textwidth} + \SU{bob} + \begin{shellcomp}[fontsize=\small] +cat numbers `\StartConsole` +zero +one +two +three +four `\SU{}` +cat numbers | tail -n 2 `\StartConsole` +three +four `\SU{}` +cat numbers | grep "t" `\StartConsole` +two +three + \end{shellcomp} + + \end{minipage} + \end{frame} + + \begin{frame}[t, fragile]{Piping Example} + List unique owners of files in current directory: + \vspace{0.5cm} + + \begin{minipage}{0.4\textwidth} + \begin{itemize} + \item List files in directory + \item Omit first two lines + \item Truncate whitespace + \item Cut (delete) all columns except the third + \item Sort alphabetically + \item Only show unique entries + \end{itemize} + \end{minipage} + \begin{minipage}{0.55\textwidth} + \SU{bob,pwd=/} + \begin{shellcomp}[fontsize=\small] +ls -l |\ `\StartConsole` + tail -n +2 |\ + sed 's/\s\s*/ /g' |\ + cut -d ' ' -f 3 |\ + sort | uniq + +bob +root + \end{shellcomp} + \end{minipage} + \end{frame} + + \begin{frame}[t, fragile]{Redirection} + \begin{columns}[T] + \column{0.45\textwidth} + \begin{itemize} + \item Read/write to files (/streams) + \item{\makebox[1.7cm]{\:\texttt{> file\hfill}}} Write output to file + \item{\makebox[1.7cm]{\:\texttt{>\hspace{0cm}> file\hfill}}} Append output to file + \item{\makebox[1.7cm]{\:\texttt{< file\hfill}}} Read input from file + \end{itemize} + \column{0.6\textwidth} + \SU{alice} + \begin{shellcomp}[fontsize=\small] +echo "BlaBla" > hello.txt +cat hello.txt `\StartConsole` +BlaBla `\SU{}` +echo "Hello " > hello.txt +cat hello.txt `\StartConsole` +Hello `\SU{}` +echo "World!" >> hello.txt +cat < hello.txt `\StartConsole` +Hello World! + \end{shellcomp} + \end{columns} + \end{frame} + + \begin{frame}[t, fragile]{Redirection} + Redirection is useful! + \begin{itemize} + \item Store final or intermediate results + \SU{alice} + \begin{shell} +ls -l | tail -n +2 | sed 's/\s\s*/ /g' > result +cut -d ' ' -f 3 < result | sort | uniq + \end{shell} + \item Append output to files + \SU{bob} + \begin{shell} +./logger_script >> log.txt +echo "Logging done!" >> log.txt + \end{shell} + \end{itemize} + \end{frame} + + \section{Processes} + + \begin{frame}[t,fragile]{Process} + \begin{itemize} + \item Every process has a unique ID (PID) + \item View processes with \texttt{ps aux} + \begin{itemize} + \item Filter using \texttt{grep} + \end{itemize} + \end{itemize} + \SU{bob} + \begin{shell} +ps aux | grep firefox `\StartConsole` +bob 2681 0.1 2.6 3702244 428076 ? Sl 07:47 0:48 /usr/lib/firefox/firefox + \end{shell} + \end{frame} + + \begin{frame}[t,fragile]{Process Management} + \begin{itemize} + \item Kill process by pid: \texttt{kill pid} + \item Kill process by name: \texttt{killall name} + \item Kill process with best matching name: \texttt{pkill name} + \item Use \texttt{-SIGKILL} flag if normal kill does not work + \begin{itemize} + \item \texttt{man 7 signal} for more information + \end{itemize} + \end{itemize} + \SU{bob} + \begin{shell}[fontsize=\small] +ps aux | grep firefox `\StartConsole` +bob 2681 0.1 2.6 3702244 428076 ? Sl 07:47 0:48 /usr/lib/firefox/firefox `\SU{}` +kill 2681 +pkill firef +killall -SIGKILL firefox + \end{shell} + \end{frame} + + % \begin{frame}[t,fragile]{HTOP} + % \begin{itemize} + % \item \todo{htop} + % \end{itemize} + % \end{frame} + + \section{PATH} + \begin{frame}[t,fragile]{The PATH Variable} + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item Commands are programs + \begin{itemize} + \item Flag, options, etc. are arguments to this programs + \end{itemize} + \item How does the shell know where programs are? + \item The shell searches the \texttt{PATH} variable + \item \texttt{ls} \textrightarrow \: \texttt{/usr/bin/ls} + \end{itemize} + \column{0.5\textwidth} + \begin{shell} +echo $PATH `\StartConsole +/usr/sbin:/usr/bin + \end{shell}%\todo{\$ breaks highlighting} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Adding Your Own Paths} + \begin{itemize} + \item Let's say you want to add your script directory + \item Temporarily: \texttt{export PATH=\$PATH:\textasciitilde/scripts} + \item Permanently: Add the above to \texttt{\textasciitilde/.bashrc} + \end{itemize} + \end{frame} + + \begin{frame}[t, fragile]{Writing Shell Scripts} + \begin{columns}[T] + \column{0.4\textwidth} + \begin{itemize} + \item Scripts are just a sequence of commands + \item Very easy automation! + \end{itemize} + \column{0.6\textwidth} + \SU{bob} + \begin{shellcomp}[fontsize=\small] +cat music.sh `\StartConsole` +#!/usr/bin/bash +filename="$2.%(ext)s" +echo "$1" +youtube-dl -x "$1" -o "$filename" + \end{shellcomp} + \end{columns} + \SU{alice,pwd=~/scripts} + \begin{shellcomp}[fontsize=\small] +chmod +x music.sh +./music.sh "https://www.youtube.com/watch?v=dQw4w9WgXcQ" music +music.sh "https://www.youtube.com/watch?v=dQw4w9WgXcQ" music + \end{shellcomp} + \end{frame} + + \section{Superuser} + \begin{frame}[t,fragile]{Me vs Superuser} + \begin{columns}[T] + \column{0.5\textwidth} + Me: + \begin{itemize} + \item Home is \texttt{/home/bob} + \item Have full access to that folder + \item No access to others homes + \item Can change own settings + \item May not change system-wide settings + \item Start/Stop user services + \end{itemize} + \column{0.5\textwidth} + Superuser: + \begin{itemize} + \item Home is \texttt{/root} + \item Have full access all data + \item Can change everything + \item No access to others homes + \item May not change system-wide settings + \item Can install/remove software + \item Start/Stop all services + \end{itemize} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Sudo} + \begin{itemize} + \item Run command as \textbf{su}per user \textbf{do}es + \ides{Syntax:} \texttt{sudo command} + \item If you forget sudo, run: \texttt{sudo !!} + \item Change user to sudo: \texttt{sudo su} + \end{itemize} + \end{frame} + + \section{Managing software} + \begin{frame}[t,fragile]{How you get software on Linux} + \begin{itemize} + \item Don't download installers from the internet! + \item Software is managed by the distribution and available through a central repository. + \item Software is \emph{packaged} + \item Similarly to Microsoft's or Apple's app stores + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Installing Packages} + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item Depends on distribution! + \item Package Manager is most important feature of a Linux distribution + \end{itemize} + \column{0.5\textwidth} + Debian, Ubuntu, Mint: + \StartConsole + \begin{shellcomp} +sudo apt install firefox + \end{shellcomp} + Arch, Manjaro: + \StartConsole + \begin{shellcomp} +sudo pacman -S firefox + \end{shellcomp} + RedHat, Fedora: + \StartConsole + \begin{shellcomp} +sudo dnf install firefox + \end{shellcomp} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Searching for Packages} + \begin{columns}[T] + \column{0.5\textwidth} + Debian, Ubuntu, Mint: + \begin{shellcomp} +apt search firefox + \end{shellcomp} + Arch, Manjaro: + \begin{shellcomp} +pacman -Ss firefox + \end{shellcomp} + RedHat, Fedora: + \begin{shellcomp} +dnf search firefox + \end{shellcomp} + \column{0.5\textwidth} + \begin{itemize} + \item The basic package search is usually quite limited + \item Consult the internet for finding the right programs! + \item Arch Wiki has lists of many programs + \end{itemize} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Updating Packages} + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item All packages can be upgraded at once + \item Do this every other week! + \end{itemize} + \column{0.5\textwidth} + Debian, Ubuntu, Mint: + \begin{shellcomp} +sudo apt update +sudo apt upgrade + \end{shellcomp} + Arch, Manjaro: + \begin{shellcomp} +sudo pacman -Syu + \end{shellcomp} + RedHat, Fedora: + \begin{shellcomp} +sudo dnf update + \end{shellcomp} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Building From Source} + \begin{itemize} + \item Sometimes software is unavailable in the repositories + \item Can download sources and compile them manually + \item Careful! No automatic updates, malware, package manager + conflicts, \ldots + \end{itemize} + \end{frame} + + \section{SSH} + \begin{frame}[t,fragile]{SSH} + \begin{itemize} + \item \emph{Secure shell} + \item SSH allows to log in to another computer over the network + \item Server administration, running jobs on supercomputers, + log in to your computer at home + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Using SSH} + \begin{itemize} + \item \texttt{ssh USERNAME@HOSTNAME} + \item Will ask for user password + \begin{itemize} + \item Cumbersome + \item Bad for security + \end{itemize} + \end{itemize} + \SU{bob} + \begin{shell} +ssh bob@remotehost`\StartConsole` +bob@remotehost's`\ `password: +Last login: Wed Apr 6 08:54:53 2022 from localhost +You have mail. +`\SU{bobremote}` +`\StartConsole` + \end{shell} + \end{frame} + + \begin{frame}[t,fragile]{Login Without Password} + \begin{itemize} + \item Generate an SSH key with \texttt{ssh-keygen} + \item Copy key \texttt{\textasciitilde/.ssh/id\_rsa.pub} to \texttt{\textasciitilde/.ssh/authorized\_keys} on the server + \item Use \texttt{ssh-copy-id -i \textasciitilde/.ssh/id\_rsa USERNAME@REMOTE} + \end{itemize} + \end{frame} + + \section{Version control} + \begin{frame}[t,fragile]{Git} + \begin{itemize} + \item{\texttt{thesis.pdf}} + \item{\texttt{thesis\_old.pdf}} + \item{\texttt{thesis\_copy.pdf}} + \item{\texttt{thesis\_finalversion.pdf}} + \item{\texttt{thesis\_finalversion2.pdf}} + \end{itemize} + \end{frame} + + \begin{frame}[fragile]{Git} + \emph{Git is a version-control system for tracking changes in computer files + and coordinating work on those files among multiple people.} \\ + \hspace{10cm}--- Wikipedia + \end{frame} + + \begin{frame}[t,fragile]{Git} + \begin{itemize} + \item Track changes to your code + \item Comment your changes + \item Easily revert back to older versions + \item Avoid/manage conflicts when working in teams + \item Manage release versions and development versions + \item Work on different branches at the same time + \end{itemize} + \end{frame} + + \begin{frame}[t,fragile]{Git Example} + \begin{columns}[T] + \column{0.5\textwidth} + \begin{itemize} + \item{Initialize a git repository} + \item{Add files you want to include in a commit} + \item{Create a commit for your selected changes} + \item{Push changes to server} + \end{itemize} + \column{0.5\textwidth} + \StartConsole + \begin{shellcomp} +git init + \end{shellcomp} + \begin{shellcomp} +git add changed_file.txt + \end{shellcomp} + \begin{shellcomp} +git commit + \end{shellcomp} + \begin{shellcomp} +git push + \end{shellcomp} + \end{columns} + \end{frame} + + \begin{frame}[t,fragile]{Sharing a Repository} + \begin{itemize} + \item You can use services like \emph{Github} or \emph{Gitlab} to collaborate with others on your project + \item Or host your repositories yourself! + \begin{itemize} + \item You can pull from/push to any server you can access via SSH + \end{itemize} + \end{itemize} + \end{frame} + + \section{Backup} + \begin{frame}[t,fragile]{Backup} + \begin{itemize} + \item Backup only home or full root + \begin{itemize} + \item Convenience vs storage + \end{itemize} + \item \texttt{rsync} is a fancy file-copy tool + \begin{itemize} + \item Open format + \item Incremental backup + \item Runs over SSH + \end{itemize} + \item Use some backup script based on \texttt{rsync} + \begin{itemize} + \item Or roll your own + \item E.g. \texttt{rsync-time-backup} + \end{itemize} + \item \texttt{Borg} is also famous + \end{itemize} + \SU{alice} + \begin{shellcomp} +rsync_tmbackup.sh ~/ alice@homeserver:~/Backup/ ~/rsync_exclude_list + \end{shellcomp} + \end{frame} + +% \section{Ranger} +% +% \begin{frame}[t, fragile] +% \frametitle{Ranger} +% \begin{columns}[T] +% \column{0.5\textwidth} +% \begin{itemize} +% \item File manager on the console +% \begin{itemize} +% \item Usable over SSH +% \end{itemize} +% \item Can move files, change permissions, bulk rename... +% \item Bookmarks +% \item Keyshortcuts for frequent locations +% \item Plugins +% \item Preview functionality +% \end{itemize} +% \column{0.5\textwidth} +% \includegraphics[width=1.0\columnwidth]{img/ranger1.png} +% \end{columns} +% \end{frame} +% +% \begin{frame}[t, fragile] +% \frametitle{Ranger image preview} +% \begin{center} +% \includegraphics[width=0.9\textwidth]{img/ranger_img_preview.png} +% \end{center} +% \end{frame} +% + + \section{Epilogue} + \begin{frame}[t,fragile]{Commands I Find Useful} + \begin{itemize} + \item \verb+grep -Rni "linux"+ + \item \verb+du -d 1 -h+ + \item \verb_for f in *.md; do echo pandoc -o $(basename $f .md).pdf $f; done_ + \item \verb+find -type l -iname "*.cr3" | wc -l+ + \end{itemize} + \end{frame} + % \begin{frame}[t,fragile] + % \vspace{2cm} + % % Feedback is very welcome: \url{https://feedback.thealternative.ch} + % \end{frame} + % for f in *; do echo "unzip -d ${f/\.zip/} $f" ; done + +\end{document} diff --git a/HS22_ConsoleToolkit/slideStyle.sty b/HS22_ConsoleToolkit/slideStyle.sty new file mode 100644 index 0000000000000000000000000000000000000000..d5495388be8b7539bf8d6ea3d67f8140b9387ca7 --- /dev/null +++ b/HS22_ConsoleToolkit/slideStyle.sty @@ -0,0 +1,27 @@ +\ProvidesPackage{slideStyle} + +\usecolortheme[RGB={37,68,113}]{structure} +\usetheme{Dresden} + +% Uncomment to disable top overview +% \setbeamertemplate{headline}{} + +%Define TheAlt colors +\definecolor{ldorange}{HTML}{F18A20} +\definecolor{ldblue}{HTML}{254471} + +%Apply TheAlt colors to theme +\setbeamercolor{section in head/foot}{fg=ldorange} +\setbeamercolor{author in head/foot}{fg=white} +\setbeamercolor{subsection in head/foot}{fg=white} +\setbeamercolor{caption name}{fg=vlg} +\setbeamercolor{caption}{fg=vlg} +\setbeamercolor{frametitle}{fg=ldblue} +\setbeamercolor{title}{fg=ldorange} +\setbeamercolor{institute}{fg=ldblue} + +\setbeamerfont{title}{series=\bfseries} + +\setbeamertemplate{caption}{\raggedright\insertcaption\par} +\setbeamertemplate{navigation symbols}{} +\setbeamertemplate{bibliography item}[text]