update

2022-02-05 13:41:25 +09:00
parent e73e9364cd
commit 4ee56900e9
7 changed files with 534 additions and 285 deletions
--- a/source/_posts/2021/installing-arch-linux.md
+++ b/source/_posts/2021/installing-arch-linux.md
@@ -10,13 +10,17 @@ This note includes all commands I typed when I set up Arch Linux on my new serve

 # Setup

-## Wipe a whole disk
+## Wipe a device

 ```bash
-wipefs -a /dev/sda
+wipefs -a /dev/sda # erase file-system (fast)
+
+# or
+
+shred -v -n 1 --random-source=/dev/urandom /dev/sda # write pseudo-random data to the device (takes longer)
 ```

-## Create partition
+## Create partitions

 ```bash
 parted
@@ -25,19 +29,17 @@ parted
 ```bash
 select /dev/sda
 mktable gpt
-mkpart EFI fat32 0 512MB # EFI
+mkpart EFI fat32 0 512MB # EFI (Extensible Firmware Interface)
 mkpart Arch ext4 512MB 100% # Arch
-set 1 esp on # flag partition 1 as ESP
+set 1 esp on # flag partition 1 as ESP (EFI System Partition)
 quit
 ```

-## Install file-system
+## Install file-system for UEFI/GPT setup

 ```bash
 mkfs.vfat -F 32 /dev/sda1 # EFI
 mkfs.ext4 /dev/sda2 # Arch
-
-e2fsck -cc -C 0 /dev/sda2 # fsck
 ```

 ## Mount disks
@@ -48,18 +50,23 @@ mount /dev/sda2 /mnt
 mount /dev/sda1 /mnt/boot
 ```

-## Install Linux kernel
+## Install the base system

 ```bash
-# choose between 'linux' or 'linux-lts'
+# choose between 'linux' and 'linux-lts'
 pacstrap /mnt base linux-lts linux-firmware
-genfstab -U /mnt >> /mnt/etc/fstab
 arch-chroot /mnt
 ```

+## Configure pacman
+
 ```bash
+# optimize mirrorlist
 pacman -S reflector
 reflector --protocol https --latest 30 --sort rate --save /etc/pacman.d/mirrorlist --verbose # optimize mirror list
+
+# enable color output
+sed '/Color/s/^#//' -i /etc/pacman.conf
 ```

 ## Install essentials
@@ -68,7 +75,13 @@ reflector --protocol https --latest 30 --sort rate --save /etc/pacman.d/mirrorli
 pacman -S vim man-db man-pages git base-devel
 ```

-## Add fstab entries
+## Configure fstab
+
+```bash
+pacman -S xfsprogs # for XFS
+
+genfstab -U /mnt >> /etc/fstab # Generate fstab based off of /mnt
+```

 ```ini /etc/fstab
 # backup
@@ -76,6 +89,9 @@ UUID=<UUID> /mnt/backup ext4 defaults 0 2

 # archive (do not prevent boot even if fsck fails)
 UUID=<UUID> /mnt/archive ext4 defaults,nofail,x-systemd.device-timeout=4 0 2
+
+# xfs drive
+UUID=<UUID> /mnt/archive2 xfs defaults 0 0
 ```

 You can find `<UUID>` from `lsblk -f`.
@@ -84,16 +100,7 @@ You can find `<UUID>` from `lsblk -f`.
 findmnt --verify --verbose # verify fstab
 ```

-## Locales
-
-```bash
-ln -sf /usr/share/zoneinfo/Asia/Tokyo /etc/localtime
-hwclock --systohc
-vim /etc/locale.gen & locale-gen
-echo "LANG=en_US.UTF-8" > /etc/locale.conf
-```
-
-## Install bootloader
+## Install bootloader (GRUB 2)

 ```bash
 pacman -S \
@@ -103,13 +110,25 @@ pacman -S \
 grub-install --target=x86_64-efi --efi-directory=/boot --bootloader-id=GRUB

 vim /etc/default/grub
-# GRUB_TIMEOUT=3
-# GRUB_DISABLE_SUBMENU=y
+sed -E \
+  -e 's/(GRUB_TIMEOUT=).+/\13/' \
+  -e 's/(GRUB_DISABLE_SUBMENU=).+/\1y/' \
+  -i /etc/default/grub
 grub-mkconfig -o /boot/grub/grub.cfg
 ```

 - [GRUB/Tips and tricks - ArchWiki](https://wiki.archlinux.org/title/GRUB/Tips_and_tricks)

+## Configure locales
+
+```bash
+ln -sf /usr/share/zoneinfo/Asia/Tokyo /etc/localtime
+hwclock --systohc
+vim /etc/locale.gen
+locale-gen
+echo "LANG=en_US.UTF-8" > /etc/locale.conf
+```
+
 ## Setup network

 ```bash
@@ -134,8 +153,8 @@ Name=enp5s0
 Address=10.0.1.2/24
 Gateway=10.0.1.1
 DNS=10.0.1.100   # self-hosted DNS resolver
-DNS=1.1.1.1      # Cloudflare for the fallback DNS server
-MACVLAN=dns-shim # to handle local DNS lookup to 10.0.1.100, which is managed by Docker macvlan driver
+DNS=9.9.9.9      # Quad9 for the fallback DNS server
+MACVLAN=dns-shim # to route local DNS lookup to 10.0.1.100, which is managed by Docker macvlan driver
 ```

 ```ini /etc/systemd/network/dns-shim.netdev
@@ -179,7 +198,10 @@ networkctl status
 ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf

 # for self-hosted dns resolver
-sed -r -i -e 's/#?DNSStubListener=yes/DNSStubListener=no/g' -e 's/#DNS=/DNS=10.0.1.100/g' /etc/systemd/resolved.conf
+sed -E \
+  -e 's/^#?DNSStubListener=.*/DNSStubListener=no/' \
+  -e 's/^DNS=.*/DNS=10.0.1.100/' \
+  -i /etc/systemd/resolved.conf

 systemctl enable --now systemd-resolved
 resolvectl status
@@ -187,9 +209,9 @@ resolvectl query ddg.gg
 drill ddg.gg
 ```

-If `networkctl` keep showing `enp5s0` as `degraded`, then run `ip addr add 10.0.1.2/24 dev enp5s0 ` to manually assign static IP address for the workaround.
+If `networkctl` keep showing `enp5s0` as `degraded`, then run `ip addr add 10.0.1.2/24 dev enp5s0` to manually assign static IP address for the workaround.

-## Exit chroot
+## Leave chroot environment

 ```bash
 exit # leave chroot
@@ -219,10 +241,10 @@ pacman -S zsh
 chsh -s /bin/zsh

 # Install useful utils (totally optional)
-yay -S pyenv exa antibody direnv fd ripgrep fzy peco ghq-bin hub neofetch tmux git-delta lazygit jq lostfiles ncdu htop rsync youtube-dl prettier tree age
+yay -S pyenv exa antibody direnv fd ripgrep fzy peco ghq-bin hub neofetch tmux git-delta lazygit jq lostfiles ncdu htop rsync youtube-dl prettier age
 ```

-## Setup operator user (i.e., a user without superuser privilege)
+## Setup operator user (i.e., a user without a root privilege)

 ```bash
 passwd # change root password
@@ -230,14 +252,16 @@ passwd # change root password
 useradd -m -s /bin/zsh <user> # add operator user
 passwd <user> # change operator user password

-userdbctl # verify users
-userdbctl group # verify groups
+userdbctl # show users
+userdbctl group # show groups

-pacman -S sudo
-echo "%sudo ALL=(ALL) NOPASSWD:/usr/bin/pacman" > /etc/sudoers.d/pacman # allow users in sudo group to run pacman without password (optional)
+pacman -S sudo # install sudo
+
+# add operator user to sudo group
 groupadd sudo
-usermod -aG sudo <user> # add operator user to sudo group
-visudo -c
+usermod -aG sudo <user>
+echo "%sudo ALL=(ALL) NOPASSWD:/usr/bin/pacman" > /etc/sudoers.d/pacman # allow users in sudo group to run pacman without password (optional)
+visudo -c # validate sudoers
 ```

 ## SSH
@@ -248,6 +272,15 @@ vim /etc/ssh/sshd_config
 systemctl enable --now sshd
 ```

+on the client machine:
+
+```bash
+# brew install ssh-copy-id
+ssh-copy-id <user>@<serverIp>
+```
+
+### Make SSH forwarding work with tmux and sudo
+
 ```bash ~/.ssh/rc
 if [ ! -S ~/.ssh/ssh_auth_sock ] && [ -S "$SSH_AUTH_SOCK" ]; then
  ln -sf $SSH_AUTH_SOCK ~/.ssh/ssh_auth_sock
@@ -263,30 +296,72 @@ setenv -g SSH_AUTH_SOCK $HOME/.ssh/ssh_auth_sock
 Defaults env_keep += SSH_AUTH_SOCK
 ```

-on the host machine:
+See also: [Happy ssh agent forwarding for tmux/screen · Reboot and Shine](https://werat.dev/blog/happy-ssh-agent-forwarding/)
+
+## MTA
+
+### mail
+
+Needed for `smartd`.

 ```bash
-ssh-copy-id <user>@<ip>
+pacman -S s-nail
 ```

-See also: [Happy ssh agent forwarding for tmux/screen · Reboot and Shine](https://werat.dev/blog/happy-ssh-agent-forwarding/)
+- [S-nail - ArchWiki](https://wiki.archlinux.org/title/S-nail)
+
+### sendmail
+
+Needed for `cfddns`.
+
+```
+yay -S sendmail
+```

 ## S.M.A.R.T.

 ```bash
 pacman -S smartmontools
-systemctl enable --now smartd
+```

+### Setup automated checkups
+
+```ini /etc/smartd.conf
+# scan all but removable devices and notify any test failures
+DEVICESCAN -a -o on -S on -n standby,q -s (S/../.././02|L/../../6/03) -m me@example.com
+```
+
+```bash
+systemctl enable --now smartd
+```
+
+### Manual testing
+
+```bash
 smartctl -t short /dev/sda
 smartctl -l selftest /dev/sda
 ```

+See: [S.M.A.R.T. - ArchWiki](https://wiki.archlinux.org/title/S.M.A.R.T.)
+
+## lm_sensors
+
+```bash
+pacman -S lm_sensors
+
+sensors-detect
+
+systemctl enable --now lm_sensors
+```
+
+See: [lm_sensors - ArchWiki](https://wiki.archlinux.org/title/Lm_sensors)
+
 ## NVIDIA drivers

 ```bash
 pacman -S nvidia-lts # 'nvidia' for 'linux'
 reboot
-nvidia-smi # test runtime
+nvidia-smi # validate runtime
 ```

 ## Docker
@@ -294,7 +369,6 @@ nvidia-smi # test runtime
 ```bash
 pacman -S docker docker-compose
 yay -S nvidia-container-runtime
-systemctl enable --now docker
 ```

 ```json /etc/docker/daemon.json
@@ -315,7 +389,7 @@ systemctl enable --now docker
 ```

 ```bash
-systemctl restart docker
+systemctl enable --now docker

 usermod -aG docker <user>

@@ -341,9 +415,261 @@ services:
 - [Configure logging drivers | Docker Documentation](https://docs.docker.com/config/containers/logging/configure/)
 - [Architecture Overview — NVIDIA Cloud Native Technologies documentation](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/arch-overview.html)

-# Additional setup
+## Backup (restic)

-## nginx-proxy
+```bash
+yay -S restic
+```
+
+```ini /etc/systemd/system/restic.service
+[Unit]
+Description=Restic Backup Service
+
+[Service]
+Nice=19
+IOSchedulingClass=idle
+KillSignal=SIGINT
+ExecStart=/etc/restic/run.sh
+```
+
+```ini /etc/systemd/system/restic.timer
+[Unit]
+Description=Restic Backup Timer
+
+[Timer]
+OnCalendar=*-*-* 0,6,12,18:0:0
+RandomizedDelaySec=15min
+Persistent=true
+
+[Install]
+WantedBy=timers.target
+```
+
+```bash /etc/restic/config.sh
+export RESTIC_REPOSITORY=/path/to/backup
+export RESTIC_PASSWORD=<passphrase>
+export RESTIC_PROGRESS_FPS=1
+```
+
+```bash /etc/restic/run.sh
+#!/bin/bash -ue
+# usage: run.sh
+# https://restic.readthedocs.io/en/latest/040_backup.html#
+
+source /etc/restic/config.sh
+
+date
+
+# system
+restic backup --tag system -v \
+  --one-file-system \
+  --exclude .cache \
+  --exclude .vscode-server \
+  --exclude .vscode-server-insiders \
+  --exclude TabNine \
+  --exclude /var/lib/docker/overlay2 \
+  / /boot
+
+# data
+restic backup --tag data -v \
+  --exclude 'appdata_*/preview' \ # Nextcloud cache
+  --exclude 'appdata_*/dav-photocache' \ # Nextcloud cache
+  /mnt/data
+
+# prune
+restic forget --prune --group-by tags \
+  --keep-last 4 \
+  --keep-within-daily 7d \
+  --keep-within-weekly 1m \
+  --keep-within-monthly 3m
+
+# verify
+restic check
+```
+
+```bash /etc/restic/show.sh
+#!/bin/bash
+# usage: show.sh <file|directory>
+# https://restic.readthedocs.io/en/latest/050_restore.html
+
+source /etc/restic/config.sh
+
+TARGET=${1:-$(pwd)}
+MODE="ls -l"
+if [[ -f $TARGET ]]; then
+  TARGET=$(realpath ${TARGET})
+  MODE=dump
+fi
+
+TAG=$(restic snapshots --json | jq -r '[.[].tags[0]] | unique | .[]' | fzy)
+ID=$(restic snapshots --tag $TAG --json | jq -r ".[] | [.time, .short_id] | @tsv" | fzy | awk '{print $2}')
+
+>&2 echo "Command: restic ${MODE} ${ID} ${TARGET}"
+
+restic $MODE $ID ${TARGET}
+```
+
+```bash /etc/restic/restore.sh
+#!/bin/bash
+# https://restic.readthedocs.io/en/latest/050_restore.html
+
+source /etc/restic/config.sh
+
+TARGET=${1:?Specify TARGET}
+TARGET=$(realpath ${TARGET})
+
+TAG=$(restic snapshots --json | jq -r '[.[].tags[0]] | unique | .[]' | fzy)
+ID=$(restic snapshots --tag $TAG --json | jq -r ".[] | [.time, .short_id] | @tsv" | fzy | awk '{print $2}')
+
+>&2 echo "Command: restic restore ${ID} -i ${TARGET} -t /"
+
+read -p "Press enter to continue"
+
+restic restore $ID -i ${TARGET} -t /
+```
+
+```bash
+chmod 700 /etc/restic/config.sh
+ln -sf /etc/restic/restic.{service,timer} /etc/systemd/system/
+systemctl enable --now restic.timer
+```
+
+- [Restic Documentation — restic 0.12.1 documentation](https://restic.readthedocs.io/en/stable/)
+
+## VPN (WireGuard)
+
+```bash
+pacman -S wireguard-tools
+
+# gen private key
+(umask 0077; wg genkey > server.key)
+
+# gen public key
+wg pubkey < server.key > server.pub
+
+# gen preshared key for each client
+(umask 0077; wg genpsk > secret1.psk)
+(umask 0077; wg genpsk > secret2.psk)
+...
+```
+
+```ini /etc/wireguard/wg0.conf
+[Interface]
+Address = 10.0.10.1/24
+ListenPort = 12345
+PrivateKey = <content of server.key>
+
+PostUp   = iptables -A FORWARD -i %i -j ACCEPT; iptables -t nat -A POSTROUTING -o dns-shim -d 10.0.1.100/32 -j MASQUERADE; iptables -t nat -A POSTROUTING -o enp5s0 ! -d 10.0.1.100/32 -j MASQUERADE
+PostDown = iptables -D FORWARD -i %i -j ACCEPT; iptables -t nat -D POSTROUTING -o dns-shim -d 10.0.1.100/32 -j MASQUERADE; iptables -t nat -D POSTROUTING -o enp5s0 ! -d 10.0.1.100/32 -j MASQUERADE
+
+[Peer]
+PublicKey = <public key>
+PresharedKey = <content of secret1.psk>
+AllowedIPs = 10.0.10.2/32
+
+[Peer]
+PublicKey = <public key>
+PresharedKey = <content of secret2.psk>
+AllowedIPs = 10.0.10.3/32
+```
+
+```bash
+sysctl -w net.ipv4.ip_forward=1
+ufw allow 12345/udp # if ufw is running
+systemctl enable --now wg-quick@wg0
+wg show # show active settings
+```
+
+- https://wiki.archlinux.org/title/WireGuard
+- https://github.com/linuxserver/docker-wireguard
+- https://philipdeljanov.com/posts/2019/03/21/setting-up-a-wireguard-vpn/
+
+## Fail2ban
+
+```
+pacman -S fail2ban
+```
+
+```ini /etc/fail2ban/filter.d/bad-auth.conf
+[Definition]
+failregex = .* client login failed: .+ client:\ <HOST>
+ignoreregex =
+journalmatch = CONTAINER_NAME=mail-front-1
+```
+
+```ini /etc/fail2ban/jail.local
+[DEFAULT]
+ignoreip = 127.0.0.1/8 10.0.1.0/24
+
+[sshd]
+enabled = true
+port = 22,10122
+bantime = 1h
+mode = aggressive
+
+# https://mailu.io/1.9/faq.html?highlight=fail2ban#do-you-support-fail2ban
+[mailu]
+enabled = true
+backend = systemd
+filter = bad-auth
+findtime = 15m
+maxretry = 10
+bantime = 1w
+action = docker-action
+```
+
+```ini /etc/fail2ban/action.d/docker-action.conf
+[Definition]
+
+actionstart = iptables -N f2b-bad-auth
+              iptables -A f2b-bad-auth -j RETURN
+              iptables -I DOCKER-USER -p tcp -m multiport --dports 1:1024 -j f2b-bad-auth
+
+actionstop = iptables -D DOCKER-USER -p tcp -m multiport --dports 1:1024 -j f2b-bad-auth
+             iptables -F f2b-bad-auth
+             iptables -X f2b-bad-auth
+
+actioncheck = iptables -n -L DOCKER-USER | grep -q 'f2b-bad-auth[ \t]'
+
+actionban = iptables -I f2b-bad-auth 1 -s <ip> -j DROP
+
+actionunban = iptables -D f2b-bad-auth -s <ip> -j DROP
+```
+
+```bash
+systemctl enable --now fail2ban
+fail2ban-client status
+```
+
+## cfddns
+
+Dynamic DNS for Cloudflare.
+
+> Star [the GitHub repository](https://github.com/uetchy/cfddns) if you like it :)
+
+```
+yay -S cfddns
+```
+
+```yml /etc/cfddns/cfddns.yml
+token: <token>
+notification:
+  enabled: true
+  from: cfddns@localhost
+  to: me@example.com
+```
+
+```ini /etc/cfddns/domains
+example.com
+dev.example.com
+example.org
+```
+
+```
+systemctl enable --now cfddns
+```
+
+## Reverse proxy (nginx-proxy)

 ```bash
 git clone --recurse-submodules https://github.com/evertramos/nginx-proxy-automation.git /srv/proxy
@@ -362,55 +688,17 @@ make # pull, build, start
 make applypatches # run only once
 ```

-## Fail2ban
+## Monitor (Telegraf + InfluxDB + Grafana)

-```
-pacman -S fail2ban
-```
-
-```ini /etc/fail2ban/filter.d/bad-auth.conf
-[INCLUDES]
-before = common.conf
-
-[Definition]
-failregex = .* client login failed: .+ client:\ <HOST>
-ignoreregex =
-```
-
-```ini /etc/fail2ban/jail.local
-[DEFAULT]
-ignoreip = 127.0.0.1/8 10.0.1.0/24
-
-[sshd]
-enabled = true
-port = 22,10122
-bantime = 1h
-mode = aggressive
-
-# https://github.com/Mailu/Mailu/blob/master/docs/faq.rst#do-you-support-fail2ban
-[mailu]
-enabled = true
-backend = systemd
-journalmatch = CONTAINER_NAME=mail_front_1
-filter = bad-auth
-findtime = 1h
-maxretry = 3
-bantime = 1w
-chain = DOCKER-USER
-banaction = iptables-allports
-```
-
-```patch /etc/systemd/system/fail2ban.service
- After=network.target iptables.service firewalld.service ip6tables.service ipset.service nftables.service
-+ After=network.target iptables.service firewalld.service ip6tables.service ipset.service nftables.service docker.service
-```
+### Grafana + InfluxDB (Docker)

 ```bash
-systemctl enable --now fail2ban
-fail2ban-client status sshd
+git clone https://github.com/uetchy/docker-monitor.git /srv/monitor
+cd /srv/monitor
+
 ```

-## Telegraf
+### Telegraf (Host)

 ```bash
 yay -S telegraf
@@ -494,15 +782,6 @@ yay -S telegraf
 [[inputs.sensors]]
  interval = "60s"
  remove_numbers = false
-
-# Run executable as long-running input plugin
-[[inputs.execd]]
-  interval = "15s"
-  command = ["/metrics.sh"]
-  name_override = "metrics"
-  signal = "STDIN"
-  restart_delay = "20s"
-  data_format = "logfmt"
 ```

 ```ini /etc/sudoers.d/telegraf
@@ -518,156 +797,6 @@ telegraf -config /etc/telegraf/telegraf.conf -test
 systemctl enable --now telegraf
 ```

-## cfddns
-
-Dynamic DNS for Cloudflare.
-
-> Star [the GitHub repository](https://github.com/uetchy/cfddns) if you like it :)
-
-```
-yay -S cfddns sendmail
-```
-
-```yml /etc/cfddns/cfddns.yml
-token: <token>
-notification:
-  enabled: true
-  from: cfddns@localhost
-  to: me@example.com
-```
-
-```ini /etc/cfddns/domains
-example.com
-dev.example.com
-example.org
-```
-
-```
-systemctl enable --now cfddns
-```
-
-## Backup
-
-```bash
-pacman -S restic
-```
-
-```ini /etc/backup/restic.service
-[Unit]
-Description=Daily Backup Service
-
-[Service]
-Type=simple
-Nice=19
-IOSchedulingClass=2
-IOSchedulingPriority=7
-ExecStart=/etc/backup/run.sh
-```
-
-```ini /etc/backup/restic.timer
-[Unit]
-Description=Daily Backup Timer
-
-[Timer]
-WakeSystem=false
-OnCalendar=*-*-* 14:00
-RandomizedDelaySec=5min
-
-[Install]
-WantedBy=timers.target
-```
-
-```bash /etc/backup/run.sh
-#!/bin/bash -ue
-# usage: run.sh
-# https://restic.readthedocs.io/en/latest/040_backup.html#
-
-export RESTIC_REPOSITORY=/path/to/backup
-export RESTIC_PASSWORD=<passphrase>
-export RESTIC_PROGRESS_FPS=1
-
-date
-
-# system
-restic backup --tag system -v \
-  --one-file-system \
-  --exclude .cache \
-  --exclude .vscode-server \
-  --exclude .vscode-server-insiders \
-  --exclude TabNine \
-  --exclude /var/lib/docker/overlay2 \
-  / /boot
-
-# data
-restic backup --tag data -v \
-  --exclude 'appdata_*/preview' \ # Nextcloud cache
-  --exclude 'appdata_*/dav-photocache' \ # Nextcloud cache
-  /mnt/data
-
-# prune
-restic forget --prune --group-by tags \
-  --keep-within-daily 7d \
-  --keep-within-weekly 1m \
-  --keep-within-monthly 3m
-
-# verify
-restic check
-```
-
-```bash /etc/backup/show.sh
-#!/bin/bash
-# usage: show.sh <file|directory>
-# https://restic.readthedocs.io/en/latest/050_restore.html
-
-export RESTIC_REPOSITORY=/path/to/backup
-export RESTIC_PASSWORD=<passphrase>
-export RESTIC_PROGRESS_FPS=1
-
-TARGET=${1:-$(pwd)}
-MODE="ls -l"
-if [[ -f $TARGET ]]; then
-  TARGET=$(realpath ${TARGET})
-  MODE=dump
-fi
-
-TAG=$(restic snapshots --json | jq -r '[.[].tags[0]] | unique| .[]' | fzy)
-ID=$(restic snapshots --tag $TAG --json | jq -r ".[] | [.time, .short_id] | @tsv" | fzy | awk '{print $2}')
-
->&2 echo "Command: restic ${MODE} ${ID} ${TARGET}"
-
-restic $MODE $ID ${TARGET}
-```
-
-```bash /etc/backup/restore.sh
-#!/bin/bash
-
-# https://restic.readthedocs.io/en/latest/050_restore.html
-
-export RESTIC_REPOSITORY=/path/to/backup
-export RESTIC_PASSWORD=<passphrase>
-export RESTIC_PROGRESS_FPS=1
-
-TARGET=${1:?Specify TARGET}
-TARGET=$(realpath ${TARGET})
-
-TAG=$(restic snapshots --json | jq -r '[.[].tags[0]] | unique | .[]' | fzy)
-ID=$(restic snapshots --tag $TAG --json | jq -r ".[] | [.time, .short_id] | @tsv" | fzy | awk '{print $2}')
-
->&2 echo "Command: restic restore ${ID} -i ${TARGET} -t /"
-
-read -p "Press enter to continue"
-
-restic restore $ID -i ${TARGET} -t /
-```
-
-```bash
-chmod 700 /etc/backup/{run,show}.sh
-ln -sf /etc/backup/restic.{service,timer} /etc/systemd/system/
-systemctl enable --now restic
-```
-
- [Restic Documentation — restic 0.12.1 documentation](https://restic.readthedocs.io/en/stable/)
-
 ## Kubernetes

 ```bash
@@ -683,10 +812,22 @@ kubectl get cm -n kube-system kubeadm-config -o yaml
 - [Kubernetes - ArchWiki](https://wiki.archlinux.org/index.php/Kubernetes)
 - [Kubernetes Ingress Controller with NGINX Reverse Proxy and Wildcard SSL from Let's Encrypt - Shogan.tech](https://www.shogan.co.uk/kubernetes/kubernetes-ingress-controller-with-nginx-reverse-proxy-and-wildcard-ssl-from-lets-encrypt/)

+## Firewall (ufw)
+
+```bash
+pacman -S ufw
+systemctl enable --now ufw
+```
+
+See:
+
+- https://wiki.archlinux.org/title/Uncomplicated_Firewall
+- [Introduction to Netfilter – To Linux and beyond !](https://home.regit.org/netfilter-en/netfilter/)
+
 ## Audio

 ```bash
-pacman -S alsa-utils # maybe requires rebooting system
+pacman -S alsa-utils # may require rebooting system
 usermod -aG audio <user>

 # list devices as root
@@ -751,14 +892,54 @@ pcm.!default {
 - [ALSA project - the C library reference: PCM (digital audio) plugins](https://www.alsa-project.org/alsa-doc/alsa-lib/pcm_plugins.html)
 - [Asoundrc - AlsaProject](https://www.alsa-project.org/wiki/Asoundrc)

-## Firewall
+## Telegram notifier

-```bash
-pacman -S firewalld
-systemctl enable --now firewalld
+```bash /usr/local/bin/telegram-notifier
+#!/bin/bash
+
+BOT_TOKEN=<your bot token>
+CHAT_ID=<your chat id>
+PAYLOAD=$(ruby -r json -e "print ({text: ARGF.to_a.join, chat_id: $CHAT_ID}).to_json" </dev/stdin)
+
+OK=$(curl -s -X "POST" \
+  -H "Content-Type: application/json; charset=utf-8" \
+  -d "$PAYLOAD" \
+  https://api.telegram.org/bot${BOT_TOKEN}/sendMessage | jq .ok)
+
+if [[ $OK == true ]]; then
+  exit 0
+else
+  exit 1
+fi
 ```

-See [Introduction to Netfilter – To Linux and beyond !](https://home.regit.org/netfilter-en/netfilter/).
+## LUKS
+
+<!-- I've published a Vagrantfile for playing around LUKS. -->
+
+### Encrypt existing un-encrypted root partition
+
+Boot Arch Linux from archiso USB device, then:
+
+```bash
+pacman -Syu
+resize2fs -p -M /dev/sdaX
+# resize2fs -p /dev/sdaX 1838M
+cryptsetup reencrypt --encrypt --reduce-device-size 32M /dev/sdaX
+cryptsetup --allow-discards --perf-no_read_workqueue --perf-no_write_workqueue --persistent open /dev/sdaX root
+resize2fs /dev/mapper/root
+
+mkdir /mnt/{root,boot}
+mount /dev/sda1 /mnt/boot
+mount /dev/mapper/root /mnt/root
+systemd-nspawn --boot --bind=/mnt/boot:/boot --directory=/mnt/root
+
+# add sysroot line
+vim /etc/mkinitcpio-systemd-tool/config/crypttab
+vim /etc/mkinitcpio-systemd-tool/config/fstab
+mkinitcpio -P
+reboot
+```

 # Maintenance

@@ -782,9 +963,9 @@ journalctl -p err -b-1 -r # show error logs from previous boot in reverse order
 journalctl -u sshd -f # tail logs from sshd unit
 journalctl --no-pager -n 25 -k # show latest 25 logs from the kernel without pager
 journalctl --since=yesterday --until "2020-07-10 15:10:00" # show logs within specific time range
-journalctl CONTAINER_NAME=service_web_1 # show error from docker container named 'service_web_1'
+journalctl CONTAINER_NAME=service_web_1 # show error from the docker container named 'service_web_1'
 journalctl _PID=2434 -e # filter logs based on PID and jump to the end of the logs
-journalctl -g 'timed out' # filter logs based on regular expression. if the pattern is all lowercase, matching is case insensitive
+journalctl -g 'timed out' # filter logs based on a regular expression. if the pattern is all lowercase, it will become insensitive mode
 ```

 - g - go to the first line
@@ -812,11 +993,12 @@ dmidecode # shows configured clock speed
 smartctl -H /dev/sdd

 # umount the drive before this ops
+# [!] Never perform `fsck` on an unmounted LUKS drive, as it may lead to data loss
 e2fsck -C 0 -p /dev/sdd1 # preen
 e2fsck -C 0 -cc /dev/sdd1 # badblocks
 ```

-# Common issues
+# Troubleshooting

 ## Longer SSH login (D-bus glitch)

@@ -827,7 +1009,7 @@ systemctl restart polkit

 - [A comprehensive guide to fixing slow SSH logins – JRS Systems: the blog](https://jrs-s.net/2017/07/01/slow-ssh-logins/)

-## Annoying `systemd-homed is not available` log messages
+## Annoying `systemd-homed is not available` flooding journald logs

 Move `pam_unix` before `pam_systemd_home`.

@@ -870,7 +1052,7 @@ Audit=no

 ## Missing `/dev/nvidia-{uvm*,modeset}`

-This occurs after updating linux kernel.
+This usually happens right after updating the Linux kernel.

 - Run `docker run --rm --gpus all --device /dev/nvidia0 --device /dev/nvidiactl --device /dev/nvidia-modeset --device /dev/nvidia-uvm --device /dev/nvidia-uvm-tools -it nvidia/cuda:10.2-cudnn7-runtime nvidia-smi` once.