diff --git a/libvirthost.yml b/libvirthost.yml new file mode 100644 index 0000000..db93ed0 --- /dev/null +++ b/libvirthost.yml @@ -0,0 +1,6 @@ +--- +- hosts: libvirthost + roles: + - common + - crypto + - libvirthost diff --git a/roles/crypto/tasks/main.yml b/roles/crypto/tasks/main.yml new file mode 100644 index 0000000..f6a0c11 --- /dev/null +++ b/roles/crypto/tasks/main.yml @@ -0,0 +1,15 @@ +--- +- name: Cryptsetup part of this tasklist + shell: | + mkdir -p -m0700 /srv/common/luks + ( umask 0077 && dd if=/dev/urandom bs=1 count=4096 of=/srv/common/luks/crypt_dev_md1 conv=fsync ) + yes fai | cryptsetup luksAddKey /dev/md1 /srv/common/luks/crypt_dev_md1 + echo "crypt_dev_md1 /dev/md1 /srv/common/luks/crypt_dev_md1 luks,discard,key-slot=0" > /etc/crypttab + echo "KEYFILE_PATTERN=*" >> /etc/cryptsetup-initramfs/conf-hook + echo "UMASK=0077" >> /etc/initramfs-tools/conf.d/umask.conf + update-initramfs -u + args: + executable: /bin/sh + creates: /srv/common/luks + become: yes + diff --git a/roles/libvirthost/files/etc/default/grub.d/vfio.cfg b/roles/libvirthost/files/etc/default/grub.d/vfio.cfg new file mode 100644 index 0000000..7644897 --- /dev/null +++ b/roles/libvirthost/files/etc/default/grub.d/vfio.cfg @@ -0,0 +1,15 @@ +GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt rd.driver.pre=vfio-pci isolcpus=1,2,3,4,5,6,7 noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off tsx=on tsx_async_abort=off mitigations=off rcu_nocb_poll rcu-nocbs=2-7 default_hugepagesz=1G" +#GRUB_CMDLINE_LINUX_DEFAULT="quiet intel_iommu=on iommu=pt rd.driver.pre=vfio-pci vfio-pci.ids=10de:1b06,10de:10ef vfio_iommu_type1.allow_unsafe_interrupts=1 isolcpus=2,3,4,5,6,7" +#00:00.0 Host bridge [0600]: Intel Corporation 8th Gen Core 8-core Desktop Processor Host Bridge/DRAM Registers [Coffee Lake S] [8086:3e30] (rev 0d) +#00:01.0 PCI bridge [0604]: Intel Corporation Skylake PCIe Controller (x16) [8086:1901] (rev 0d) +#00:02.0 VGA compatible controller [0300]: Intel Corporation Device [8086:3e98] (rev 02) +#00:12.0 Signal processing controller [1180]: Intel Corporation Cannon Lake PCH Thermal Controller [8086:a379] (rev 10) +#00:14.0 USB controller [0c03]: Intel Corporation Cannon Lake PCH USB 3.1 xHCI Host Controller [8086:a36d] (rev 10) +#00:14.2 RAM memory [0500]: Intel Corporation Cannon Lake PCH Shared SRAM [8086:a36f] (rev 10) +#00:16.0 Communication controller [0780]: Intel Corporation Cannon Lake PCH HECI Controller [8086:a360] (rev 10) +#00:17.0 RAID bus controller [0104]: Intel Corporation SATA Controller [RAID mode] [8086:2822] (rev 10) +#00:1c.0 PCI bridge [0604]: Intel Corporation Cannon Lake PCH PCI Express Root Port [8086:a338] (rev f0) +#00:1f.0 ISA bridge [0601]: Intel Corporation Z390 Chipset LPC/eSPI Controller [8086:a305] (rev 10) +#00:1f.4 SMBus [0c05]: Intel Corporation Cannon Lake PCH SMBus Controller [8086:a323] (rev 10) +#00:1f.5 Serial bus controller [0c80]: Intel Corporation Cannon Lake PCH SPI Controller [8086:a324] (rev 10) +#00:1f.6 Ethernet controller [0200]: Intel Corporation Ethernet Connection (7) I219-V [8086:15bc] (rev 10) diff --git a/roles/libvirthost/files/etc/libvirt/hooks/qemu b/roles/libvirthost/files/etc/libvirt/hooks/qemu new file mode 100755 index 0000000..21e225a --- /dev/null +++ b/roles/libvirthost/files/etc/libvirt/hooks/qemu @@ -0,0 +1,8 @@ +#!/bin/sh +GUEST_NAME="$1" +STATE_NAME="$2" + +if [ "$STATE_NAME" = "stopped" ] && [ "$GUEST_NAME" = "rift" ]; then + systemctl suspend; +fi + diff --git a/roles/libvirthost/files/etc/modprobe.d/kvm-intel.conf b/roles/libvirthost/files/etc/modprobe.d/kvm-intel.conf new file mode 100644 index 0000000..42261eb --- /dev/null +++ b/roles/libvirthost/files/etc/modprobe.d/kvm-intel.conf @@ -0,0 +1,4 @@ +options kvm-intel nested=1 +options kvm-intel enable_shadow_vmcs=1 +options kvm-intel enable_apicv=1 +options kvm-intel ept=1 diff --git a/roles/libvirthost/files/etc/modprobe.d/vfio-pci.conf b/roles/libvirthost/files/etc/modprobe.d/vfio-pci.conf new file mode 100644 index 0000000..f2d3e44 --- /dev/null +++ b/roles/libvirthost/files/etc/modprobe.d/vfio-pci.conf @@ -0,0 +1,2 @@ +options vfio-pci ids=10de:1b06,10de:10ef,8086:a36d,8086:a36f + diff --git a/roles/libvirthost/files/etc/modules-load.d/vfio.conf b/roles/libvirthost/files/etc/modules-load.d/vfio.conf new file mode 100644 index 0000000..8885d91 --- /dev/null +++ b/roles/libvirthost/files/etc/modules-load.d/vfio.conf @@ -0,0 +1,4 @@ +vfio +vfio_iommu_type1 +vfio_pci +vfio_virqfd diff --git a/roles/libvirthost/files/etc/network/interfaces.d/eno1 b/roles/libvirthost/files/etc/network/interfaces.d/eno1 new file mode 100644 index 0000000..e32e661 --- /dev/null +++ b/roles/libvirthost/files/etc/network/interfaces.d/eno1 @@ -0,0 +1,2 @@ +iface eno1 manual + diff --git a/roles/libvirthost/files/etc/network/interfaces.d/virbr0 b/roles/libvirthost/files/etc/network/interfaces.d/virbr0 new file mode 100644 index 0000000..9ef7022 --- /dev/null +++ b/roles/libvirthost/files/etc/network/interfaces.d/virbr0 @@ -0,0 +1,7 @@ +auto virbr0 +iface virbr0 inet static + address 10.91.1.3 + gateway 10.91.1.1 + netmask 255.255.255.0 + bridge_ports eno1 + diff --git a/roles/libvirthost/files/etc/systemd/system/kvm-tuning.service b/roles/libvirthost/files/etc/systemd/system/kvm-tuning.service new file mode 100644 index 0000000..df96523 --- /dev/null +++ b/roles/libvirthost/files/etc/systemd/system/kvm-tuning.service @@ -0,0 +1,11 @@ +[Unit] +Description=Tune some system parameters for better KVM performance + +[Service] +Type=simple +RemainAfterExit=yes +ExecStart=/usr/local/bin/kvm-tuning.sh + +[Install] +WantedBy=multi-user.target + diff --git a/roles/libvirthost/files/etc/systemd/system/restore_rift.service b/roles/libvirthost/files/etc/systemd/system/restore_rift.service new file mode 100644 index 0000000..e53e6b5 --- /dev/null +++ b/roles/libvirthost/files/etc/systemd/system/restore_rift.service @@ -0,0 +1,11 @@ +[Unit] +Description=Restore rift +After=suspend.target hibernate.target hybrid-sleep.target + +[Service] +ExecStart=/usr/bin/virsh start rift +#User=my_user_name +#Environment=DISPLAY=:0 + +[Install] +WantedBy=suspend.target hibernate.target hybrid-sleep.target diff --git a/roles/libvirthost/files/usr/local/bin/kvm-tuning.sh b/roles/libvirthost/files/usr/local/bin/kvm-tuning.sh new file mode 100755 index 0000000..918ed66 --- /dev/null +++ b/roles/libvirthost/files/usr/local/bin/kvm-tuning.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +TOTAL_CORES='0-7' +HOST_CORES='0-1' # Cores reserved for host +HOST_CORES_MASK=C0 # 0-1,6-7, bitmask 0b000011000011 +VIRT_CORES='2-7' # Cores reserved for virtual machine(s) + +# Stretch package contains an old version of cset which doesn't work (or never worked) +#cset set -c $TOTAL_CORES -s machine.slice +#cset shield --kthread on --cpu $VIRT_CORES +# Reduce VM jitter: https://www.kernel.org/doc/Documentation/kernel-per-CPU-kthreads.txt +echo 3 > /proc/sys/vm/drop_caches +echo 1 > /proc/sys/vm/compact_memory +# the kernel's dirty page writeback mechanism uses kthread workers. They introduce +# massive arbitrary latencies when doing disk writes on the host and aren't +# migrated by cset. Restrict the workqueue to use only cpu 0. +echo $HOST_CORES_MASK > /sys/bus/workqueue/devices/writeback/cpumask +echo $HOST_CORES_MASK > /sys/devices/virtual/workqueue/cpumask +echo "C0" | tee /sys/devices/virtual/workqueue/*/cpumask +# THP can allegedly result in jitter. Better keep it off. +echo never > /sys/kernel/mm/transparent_hugepage/enabled +# Force P-states to P0 +echo "Setting governor on all cores:" +echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor +echo 0 > /sys/bus/workqueue/devices/writeback/numa +echo 0 > /proc/sys/kernel/watchdog +echo 300 > /proc/sys/vm/stat_interval + + diff --git a/roles/libvirthost/tasks/main.yml b/roles/libvirthost/tasks/main.yml new file mode 100644 index 0000000..0b0b6ca --- /dev/null +++ b/roles/libvirthost/tasks/main.yml @@ -0,0 +1,85 @@ +--- +- name: install packages + apt: + state: latest + pkg: + - libvirt-daemon + - libvirt-daemon-system + - qemu-system-x86 + - bridge-utils + - ovmf + - nvme-cli + - lm-sensors + - libhugetlbfs-bin + - iotop + install_recommends: no + force_apt_get: yes + become: yes + +- name: copy config files, scripts and services + copy: + src: "{{ item }}" + dest: "/{{ item }}" + force: yes + mode: preserve + become: yes + with_items: + - etc/default/grub.d/ + - etc/modules-load.d/ + - etc/modprobe.d/ + - etc/network/interfaces.d/ + - etc/systemd/system/ + - etc/libvirt/hooks/ + - usr/local/bin/ + +- name: hugepages config + lineinfile: + state: present + path: /etc/fstab + regexp: "^hugetlbfs" + line: "hugetlbfs /dev/hugepages hugetlbfs defaults 0 0" + become: yes + +- name: add user to grups + user: + name: plantroon + groups: libvirt,kvm + append: yes + become: yes + +- name: kernel param tuning + sysctl: + state: present + name: "{{ item.name }}" + value: "{{ item.value }}" + sysctl_file: /etc/sysctl.d/vfio.conf + reload: yes + become: yes + with_items: + #TOTAL_RAM/2048kB (size in proc/meminfo) 13824 for 32 gig + - { name: 'vm.nr_hugepages', value: '32' } + - { name: 'vm.stat_interval', value: '120' } + - { name: 'kernel.watchdog', value: '0' } + +- name: enable tuning service + systemd: + name: kvm-tuning + state: started + enabled: yes + become: yes + +- name: enable restore_rift service + systemd: + name: restore_rift + state: started + enabled: yes + become: yes + +- name: update grub configs to reflect the above changes + command: update-grub + become: yes + +- name: update initramfs + command: "update-initramfs -u" + become: yes +