Add mTLS for all webhooks

Fix multiline sshkeys
Fix getty and cleanup in COS
Add router tests
Add oh-my-tmux setup script

README.md

@@ -137,6 +137,7 @@ ungrouped:
     pve_node: pve
     router_host: router
     router_password: <YOUR_PASSWORD>
+    router_sshkeys: [<YOUR_KEYS>]
     router_disk_size: 6G
     router_mem_gb: 2
     router_ncores: 6
@@ -151,7 +152,7 @@ ungrouped:
     idm_disk_size: 10G
     idm_mem_gb: 4
     idm_ncores: 6
-    idm_sshkeys: <YOUR_KEYS>
+    idm_sshkeys: [<YOUR_KEYS>]
     idm_ip: <YOUR_UNIQUE_IP_ADDR_FOR_THIS_NODE>
     ipa_password: <YOUR_PASSWORD>
     idm_default_group: cluster-user
@@ -160,7 +161,7 @@ ungrouped:
     mgmt_disk_size: 20G
     mgmt_mem_gb: 2
     mgmt_ncores: 6
-    mgmt_sshkeys: <YOUR_KEYS>
+    mgmt_sshkeys: [<YOUR_KEYS>]
     mgmt_rds_disk: "/dev/disk/by-id/<YOUR_DISK_LABEL>"
     mgmt_ip: <YOUR_UNIQUE_IP_ADDR_FOR_THIS_NODE>
     mgmt_netmask: 255.255.255.0
@@ -174,7 +175,7 @@ ungrouped:
     mgmt_cluster_name: staging
     login_message_of_the_day: <YOUR_MOTD>
     login_password: <YOUR_PASSWORD>
-    login_sshkeys: <YOUR_KEYS>
+    login_sshkeys: [<YOUR_KEYS>]
     login_disk_size_from_arch_map:
       x86_64: 10G
       aarch64: 10G

images/Containerfile.cos.warewulf

@@ -30,7 +30,7 @@ RUN echo "/boot/" > /etc/warewulf/excludes && \
 RUN echo "#!/bin/sh" > /etc/warewulf/container_exit.sh && \
     echo "export LANG=C LC_CTYPE=C" >> /etc/warewulf/container_exit.sh && \
     echo "set -x" >> /etc/warewulf/container_exit.sh && \
-    echo "dnf clean" >> /etc/warewulf/container_exit.sh
+    echo "dnf clean all" >> /etc/warewulf/container_exit.sh
 
 # Extract vmlinuz for aarch64 if compressed
 RUN arch=$(uname -m) && \
@@ -43,6 +43,10 @@ RUN arch=$(uname -m) && \
         fi; \
     fi
 
-# RUN dnf clean all
+# Restore getty 
+RUN systemctl unmask getty.target
+
+# Clean up
+RUN dnf clean all
 
 CMD ["/bin/echo", "-e", "For Warewulf provisioning only, do not run"]

playbook-svc-login.yml

@@ -100,6 +100,19 @@
 
     ### Configure webhookd provision notifications ###
 
+    - name: Create /etc/webhookd dir
+      ansible.builtin.file:
+        state: directory
+        path: /etc/webhookd
+        mode: 755
+
+    - name: Upload webhookd mTLS credentials
+      ansible.builtin.copy:
+        src: "{{backup_directory}}/mgmt.{{domain}}/{{item}}"
+        dest: "/etc/webhookd/{{item}}"
+        mode: 600
+      loop: [client.pem, client.key, server.pem]
+
     - name: Setup pam_exec user provision script
       ansible.builtin.template:
         src: webhookd_notify_provision.sh.j2

playbook-svc-mgmt.yml

@@ -11,7 +11,80 @@
       ansible.builtin.dnf:
         name: ["/bin/mailx", "Lmod"]
 
+    - name: Create /etc/webhookd dir
+      ansible.builtin.file:
+        state: directory
+        path: /etc/webhookd
+        mode: 755
+
+    - name: Generate webhookd privatekeys
+      community.crypto.openssl_privatekey:
+        path: "/etc/webhookd/{{item}}.key"
+        mode: 600
+      loop: [server, client]
+
+    - name: Create CSR for new certificate
+      community.crypto.openssl_csr_pipe:
+        privatekey_path: "/etc/webhookd/{{item}}.key"
+        use_common_name_for_san: false
+        common_name: local CA
+        subject_alt_name:
+          - "DNS:mgmt.{{domain}}"
+          - "DNS:{{hostvars[inventory_hostname]['ansible_default_ipv4']['address']}}"
+      register: csr
+      loop: [server, client]
+
+    - name: Generate webhookd publickeys
+      community.crypto.x509_certificate:
+        privatekey_path: "/etc/webhookd/{{item.item}}.key"
+        path: "/etc/webhookd/{{item.item}}.pem"
+        csr_content: "{{item.csr}}"
+        provider: selfsigned
+        selfsigned_not_after: "+3650d"
+        mode: 644
+      loop: "{{csr.results}}"
+
+    - name: Fetch webhookd keypairs
+      ansible.builtin.fetch:
+        flat: true
+        src: "/etc/webhookd/{{item}}"
+        dest: "{{host_backup_directory}}/{{item}}"
+      loop: [server.pem, server.key, client.pem, client.key]
+
+    - name: Allow binding webhook port in SELinux
+      community.general.seport:
+        ports: "{{mgmt_webhook_port}}"
+        proto: tcp
+        setype: http_port_t
+        state: present
+
+    - name: Allow network for httpd
+      ansible.posix.seboolean:
+        name: httpd_can_network_connect
+        state: true
+        persistent: true
+
   roles:
+    - role: geerlingguy.nginx
+      nginx_vhosts:
+        - listen: "{{mgmt_webhook_port}} ssl"
+          server_name: "webhook"
+          state: "present"
+          index: "/"
+          extra_parameters: |
+            location / {
+                proxy_pass http://127.0.0.1:8080; # webhookd defaults to this port
+            }
+            ssl_protocols       TLSv1.1 TLSv1.2;
+            ssl_ciphers         HIGH:!aNULL:!MD5;
+            ssl_session_cache   shared:SSL:10m;
+            ssl_session_timeout 10m;
+
+            ssl_certificate /etc/webhookd/server.pem;
+            ssl_certificate_key /etc/webhookd/server.key;
+            ssl_client_certificate /etc/webhookd/client.pem;
+            ssl_verify_client on;
+
     - role: linux-system-roles.postfix
       postfix_conf: { relayhost: "{{postfix_smtp_relay}}" }
 
@@ -66,6 +139,12 @@
   post_tasks:
     ### Update slurm nodes ###
 
+    - name: Copy setup oh-my-tmux script
+      ansible.builtin.copy:
+        src: scripts/setup-oh-my-tmux.sh
+        dest: /root/setup-oh-my-tmux.sh
+        mode: 755
+
     - name: "Setup HBAC: Disable allow all"
       freeipa.ansible_freeipa.ipahbacrule:
         ipaadmin_password: "{{ipa_password}}"
@@ -168,12 +247,24 @@
 
     - name: Setup munge key overlay
       ansible.builtin.copy:
-        src: munge.key.ww
+        content: '{%raw%}{{Include "/etc/munge/munge.key"}}{%endraw%}' # XXX templated by warewulf, not ansible
         dest: /srv/warewulf/overlays/generic/etc/munge/munge.key.ww
         owner: munge
         group: munge
         mode: 0400
 
+    - name: Ensure webhook key overlay path exists
+      ansible.builtin.file:
+        path: /srv/warewulf/overlays/generic/etc/webhookd/
+        state: directory
+
+    - name: Setup webhook key overlay
+      ansible.builtin.copy:
+        content: '{{''{{''}}Include "/etc/webhookd/{{item}}"}}' # XXX templated by warewulf, not ansible
+        dest: /srv/warewulf/overlays/generic/etc/webhookd/{{item}}.ww
+        mode: 0600
+      loop: [client.pem, client.key, server.pem]
+
     - name: Ensure add host overlay path exists
       ansible.builtin.file:
         path: /srv/warewulf/overlays/generic/usr/local/bin/
@@ -312,7 +403,7 @@
     - name: Build webhookd
       ansible.builtin.shell:
         cmd: |
-          go install -buildmode=pie -ldflags=-s github.com/ncarlier/webhookd@aad75eb
+          go install -buildmode=pie -ldflags=-s github.com/ncarlier/webhookd@0df2a52
           cp go/bin/webhookd /usr/bin/webhookd
           rm -rf go
         chdir: /root

playbook-vm-idm.yml

@@ -24,7 +24,7 @@
         vm_name: "idm.{{domain}}"
         vm_cores: "{{idm_ncores}}"
         vm_mem_gb: "{{idm_mem_gb}}"
-        vm_sshkeys: "{{idm_sshkeys | regex_replace('\\\\n', '\n')}}"
+        vm_sshkeys: "{{idm_sshkeys}}"
         vm_user: "root"
         vm_password: "{{idm_password}}"
         vm_nameservers: "{{router_ip}}"

playbook-vm-login.yml

@@ -29,7 +29,7 @@
         vm_name: "login-{{arch_to_dns_map[item]}}.{{domain}}"
         vm_cores: "{{login_ncores_from_arch_map[item]}}"
         vm_mem_gb: "{{login_mem_gb_from_arch_map[item]}}"
-        vm_sshkeys: "{{login_sshkeys | regex_replace('\\\\n', '\n')}}"
+        vm_sshkeys: "{{login_sshkeys}}"
         vm_user: "root"
         vm_password: "{{login_password}}"
         vm_nameservers: "{{router_ip}}"
@@ -49,7 +49,7 @@
         vm_arch: "{{item}}"
         vm_cores: "{{login_ncores_from_arch_map[item]}}"
         vm_mem_gb: "{{login_mem_gb_from_arch_map[item]}}"
-        vm_sshkeys: "{{login_sshkeys | regex_replace('\\\\n', '\n')}}"
+        vm_sshkeys: "{{login_sshkeys}}"
         vm_user: "root"
         vm_password: "{{login_password}}"
         vm_nameservers: "{{router_ip}}"

playbook-vm-mgmt.yml

@@ -44,7 +44,7 @@
         vm_name: "mgmt.{{domain}}"
         vm_cores: "{{mgmt_ncores}}"
         vm_mem_gb: "{{mgmt_mem_gb}}"
-        vm_sshkeys: "{{mgmt_sshkeys | regex_replace('\\\\n', '\n')}}"
+        vm_sshkeys: "{{mgmt_sshkeys}}"
         vm_user: "root"
         vm_password: "{{mgmt_password}}"
         vm_nameservers: "{{router_ip}}"

playbook-vm-router.yml

@@ -59,3 +59,15 @@
 
     - include_tasks: tasks/pve_vm_state.yml
       vars: { vm_name: "router.{{domain}}", vm_state: restarted }
+
+- hosts: router.{{domain}}
+  become: true
+  tasks:
+    - name: Check router connection works
+      ansible.builtin.shell: "hostname"
+      register: hostname
+
+    - name: Check if hostname is as expected
+      ansible.builtin.fail:
+        msg: "Expected hostname was 'router.{{domain}}', but got '{{hostname.stdout.strip()}}'"
+      when: hostname.stdout.strip() != inventory_hostname

requirements.yml

@@ -2,6 +2,9 @@ roles:
   - name: geerlingguy.mysql
     version: "4.3.4"
 
+  - name: geerlingguy.nginx
+    version: "3.2.0"
+
   - name: stackhpc.openhpc
     version: "v0.26.0"
 

scripts/setup-oh-my-tmux.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -eu
+
+cd
+
+if [ -d ".tmux" ]; then rm -rf .tmux; fi
+git clone https://github.com/gpakosz/.tmux.git
+ln -s -f .tmux/.tmux.conf .
+cp .tmux/.tmux.conf.local .

staging.rb

@@ -59,9 +59,9 @@ def self.write_inventory(pve_ip:, storage_pool:, extra_hosts:, host_common_hash:
     raise "Public key file #{SSH_PUBLIC_KEY} not found" unless File.file?(SSH_PUBLIC_KEY)
     raise "Public key file #{SSH_PRIVATE_KEY} not found" unless File.file?(SSH_PRIVATE_KEY)
 
-    ssh_pub_keys = [File.read(SSH_PUBLIC_KEY).strip]
+    ssh_pub_keys = [File.read(SSH_PUBLIC_KEY).strip, File.read(SSH_PUBLIC_KEY).strip]
     test_password = 'vagrant0' # IPA needs >= 8 characters
-    ssh_keys = ssh_pub_keys.join("\n")
+    ssh_keys = ssh_pub_keys
     pve_vars = {
       pve_username: 'root@pam',
       pve_password: 'vagrant',
@@ -180,6 +180,7 @@ def self.write_inventory(pve_ip:, storage_pool:, extra_hosts:, host_common_hash:
       "ungrouped": {
         "hosts":
         extra_hosts.merge(
+          "router.#{DOMAIN}": host_inventory_block(router_node_vars[:router_ip], SSH_PRIVATE_KEY, host_common_hash),
           "idm.#{DOMAIN}": host_inventory_block(idm_node_vars[:idm_ip], SSH_PRIVATE_KEY, host_common_hash),
           "mgmt.#{DOMAIN}": host_inventory_block(mgmt_node_vars[:mgmt_ip], SSH_PRIVATE_KEY, host_common_hash)
         ).merge(

tasks/pve_create_vm.yml

@@ -16,7 +16,7 @@
     scsihw: "virtio-scsi-pci"
     sata:
       sata0: "{{storage_pool}}:cloudinit,media=cdrom"
-    sshkeys: "{{vm_sshkeys}}"
+    sshkeys: "{{vm_sshkeys|join('\n')}}"
     ciuser: "{{vm_user}}"
     cipassword: "{{vm_password}}"
     ipconfig: "{{vm_ipconfig}}"

templates/ci-virt-user-data.yml.j2

@@ -6,7 +6,7 @@ user: {{vm_user}}
 disable_root: false
 password: {{vm_password}}
 ssh_authorized_keys:
-{% for item in vm_sshkeys.split('\n') %}
+{% for item in vm_sshkeys %}
   - {{item|trim}}
 {% endfor %}
 chpasswd:

templates/config.xml.j2

@@ -219,7 +219,7 @@
       <scope>system</scope>
       <groupname>admins</groupname>
       <password>{{router_password|password_hash('blowfish')}}</password>
-      <authorizedkeys>{{router_sshkeys|b64encode}}</authorizedkeys>
+      <authorizedkeys>{{router_sshkeys|join('\n')|b64encode}}</authorizedkeys>
       <uid>0</uid>
     </user>
     <nextuid>2000</nextuid>

templates/webhookd.j2

@@ -1,4 +1,3 @@
 ###
 # Webhookd configuration
-###
-WHD_LISTEN_ADDR=":{{mgmt_webhook_port}}"
+###

templates/webhookd_notify_add_host.sh.j2

@@ -8,7 +8,8 @@ host="$(hostname --short)"
 register_dns=false
 if [ "${1:-}" = "--register-dns" ]; then register_dns=true; fi
 
-curl -Ss -XPOST "http://mgmt.{{domain}}:{{mgmt_webhook_port}}/webhookd_exec_add_host?otp=$otp&host=$host&register_dns=$register_dns"
+curl -Ss -XPOST "https://mgmt.{{domain}}:{{mgmt_webhook_port}}/webhookd_exec_add_host?otp=$otp&host=$host&register_dns=$register_dns" \
+    --cert /etc/webhookd/client.pem --key /etc/webhookd/client.key --cacert /etc/webhookd/server.pem
 
 if ipa-client-install \
     --unattended --force-join --no-ntp --no-dns-sshfp --no-nisdomain --no-ssh --no-sshd --ssh-trust-dns \

templates/webhookd_notify_del_host.sh.j2

@@ -4,6 +4,7 @@ set -eu
 
 host="$(hostname --short)"
 
-curl -Ss -XPOST "http://mgmt.{{domain}}:{{mgmt_webhook_port}}/webhookd_exec_del_host?host=$host"
+curl -Ss -XPOST "https://mgmt.{{domain}}:{{mgmt_webhook_port}}/webhookd_exec_del_host?host=$host" \
+    --cert /etc/webhookd/client.pem --key /etc/webhookd/client.key --cacert /etc/webhookd/server.pem
 
 exit 0

templates/webhookd_notify_provision.sh.j2

@@ -1,6 +1,7 @@
 #!/bin/sh
 
-if [ "${PAM_TYPE}" = "open_session" ] && [ "${PAM_USER}" != "root" ] && [ ! -d "/nfs/home/${PAM_USER}" ]; then
-    curl -Ss -XPOST "http://mgmt.{{domain}}:{{mgmt_webhook_port}}/webhookd_exec_provision?user=$PAM_USER"
+if [ "${PAM_TYPE}" = "open_session" ] && [ "${PAM_USER}" != "root" ] && [ ! -d "/home/${PAM_USER}" ]; then
+    curl -Ss -XPOST "https://mgmt.{{domain}}:{{mgmt_webhook_port}}/webhookd_exec_provision?user=$PAM_USER" \
+        --cert /etc/webhookd/client.pem --key /etc/webhookd/client.key --cacert /etc/webhookd/server.pem
 fi
 exit 0