From f2bbb426dc7875c54cde92b5709cf98dd7df0890 Mon Sep 17 00:00:00 2001 From: Adam Rozman Date: Wed, 28 Jun 2023 12:10:27 +0000 Subject: [PATCH] add custom ipxe firmware building support This commit adds the following: - Support for detecting iPXE certificates - When iPXE certs are detected, the iPXE services and configs will be advertised with https addresses - Enforce chainloading of the iPXE firmware provided by the user even if the client machine comes with an iPXE firmware by default - Adds modular iPXE apache2 config file to enable separate iPXE specific virtual host (port) where TLS is enforced - Custom iPXE config file template has been added thus Ironic can generate node specific iPXE config files that use the custom iPXE firmware - A set of environment variable that control the new iPXE related features Signed-off-by: Adam Rozman --- Dockerfile | 3 +- ironic-config/apache2-ipxe.conf.j2 | 35 +++++++++++++ ironic-config/dnsmasq.conf.j2 | 12 ++++- ironic-config/ipxe_config.template | 81 ++++++++++++++++++++++++++++++ ironic-config/ironic.conf.j2 | 3 ++ scripts/ironic-common.sh | 1 + scripts/rundnsmasq | 8 +++ scripts/runhttpd | 11 ++++ scripts/tls-common.sh | 25 +++++++++ 9 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 ironic-config/apache2-ipxe.conf.j2 create mode 100644 ironic-config/ipxe_config.template diff --git a/Dockerfile b/Dockerfile index 03ecaa70d..57a1e59e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ COPY --from=ironic-builder /tmp/ipxe/src/bin/undionly.kpxe /tmp/ipxe/src/bin-x86 COPY --from=ironic-builder /tmp/esp.img /tmp/uefi_esp.img COPY ironic-config/ironic.conf.j2 /etc/ironic/ -COPY ironic-config/inspector.ipxe.j2 ironic-config/httpd-ironic-api.conf.j2 /tmp/ +COPY ironic-config/inspector.ipxe.j2 ironic-config/httpd-ironic-api.conf.j2 ironic-config/ipxe_config.template /tmp/ # DNSMASQ COPY ironic-config/dnsmasq.conf.j2 /etc/ @@ -55,6 +55,7 @@ COPY ironic-config/dnsmasq.conf.j2 /etc/ COPY ironic-config/httpd.conf.j2 /etc/httpd/conf/ COPY ironic-config/httpd-modules.conf /etc/httpd/conf.modules.d/ COPY ironic-config/apache2-vmedia.conf.j2 /etc/httpd-vmedia.conf.j2 +COPY ironic-config/apache2-ipxe.conf.j2 /etc/httpd-ipxe.conf.j2 # IRONIC-INSPECTOR # RUN mkdir -p /var/lib/ironic /var/lib/ironic-inspector && \ diff --git a/ironic-config/apache2-ipxe.conf.j2 b/ironic-config/apache2-ipxe.conf.j2 new file mode 100644 index 000000000..32fb9e458 --- /dev/null +++ b/ironic-config/apache2-ipxe.conf.j2 @@ -0,0 +1,35 @@ +Listen {{ env.IPXE_TLS_PORT }} + + + ErrorLog /dev/stderr + LogLevel debug + CustomLog /dev/stdout combined + + SSLEngine on + SSLProtocol {{ env.IPXE_SSL_PROTOCOL }} + SSLCertificateFile {{ env.IPXE_CERT_FILE }} + SSLCertificateKeyFile {{ env.IPXE_KEY_FILE }} + + + Order Allow,Deny + Allow from all + + + Order Deny,Allow + Deny from all + + + + + SSLRequireSSL + + + SSLRequireSSL + + + SSLRequireSSL + + + SSLRequireSSL + + diff --git a/ironic-config/dnsmasq.conf.j2 b/ironic-config/dnsmasq.conf.j2 index 9d3f59259..b6e7a72b0 100644 --- a/ironic-config/dnsmasq.conf.j2 +++ b/ironic-config/dnsmasq.conf.j2 @@ -29,13 +29,23 @@ dhcp-option=option{% if ":" in env["DNS_IP"] %}6{% endif %}:dns-server,{{ env["D # IPv4 Configuration: dhcp-match=ipxe,175 # Client is already running iPXE; move to next stage of chainloading +{%- if env.IPXE_TLS_SETUP == "true" %} +# iPXE with (U)EFI +dhcp-boot=tag:efi,tag:ipxe,http://{{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}/custom-ipxe/snponly.efi +# iPXE with BIOS +dhcp-boot=tag:ipxe,http://{{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}/custom-ipxe/undionly.kpxe +{% else %} dhcp-boot=tag:ipxe,http://{{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}/boot.ipxe +{% endif %} # Note: Need to test EFI booting dhcp-match=set:efi,option:client-arch,7 dhcp-match=set:efi,option:client-arch,9 dhcp-match=set:efi,option:client-arch,11 -# Client is PXE booting over EFI without iPXE ROM; send EFI version of iPXE chainloader +# Client is PXE booting over EFI without iPXE ROM; send EFI version of iPXE chainloader do the same also if iPXE ROM boots but TLS is enabled +{%- if env.IPXE_TLS_SETUP == "true" %} +dhcp-boot=tag:efi,tag:ipxe,snponly.efi +{% endif %} dhcp-boot=tag:efi,tag:!ipxe,snponly.efi # Client is running PXE over BIOS; send BIOS version of iPXE chainloader diff --git a/ironic-config/ipxe_config.template b/ironic-config/ipxe_config.template new file mode 100644 index 000000000..33ba26eb4 --- /dev/null +++ b/ironic-config/ipxe_config.template @@ -0,0 +1,81 @@ +#!ipxe + +set attempts:int32 10 +set i:int32 0 + +goto deploy + +:deploy +imgfree +{%- if pxe_options.deployment_aki_path %} +{%- set aki_path_https_elements = pxe_options.deployment_aki_path.split(':') %} +{%- set aki_port_and_path = aki_path_https_elements[2].split('/') %} +{%- set aki_afterport = aki_port_and_path[1:]|join('/') %} +{%- set aki_path_https = ['https:', aki_path_https_elements[1], ':8084/', aki_afterport]|join %} +{%- endif %} +{%- if pxe_options.deployment_ari_path %} +{%- set ari_path_https_elements = pxe_options.deployment_ari_path.split(':') %} +{%- set ari_port_and_path = ari_path_https_elements[2].split('/') %} +{%- set ari_afterport = ari_port_and_path[1:]|join('/') %} +{%- set ari_path_https = ['https:', ari_path_https_elements[1], ':8084/', ari_afterport]|join %} +{%- endif %} +kernel {% if pxe_options.ipxe_timeout > 0 %}--timeout {{ pxe_options.ipxe_timeout }} {% endif %}{{ aki_path_https }} selinux=0 troubleshoot=0 text {{ pxe_options.pxe_append_params|default("", true) }} BOOTIF=${mac} initrd={{ pxe_options.initrd_filename|default("deploy_ramdisk", true) }} || goto retry + +initrd {% if pxe_options.ipxe_timeout > 0 %}--timeout {{ pxe_options.ipxe_timeout }} {% endif %}{{ ari_path_https }} || goto retry +boot + +:retry +iseq ${i} ${attempts} && goto fail || +inc i +echo No response, retrying in ${i} seconds. +sleep ${i} +goto deploy + +:fail +echo Failed to get a response after ${attempts} attempts +echo Powering off in 30 seconds. +sleep 30 +poweroff + +:boot_anaconda +imgfree +kernel {% if pxe_options.ipxe_timeout > 0 %}--timeout {{ pxe_options.ipxe_timeout }} {% endif %}{{ aki_path_https }} text {{ pxe_options.pxe_append_params|default("", true) }} inst.ks={{ pxe_options.ks_cfg_url }} {% if pxe_options.repo_url %}inst.repo={{ pxe_options.repo_url }}{% else %}inst.stage2={{ pxe_options.stage2_url }}{% endif %} initrd=ramdisk || goto boot_anaconda +initrd {% if pxe_options.ipxe_timeout > 0 %}--timeout {{ pxe_options.ipxe_timeout }} {% endif %}{{ ari_path_https }} || goto boot_anaconda +boot + +:boot_ramdisk +imgfree +{%- if pxe_options.boot_iso_url %} +sanboot {{ pxe_options.boot_iso_url }} +{%- else %} +kernel {% if pxe_options.ipxe_timeout > 0 %}--timeout {{ pxe_options.ipxe_timeout }} {% endif %}{{ aki_path_https }} root=/dev/ram0 text {{ pxe_options.pxe_append_params|default("", true) }} {{ pxe_options.ramdisk_opts|default('', true) }} initrd=ramdisk || goto boot_ramdisk +initrd {% if pxe_options.ipxe_timeout > 0 %}--timeout {{ pxe_options.ipxe_timeout }} {% endif %}{{ ari_path_https }} || goto boot_ramdisk +boot +{%- endif %} + +{%- if pxe_options.boot_from_volume %} + +:boot_iscsi +imgfree +{% if pxe_options.username %}set username {{ pxe_options.username }}{% endif %} +{% if pxe_options.password %}set password {{ pxe_options.password }}{% endif %} +{% if pxe_options.iscsi_initiator_iqn %}set initiator-iqn {{ pxe_options.iscsi_initiator_iqn }}{% endif %} +sanhook --drive 0x80 {{ pxe_options.iscsi_boot_url }} || goto fail_iscsi_retry +{%- if pxe_options.iscsi_volumes %}{% for i, volume in enumerate(pxe_options.iscsi_volumes) %} +set username {{ volume.username }} +set password {{ volume.password }} +{%- set drive_id = 129 + i %} +sanhook --drive {{ '0x%x' % drive_id }} {{ volume.url }} || goto fail_iscsi_retry +{%- endfor %}{% endif %} +{% if pxe_options.iscsi_volumes %}set username {{ pxe_options.username }}{% endif %} +{% if pxe_options.iscsi_volumes %}set password {{ pxe_options.password }}{% endif %} +sanboot --no-describe || goto fail_iscsi_retry + +:fail_iscsi_retry +echo Failed to attach iSCSI volume(s), retrying in 10 seconds. +sleep 10 +goto boot_iscsi +{%- endif %} + +:boot_whole_disk +sanboot --no-describe || exit 0 diff --git a/ironic-config/ironic.conf.j2 b/ironic-config/ironic.conf.j2 index 497d217ef..61e747dc4 100644 --- a/ironic-config/ironic.conf.j2 +++ b/ironic-config/ironic.conf.j2 @@ -115,6 +115,8 @@ ramdisk_image_download_source = {{ env.IRONIC_BOOT_ISO_SOURCE }} {% endif %} {% if env.IRONIC_EXTERNAL_HTTP_URL %} external_http_url = {{ env.IRONIC_EXTERNAL_HTTP_URL }} +{% elif env.IPXE_TLS_SETUP == "true" %} +external_http_url = https://{{ env.IRONIC_URL_HOST }}:{{ env.IPXE_TLS_PORT }} {% elif env.IRONIC_VMEDIA_TLS_SETUP == "true" %} external_http_url = https://{{ env.IRONIC_URL_HOST }}:{{ env.VMEDIA_TLS_PORT }} {% endif %} @@ -204,6 +206,7 @@ kernel_append_params = nofb nomodeset vga=normal ipa-insecure=1 {% if env.IRONIC enable_netboot_fallback = true # Enable the fallback path to ironic-inspector ipxe_fallback_script = inspector.ipxe +ipxe_config_template = /tmp/ipxe_config.template [redfish] use_swift = false diff --git a/scripts/ironic-common.sh b/scripts/ironic-common.sh index 292134139..b9d2dffef 100644 --- a/scripts/ironic-common.sh +++ b/scripts/ironic-common.sh @@ -6,6 +6,7 @@ IRONIC_IP="${IRONIC_IP:-}" PROVISIONING_INTERFACE="${PROVISIONING_INTERFACE:-}" PROVISIONING_IP="${PROVISIONING_IP:-}" PROVISIONING_MACS="${PROVISIONING_MACS:-}" +IPXE_CUSTOM_FIRMWARE_DIR="${IPXE_CUSTOM_FIRMWARE_DIR:-/shared/custom_ipxe_firmware}" get_provisioning_interface() { diff --git a/scripts/rundnsmasq b/scripts/rundnsmasq index caf83d5fe..817898c92 100755 --- a/scripts/rundnsmasq +++ b/scripts/rundnsmasq @@ -4,6 +4,8 @@ set -eux # shellcheck disable=SC1091 . /bin/ironic-common.sh +# shellcheck disable=SC1091 +. /bin/tls-common.sh export HTTP_PORT=${HTTP_PORT:-80} DNSMASQ_EXCEPT_INTERFACE=${DNSMASQ_EXCEPT_INTERFACE:-lo} @@ -19,7 +21,13 @@ mkdir -p /shared/html/images mkdir -p /shared/html/pxelinux.cfg # Copy files to shared mount +if [[ -r "${IPXE_CUSTOM_FIRMWARE_DIR}" ]]; then +cp "${IPXE_CUSTOM_FIRMWARE_DIR}/undionly.kpxe" \ + "${IPXE_CUSTOM_FIRMWARE_DIR}/snponly.efi" \ + "/shared/tftpboot" +else cp /tftpboot/undionly.kpxe /tftpboot/snponly.efi /shared/tftpboot +fi # Template and write dnsmasq.conf # we template via /tmp as sed otherwise creates temp files in /etc directory diff --git a/scripts/runhttpd b/scripts/runhttpd index 6b84de4d4..0ed61e94e 100755 --- a/scripts/runhttpd +++ b/scripts/runhttpd @@ -30,6 +30,9 @@ wait_for_interface_or_ip mkdir -p /shared/html chmod 0777 /shared/html +mkdir -p /shared/html/custom-ipxe +chmod 0777 /shared/html/custom-ipxe + IRONIC_BASE_URL="${IRONIC_SCHEME}://${IRONIC_URL_HOST}" INSPECTOR_EXTRA_ARGS=" ipa-inspection-callback-url=${IRONIC_BASE_URL}:${IRONIC_INSPECTOR_ACCESS_PORT}/v1/continue" @@ -79,6 +82,14 @@ if [[ "$IRONIC_VMEDIA_TLS_SETUP" == "true" ]]; then render_j2_config /etc/httpd-vmedia.conf.j2 /etc/httpd/conf.d/vmedia.conf fi +# Render httpd TLS configuration for /shared/html +if [[ "$IPXE_TLS_SETUP" == "true" ]]; then + render_j2_config "/etc/httpd-ipxe.conf.j2" "/etc/httpd/conf.d/ipxe.conf" + cp "${IPXE_CUSTOM_FIRMWARE_DIR}/undionly.kpxe" \ + "${IPXE_CUSTOM_FIRMWARE_DIR}/snponly.efi" \ + "/shared/html/custom-ipxe" +fi + # Set up inotify to kill the container (restart) whenever cert files for ironic inspector change if [[ "$IRONIC_INSPECTOR_TLS_SETUP" == "true" ]] && [[ "${RESTART_CONTAINER_CERTIFICATE_UPDATED}" == "true" ]]; then # shellcheck disable=SC2034 diff --git a/scripts/tls-common.sh b/scripts/tls-common.sh index 202a114b9..1c41de8ad 100644 --- a/scripts/tls-common.sh +++ b/scripts/tls-common.sh @@ -5,6 +5,7 @@ export IRONIC_KEY_FILE=/certs/ironic/tls.key export IRONIC_CACERT_FILE=/certs/ca/ironic/tls.crt export IRONIC_INSECURE=${IRONIC_INSECURE:-false} export IRONIC_SSL_PROTOCOL=${IRONIC_SSL_PROTOCOL:-"-ALL +TLSv1.2 +TLSv1.3"} +export IPXE_SSL_PROTOCOL=${IPXE_SSL_PROTOCOL:-"-ALL +TLSv1.2 +TLSv1.3"} export IRONIC_VMEDIA_SSL_PROTOCOL=${IRONIC_VMEDIA_SSL_PROTOCOL:-"ALL"} export IRONIC_INSPECTOR_CERT_FILE=/certs/ironic-inspector/tls.crt @@ -15,14 +16,21 @@ export IRONIC_INSPECTOR_INSECURE=${IRONIC_INSPECTOR_INSECURE:-$IRONIC_INSECURE} export IRONIC_VMEDIA_CERT_FILE=/certs/vmedia/tls.crt export IRONIC_VMEDIA_KEY_FILE=/certs/vmedia/tls.key +export IPXE_CERT_FILE=/certs/ipxe/tls.crt +export IPXE_KEY_FILE=/certs/ipxe/tls.key + export RESTART_CONTAINER_CERTIFICATE_UPDATED=${RESTART_CONTAINER_CERTIFICATE_UPDATED:-"false"} export MARIADB_CACERT_FILE=/certs/ca/mariadb/tls.crt +export IPXE_TLS_PORT="${IPXE_TLS_PORT:-8084}" + mkdir -p /certs/ironic mkdir -p /certs/ironic-inspector mkdir -p /certs/ca/ironic mkdir -p /certs/ca/ironic-inspector +mkdir -p /certs/ipxe +mkdir -p /certs/vmedia if [[ -f "$IRONIC_CERT_FILE" ]] && [[ ! -f "$IRONIC_KEY_FILE" ]]; then echo "Missing TLS Certificate key file $IRONIC_KEY_FILE" @@ -51,6 +59,15 @@ if [[ ! -f "$IRONIC_VMEDIA_CERT_FILE" ]] && [[ -f "$IRONIC_VMEDIA_KEY_FILE" ]]; exit 1 fi +if [[ -f "$IPXE_CERT_FILE" ]] && [[ ! -f "$IPXE_KEY_FILE" ]]; then + echo "Missing TLS Certificate key file $IRONIC_IPXE_CERT_FILE" + exit 1 +fi +if [[ ! -f "$IPXE_CERT_FILE" ]] && [[ -f "$IRONIC_VMEDIA_KEY_FILE" ]]; then + echo "Missing TLS Certificate file $IRONIC_VMEDIA_CERT_FILE" + exit 1 +fi + copy_atomic() { local src="$1" @@ -94,6 +111,14 @@ else export IRONIC_VMEDIA_TLS_SETUP="false" fi +if [[ -f "$IPXE_CERT_FILE" ]]; then + export IPXE_SCHEME="https" + export IPXE_TLS_SETUP="true" +else + export IPXE_SCHEME="http" + export IPXE_TLS_SETUP="false" +fi + if [[ -f "$MARIADB_CACERT_FILE" ]]; then export MARIADB_TLS_ENABLED="true" else