diff --git a/lib/pharos/cluster_manager.rb b/lib/pharos/cluster_manager.rb index 0f36f8d6d..aa252e166 100644 --- a/lib/pharos/cluster_manager.rb +++ b/lib/pharos/cluster_manager.rb @@ -77,7 +77,6 @@ def apply_phases master_only = [config.master_host] apply_phase(Phases::MigrateMaster, master_hosts, parallel: true) apply_phase(Phases::ConfigureHost, config.hosts, parallel: true) - apply_phase(Phases::ConfigureFirewalld, config.hosts, parallel: true) apply_phase(Phases::ConfigureClient, master_only, parallel: false) unless @config.etcd&.endpoints @@ -104,6 +103,7 @@ def apply_phases # configure essential services apply_phase(Phases::ConfigurePriorityClasses, master_only) apply_phase(Phases::ConfigurePSP, master_only) + apply_phase(Phases::ConfigureFirewalld, master_only) apply_phase(Phases::ConfigureCloudProvider, master_only) apply_phase(Phases::ConfigureDNS, master_only) apply_phase(Phases::ConfigureWeave, master_only) if config.network.provider == 'weave' diff --git a/lib/pharos/phases/configure_calico.rb b/lib/pharos/phases/configure_calico.rb index f5a5bacfd..e64457a04 100644 --- a/lib/pharos/phases/configure_calico.rb +++ b/lib/pharos/phases/configure_calico.rb @@ -59,8 +59,6 @@ def call master_ip: master_host.peer_address, version: CALICO_VERSION, nat_outgoing: @config.network.calico&.nat_outgoing, - firewalld_enabled: !!@config.network&.firewalld&.enabled, - reload_iptables: !!cluster_context['reload-iptables'], envs: @config.network.calico&.environment || {}, metrics_enabled: metrics_enabled?, metrics_port: metrics_port, diff --git a/lib/pharos/phases/configure_firewalld.rb b/lib/pharos/phases/configure_firewalld.rb index 9ce115628..c36262c1c 100644 --- a/lib/pharos/phases/configure_firewalld.rb +++ b/lib/pharos/phases/configure_firewalld.rb @@ -5,6 +5,13 @@ module Phases class ConfigureFirewalld < Pharos::Phase title "Configure firewalld" + PHAROS_FIREWALLD_VERSION = "0.1.0" + + register_component( + name: 'pharos-firewalld', version: PHAROS_FIREWALLD_VERSION, license: 'Apache License 2.0', + enabled: proc { |c| c.network&.firewalld&.enabled } + ) + def call if @config.network&.firewalld&.enabled configure_firewalld @@ -14,35 +21,23 @@ def call end def configure_firewalld - logger.info { 'Configuring firewalld packages ...' } - @host.configurer.configure_firewalld - - logger.info { 'Configuring firewalld rules ...' } - - write_config('services/pharos-master.xml', pharos_master_service) if @host.master? - write_config('services/pharos-worker.xml', pharos_worker_service) - write_config('ipsets/pharos.xml', pharos_ipset) - - # Masquerade was enabled in the past, if it's still enabled we need to reload firewalld rules - @firewalld_reload = true if masquerade_active? - - return unless firewalld_reload? - - cluster_context['reload-iptables'] = true - logger.info { 'Reloading firewalld ...' } - exec_script( - 'configure-firewalld.sh', - ROLE: @host.role + logger.info { 'Configuring firewalld ...' } + + apply_stack( + 'firewalld', + image_repository: @config.image_repository, + version: PHAROS_FIREWALLD_VERSION, + services: { + master: pharos_master_service, + worker: pharos_worker_service + }, + ipset: pharos_ipset ) end - def firewalld_reload? - !!@firewalld_reload - end - def disable_firewalld logger.info { 'Firewalld not enabled, disabling ...' } - exec_script('disable-firewalld.sh') + delete_stack('firewalld') end # @param file [String] @@ -60,7 +55,7 @@ def trusted_addresses addresses = @config.hosts.flat_map { |host| [host.address, host.private_address, host.private_interface_address].compact.uniq } - addresses += [@config.network.pod_network_cidr, @config.network.service_cidr] + addresses += [@config.network.pod_network_cidr, @config.network.service_cidr, '127.0.0.1'] addresses += @config.network.firewalld.trusted_subnets if @config.network.firewalld&.trusted_subnets addresses @@ -102,10 +97,6 @@ def pharos_ipset entries: trusted_addresses ) end - - def masquerade_active? - transport.exec("firewall-cmd --query-masquerade > /dev/null 2>&1").success? - end end end end diff --git a/lib/pharos/phases/configure_weave.rb b/lib/pharos/phases/configure_weave.rb index e8af7aedf..d16d00ba9 100644 --- a/lib/pharos/phases/configure_weave.rb +++ b/lib/pharos/phases/configure_weave.rb @@ -61,8 +61,6 @@ def ensure_resources ipalloc_range: @config.network.pod_network_cidr, arch: @host.cpu_arch, version: WEAVE_VERSION, - firewalld_enabled: firewalld?, - reload_iptables: reload_iptables?, known_peers: known_peers, initial_known_peers: initial_known_peers, flying_shuttle_enabled: flying_shuttle?, @@ -105,16 +103,6 @@ def known_peers @config.hosts.map(&:peer_address) end - # @return [Boolean] - def firewalld? - !!@config.network&.firewalld&.enabled - end - - # @return [Boolean] - def reload_iptables? - !!cluster_context['reload-iptables'] - end - # @return [Boolean] def flying_shuttle? return true if known_peers diff --git a/lib/pharos/resources/calico/25-node-daemonset.yml.erb b/lib/pharos/resources/calico/25-node-daemonset.yml.erb index 463d5f65a..737c14dcc 100644 --- a/lib/pharos/resources/calico/25-node-daemonset.yml.erb +++ b/lib/pharos/resources/calico/25-node-daemonset.yml.erb @@ -26,7 +26,6 @@ spec: # priority scheduling and that its resources are reserved # if it ever gets evicted. scheduler.alpha.kubernetes.io/critical-pod: '' - kontena.io/firewalld: "<%= firewalld_enabled %>" spec: hostNetwork: true hostPID: true @@ -45,17 +44,6 @@ spec: # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. terminationGracePeriodSeconds: 0 initContainers: - <% if firewalld_enabled && reload_iptables %> - # This container performs firewalld reload - - name: reload-firewalld - image: <%= image_repository %>/alpine:3.9 - command: ["/bin/sh", "-c"] - env: - - name: TIMESTAMP - value: "<%= Time.now.to_f %>" - args: - - pkill -HUP firewalld - <% end %> # This container performs upgrade from host-local IPAM to calico-ipam. # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. diff --git a/lib/pharos/resources/firewalld/configmap.yml.erb b/lib/pharos/resources/firewalld/configmap.yml.erb new file mode 100644 index 000000000..ca3d9e130 --- /dev/null +++ b/lib/pharos/resources/firewalld/configmap.yml.erb @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pharos-firewalld + namespace: kube-system +data: + pharos-master.xml: <%= services[:master].dump %> + pharos-worker.xml: <%= services[:worker].dump %> + pharos.xml: <%= ipset.dump %> + diff --git a/lib/pharos/resources/firewalld/daemonset.yml.erb b/lib/pharos/resources/firewalld/daemonset.yml.erb new file mode 100644 index 000000000..02e9cd12a --- /dev/null +++ b/lib/pharos/resources/firewalld/daemonset.yml.erb @@ -0,0 +1,74 @@ +<% %w(master worker).each do |role| %> +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: pharos-firewalld-<%= role %> + labels: + name: pharos-firewalld-<%= role %> + namespace: kube-system +spec: + template: + metadata: + labels: + name: pharos-firewalld-<%= role %> + spec: + <%- if role == 'master' -%> + nodeSelector: + node-role.kubernetes.io/master: '' + <%- else -%> + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: DoesNotExist + <%- end -%> + containers: + - name: firewalld + image: '<%= image_repository %>/pharos-firewalld:<%= version %>' + env: + - name: FIREWALLD_ROLE + value: <%= role %> + resources: + requests: + cpu: 10m + memory: 32Mi + securityContext: + privileged: true + volumeMounts: + - name: firewalld + mountPath: /etc/firewalld/configmap + - name: lib-modules + mountPath: /lib/modules + - name: xtables-lock + mountPath: /run/xtables.lock + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - kill -s RTMIN+3 1 + hostNetwork: true + restartPolicy: Always + securityContext: + seLinuxOptions: {} + priorityClassName: system-node-critical + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - name: firewalld + configMap: + name: pharos-firewalld + - name: lib-modules + hostPath: + path: /lib/modules + - name: xtables-lock + hostPath: + path: /run/xtables.lock + updateStrategy: + type: RollingUpdate +<% end %> diff --git a/lib/pharos/resources/weave/daemon-set.yml.erb b/lib/pharos/resources/weave/daemon-set.yml.erb index 71c97851f..668702bf5 100644 --- a/lib/pharos/resources/weave/daemon-set.yml.erb +++ b/lib/pharos/resources/weave/daemon-set.yml.erb @@ -12,21 +12,7 @@ spec: metadata: labels: name: weave-net - annotations: - kontena.io/firewalld: "<%= firewalld_enabled %>" spec: - <% if firewalld_enabled && reload_iptables %> - initContainers: - # This container performs firewalld reload - - name: reload-firewalld - image: <%= image_repository %>/alpine:3.9 - command: ["/bin/sh", "-c"] - env: - - name: TIMESTAMP - value: "<%= Time.now.to_f %>" - args: - - pkill -HUP firewalld - <% end %> containers: - name: weave command: @@ -61,12 +47,19 @@ spec: name: weave-passwd key: weave-passwd image: '<%= image_repository %>/weave-kube:<%= version %>' - livenessProbe: + readinessProbe: httpGet: host: 127.0.0.1 path: /status port: 6784 - initialDelaySeconds: 30 + livenessProbe: + exec: + command: + - /bin/sh + - -c + - iptables-save | grep -E -e '^-A WEAVE.+MASQUERADE$' + initialDelaySeconds: 60 + periodSeconds: 60 resources: requests: cpu: 10m diff --git a/lib/pharos/scripts/configure-firewalld.sh b/lib/pharos/scripts/configure-firewalld.sh deleted file mode 100644 index d964dcc8f..000000000 --- a/lib/pharos/scripts/configure-firewalld.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -set -e - -RELOAD="false" -# reload only if this is first run -if ! firewall-cmd --get-services | grep pharos-worker > /dev/null 2>&1 ; then - RELOAD="true" - firewall-cmd --reload - sleep 10 -fi - -if [ "$ROLE" = "master" ]; then - firewall-cmd --permanent --add-service pharos-master -fi - -firewall-cmd --permanent --add-service pharos-worker -firewall-cmd --permanent --add-source ipset:pharos --zone trusted -if firewall-cmd --query-masquerade > /dev/null 2>&1 ; then - firewall-cmd --remove-masquerade --permanent -fi - -if [[ "${RELOAD}" = "true" ]]; then - firewall-cmd --reload - sleep 10 -fi diff --git a/lib/pharos/scripts/disable-firewalld.sh b/lib/pharos/scripts/disable-firewalld.sh deleted file mode 100644 index cfb649150..000000000 --- a/lib/pharos/scripts/disable-firewalld.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -e - -if systemctl is-active --quiet firewalld; then - systemctl disable firewalld - systemctl stop firewalld -fi diff --git a/lib/pharos_cluster.rb b/lib/pharos_cluster.rb index 2bbd53328..82df361f6 100644 --- a/lib/pharos_cluster.rb +++ b/lib/pharos_cluster.rb @@ -11,7 +11,7 @@ module Pharos CNI_VERSION = '0.7.5' COREDNS_VERSION = '1.3.1' CRIO_VERSION = '1.14.6' - DNS_NODE_CACHE_VERSION = '1.15.1' + DNS_NODE_CACHE_VERSION = '1.15.2' ETCD_VERSION = ENV.fetch('ETCD_VERSION') { '3.3.10' } KUBE_VERSION = ENV.fetch('KUBE_VERSION') { '1.15.3' } KUBEADM_VERSION = ENV.fetch('KUBEADM_VERSION') { KUBE_VERSION }