Firewalld as DaemonSet

lib/pharos/cluster_manager.rb

@@ -77,7 +77,6 @@ def apply_phases
       master_only = [config.master_host]
       apply_phase(Phases::MigrateMaster, master_hosts, parallel: true)
       apply_phase(Phases::ConfigureHost, config.hosts, parallel: true)
-      apply_phase(Phases::ConfigureFirewalld, config.hosts, parallel: true)
       apply_phase(Phases::ConfigureClient, master_only, parallel: false)
 
       unless @config.etcd&.endpoints
@@ -104,6 +103,7 @@ def apply_phases
       # configure essential services
       apply_phase(Phases::ConfigurePriorityClasses, master_only)
       apply_phase(Phases::ConfigurePSP, master_only)
+      apply_phase(Phases::ConfigureFirewalld, master_only)
       apply_phase(Phases::ConfigureCloudProvider, master_only)
       apply_phase(Phases::ConfigureDNS, master_only)
       apply_phase(Phases::ConfigureWeave, master_only) if config.network.provider == 'weave'

lib/pharos/phases/configure_calico.rb

@@ -59,8 +59,6 @@ def call
           master_ip: master_host.peer_address,
           version: CALICO_VERSION,
           nat_outgoing: @config.network.calico&.nat_outgoing,
-          firewalld_enabled: !!@config.network&.firewalld&.enabled,
-          reload_iptables: !!cluster_context['reload-iptables'],
           envs: @config.network.calico&.environment || {},
           metrics_enabled: metrics_enabled?,
           metrics_port: metrics_port,

lib/pharos/phases/configure_firewalld.rb

@@ -5,6 +5,13 @@ module Phases
     class ConfigureFirewalld < Pharos::Phase
       title "Configure firewalld"
 
+      PHAROS_FIREWALLD_VERSION = "0.1.0"
+
+      register_component(
+        name: 'pharos-firewalld', version: PHAROS_FIREWALLD_VERSION, license: 'Apache License 2.0',
+        enabled: proc { |c| c.network&.firewalld&.enabled }
+      )
+
       def call
         if @config.network&.firewalld&.enabled
           configure_firewalld
@@ -14,35 +21,23 @@ def call
       end
 
       def configure_firewalld
-        logger.info { 'Configuring firewalld packages ...' }
-        @host.configurer.configure_firewalld
-
-        logger.info { 'Configuring firewalld rules ...' }
-
-        write_config('services/pharos-master.xml', pharos_master_service) if @host.master?
-        write_config('services/pharos-worker.xml', pharos_worker_service)
-        write_config('ipsets/pharos.xml', pharos_ipset)
-
-        # Masquerade was enabled in the past, if it's still enabled we need to reload firewalld rules
-        @firewalld_reload = true if masquerade_active?
-
-        return unless firewalld_reload?
-
-        cluster_context['reload-iptables'] = true
-        logger.info { 'Reloading firewalld ...' }
-        exec_script(
-          'configure-firewalld.sh',
-          ROLE: @host.role
+        logger.info { 'Configuring firewalld ...' }
+
+        apply_stack(
+          'firewalld',
+          image_repository: @config.image_repository,
+          version: PHAROS_FIREWALLD_VERSION,
+          services: {
+            master: pharos_master_service,
+            worker: pharos_worker_service
+          },
+          ipset: pharos_ipset
         )
       end
 
-      def firewalld_reload?
-        !!@firewalld_reload
-      end
-
       def disable_firewalld
         logger.info { 'Firewalld not enabled, disabling ...' }
-        exec_script('disable-firewalld.sh')
+        delete_stack('firewalld')
       end
 
       # @param file [String]
@@ -60,7 +55,7 @@ def trusted_addresses
         addresses = @config.hosts.flat_map { |host|
           [host.address, host.private_address, host.private_interface_address].compact.uniq
         }
-        addresses += [@config.network.pod_network_cidr, @config.network.service_cidr]
+        addresses += [@config.network.pod_network_cidr, @config.network.service_cidr, '127.0.0.1']
         addresses += @config.network.firewalld.trusted_subnets if @config.network.firewalld&.trusted_subnets
 
         addresses
@@ -102,10 +97,6 @@ def pharos_ipset
           entries: trusted_addresses
         )
       end
-
-      def masquerade_active?
-        transport.exec("firewall-cmd --query-masquerade > /dev/null 2>&1").success?
-      end
     end
   end
 end

lib/pharos/phases/configure_weave.rb

@@ -61,8 +61,6 @@ def ensure_resources
           ipalloc_range: @config.network.pod_network_cidr,
           arch: @host.cpu_arch,
           version: WEAVE_VERSION,
-          firewalld_enabled: firewalld?,
-          reload_iptables: reload_iptables?,
           known_peers: known_peers,
           initial_known_peers: initial_known_peers,
           flying_shuttle_enabled: flying_shuttle?,
@@ -105,16 +103,6 @@ def known_peers
         @config.hosts.map(&:peer_address)
       end
 
-      # @return [Boolean]
-      def firewalld?
-        !!@config.network&.firewalld&.enabled
-      end
-
-      # @return [Boolean]
-      def reload_iptables?
-        !!cluster_context['reload-iptables']
-      end
-
       # @return [Boolean]
       def flying_shuttle?
         return true if known_peers

lib/pharos/resources/calico/25-node-daemonset.yml.erb

@@ -26,7 +26,6 @@ spec:
         # priority scheduling and that its resources are reserved
         # if it ever gets evicted.
         scheduler.alpha.kubernetes.io/critical-pod: ''
-        kontena.io/firewalld: "<%= firewalld_enabled %>"
     spec:
       hostNetwork: true
       hostPID: true
@@ -45,17 +44,6 @@ spec:
       # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
       terminationGracePeriodSeconds: 0
       initContainers:
-        <% if firewalld_enabled && reload_iptables %>
-        # This container performs firewalld reload
-        - name: reload-firewalld
-          image: <%= image_repository %>/alpine:3.9
-          command: ["/bin/sh", "-c"]
-          env:
-            - name: TIMESTAMP
-              value: "<%= Time.now.to_f %>"
-          args:
-          - pkill -HUP firewalld
-        <% end %>
         # This container performs upgrade from host-local IPAM to calico-ipam.
         # It can be deleted if this is a fresh installation, or if you have already
         # upgraded to use calico-ipam.

lib/pharos/resources/firewalld/configmap.yml.erb

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: pharos-firewalld
+  namespace: kube-system
+data:
+  pharos-master.xml: <%= services[:master].dump %>
+  pharos-worker.xml: <%= services[:worker].dump %>
+  pharos.xml: <%= ipset.dump %>
+

lib/pharos/resources/firewalld/daemonset.yml.erb

@@ -0,0 +1,74 @@
+<% %w(master worker).each do |role| %>
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: pharos-firewalld-<%= role %>
+  labels:
+    name: pharos-firewalld-<%= role %>
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      labels:
+        name: pharos-firewalld-<%= role %>
+    spec:
+      <%- if role == 'master' -%>
+      nodeSelector:
+        node-role.kubernetes.io/master: ''
+      <%- else -%>
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: DoesNotExist
+      <%- end -%>
+      containers:
+        - name: firewalld
+          image: '<%= image_repository %>/pharos-firewalld:<%= version %>'
+          env:
+           - name: FIREWALLD_ROLE
+             value: <%= role %>
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: firewalld
+              mountPath: /etc/firewalld/configmap
+            - name: lib-modules
+              mountPath: /lib/modules
+            - name: xtables-lock
+              mountPath: /run/xtables.lock
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - /bin/sh
+                - -c
+                - kill -s RTMIN+3 1
+      hostNetwork: true
+      restartPolicy: Always
+      securityContext:
+        seLinuxOptions: {}
+      priorityClassName: system-node-critical
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+      volumes:
+        - name: firewalld
+          configMap:
+            name: pharos-firewalld
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+  updateStrategy:
+    type: RollingUpdate
+<% end %>

lib/pharos/resources/weave/daemon-set.yml.erb

@@ -12,21 +12,7 @@ spec:
     metadata:
       labels:
         name: weave-net
-      annotations:
-        kontena.io/firewalld: "<%= firewalld_enabled %>"
     spec:
-      <% if firewalld_enabled && reload_iptables %>
-      initContainers:
-        # This container performs firewalld reload
-        - name: reload-firewalld
-          image: <%= image_repository %>/alpine:3.9
-          command: ["/bin/sh", "-c"]
-          env:
-            - name: TIMESTAMP
-              value: "<%= Time.now.to_f %>"
-          args:
-          - pkill -HUP firewalld
-      <% end %>
       containers:
         - name: weave
           command:
@@ -61,12 +47,19 @@ spec:
                   name: weave-passwd
                   key: weave-passwd
           image: '<%= image_repository %>/weave-kube:<%= version %>'
-          livenessProbe:
+          readinessProbe:
             httpGet:
               host: 127.0.0.1
               path: /status
               port: 6784
-            initialDelaySeconds: 30
+          livenessProbe:
+            exec:
+              command:
+              - /bin/sh
+              - -c
+              - iptables-save | grep -E -e '^-A WEAVE.+MASQUERADE$'
+              initialDelaySeconds: 60
+              periodSeconds: 60
           resources:
             requests:
               cpu: 10m

lib/pharos_cluster.rb

@@ -11,7 +11,7 @@ module Pharos
   CNI_VERSION = '0.7.5'
   COREDNS_VERSION = '1.3.1'
   CRIO_VERSION = '1.14.6'
-  DNS_NODE_CACHE_VERSION = '1.15.1'
+  DNS_NODE_CACHE_VERSION = '1.15.2'
   ETCD_VERSION = ENV.fetch('ETCD_VERSION') { '3.3.10' }
   KUBE_VERSION = ENV.fetch('KUBE_VERSION') { '1.15.3' }
   KUBEADM_VERSION = ENV.fetch('KUBEADM_VERSION') { KUBE_VERSION }