From 5c8c4b2a50ba368f0dc9208da28acd654aa0cc47 Mon Sep 17 00:00:00 2001
From: yozhao101 <56170650+yozhao101@users.noreply.github.com>
Date: Tue, 3 Mar 2020 16:50:32 -0800
Subject: [PATCH] [Services] Restart BGP service upon unexpected critical
 process exit. (#4207)

---
 dockers/docker-database/supervisord.conf.j2          | 1 -
 dockers/docker-fpm-frr/Dockerfile.j2                 | 2 ++
 dockers/docker-fpm-frr/critical_processes            | 5 +++++
 dockers/docker-fpm-frr/supervisord.conf              | 8 +++++++-
 dockers/docker-fpm-gobgp/Dockerfile.j2               | 2 ++
 dockers/docker-fpm-gobgp/critical_processes          | 2 ++
 dockers/docker-fpm-gobgp/supervisord.conf            | 8 +++++++-
 dockers/docker-fpm-quagga/Dockerfile.j2              | 2 ++
 dockers/docker-fpm-quagga/critical_processes         | 4 ++++
 dockers/docker-fpm-quagga/supervisord.conf           | 8 +++++++-
 files/build_templates/single_instance/bgp.service.j2 | 4 ++++
 rules/docker-fpm-frr.mk                              | 2 ++
 rules/docker-fpm-gobgp.mk                            | 1 +
 rules/docker-fpm-quagga.mk                           | 2 ++
 14 files changed, 47 insertions(+), 4 deletions(-)
 create mode 100644 dockers/docker-fpm-frr/critical_processes
 create mode 100644 dockers/docker-fpm-gobgp/critical_processes
 create mode 100644 dockers/docker-fpm-quagga/critical_processes

diff --git a/dockers/docker-database/supervisord.conf.j2 b/dockers/docker-database/supervisord.conf.j2
index 442bec1438c8..9e855527edba 100644
--- a/dockers/docker-database/supervisord.conf.j2
+++ b/dockers/docker-database/supervisord.conf.j2
@@ -9,7 +9,6 @@ events=PROCESS_STATE_EXITED
 autostart=true
 autorestart=unexpected
 
-
 [program:rsyslogd]
 command=/bin/bash -c "rm -f /var/run/rsyslogd.pid && /usr/sbin/rsyslogd -n"
 priority=1
diff --git a/dockers/docker-fpm-frr/Dockerfile.j2 b/dockers/docker-fpm-frr/Dockerfile.j2
index 418676c18329..1c670682a342 100644
--- a/dockers/docker-fpm-frr/Dockerfile.j2
+++ b/dockers/docker-fpm-frr/Dockerfile.j2
@@ -50,6 +50,8 @@ COPY ["snmp.conf", "/etc/snmp/frr.conf"]
 COPY ["TSA", "/usr/bin/TSA"]
 COPY ["TSB", "/usr/bin/TSB"]
 COPY ["TSC", "/usr/bin/TSC"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 RUN chmod a+x /usr/bin/TSA && \
     chmod a+x /usr/bin/TSB && \
     chmod a+x /usr/bin/TSC
diff --git a/dockers/docker-fpm-frr/critical_processes b/dockers/docker-fpm-frr/critical_processes
new file mode 100644
index 000000000000..8ea09e1bb538
--- /dev/null
+++ b/dockers/docker-fpm-frr/critical_processes
@@ -0,0 +1,5 @@
+zebra
+staticd
+bgpd
+fpmsyncd
+bgpcfgd
diff --git a/dockers/docker-fpm-frr/supervisord.conf b/dockers/docker-fpm-frr/supervisord.conf
index fe0ce6eda1a4..3e544b64b296 100644
--- a/dockers/docker-fpm-frr/supervisord.conf
+++ b/dockers/docker-fpm-frr/supervisord.conf
@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name bgp
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
@@ -25,7 +31,7 @@ stderr_logfile=syslog
 command=/usr/sbin/rsyslogd -n
 priority=3
 autostart=false
-autorestart=false
+autorestart=unexpected
 startsecs=0
 stdout_logfile=syslog
 stderr_logfile=syslog
diff --git a/dockers/docker-fpm-gobgp/Dockerfile.j2 b/dockers/docker-fpm-gobgp/Dockerfile.j2
index 1e333d9026c0..b9b969edeb09 100644
--- a/dockers/docker-fpm-gobgp/Dockerfile.j2
+++ b/dockers/docker-fpm-gobgp/Dockerfile.j2
@@ -24,5 +24,7 @@ COPY ["start.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["*.j2", "/usr/share/sonic/templates/"]
 COPY ["daemons", "/etc/quagga/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 
 ENTRYPOINT ["/usr/bin/supervisord"]
diff --git a/dockers/docker-fpm-gobgp/critical_processes b/dockers/docker-fpm-gobgp/critical_processes
new file mode 100644
index 000000000000..2a9e47831e0d
--- /dev/null
+++ b/dockers/docker-fpm-gobgp/critical_processes
@@ -0,0 +1,2 @@
+gobgpd
+fpmsyncd
diff --git a/dockers/docker-fpm-gobgp/supervisord.conf b/dockers/docker-fpm-gobgp/supervisord.conf
index 4e635f4093d4..b814dc024fa3 100644
--- a/dockers/docker-fpm-gobgp/supervisord.conf
+++ b/dockers/docker-fpm-gobgp/supervisord.conf
@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name bgp
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
@@ -15,7 +21,7 @@ stderr_logfile=syslog
 command=/usr/sbin/rsyslogd -n
 priority=2
 autostart=false
-autorestart=false
+autorestart=unexpected
 stdout_logfile=syslog
 stderr_logfile=syslog
 
diff --git a/dockers/docker-fpm-quagga/Dockerfile.j2 b/dockers/docker-fpm-quagga/Dockerfile.j2
index f048e2789429..9d1312d073ee 100644
--- a/dockers/docker-fpm-quagga/Dockerfile.j2
+++ b/dockers/docker-fpm-quagga/Dockerfile.j2
@@ -33,5 +33,7 @@ RUN rm -rf /debs ~/.cache
 COPY ["bgpcfgd", "start.sh", "/usr/bin/"]
 COPY ["supervisord.conf", "/etc/supervisor/conf.d/"]
 COPY ["*.j2", "/usr/share/sonic/templates/"]
+COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
+COPY ["critical_processes", "/etc/supervisor"]
 
 ENTRYPOINT ["/usr/bin/supervisord"]
diff --git a/dockers/docker-fpm-quagga/critical_processes b/dockers/docker-fpm-quagga/critical_processes
new file mode 100644
index 000000000000..f151af9c4bdd
--- /dev/null
+++ b/dockers/docker-fpm-quagga/critical_processes
@@ -0,0 +1,4 @@
+zebra
+bgpd
+fpmsyncd
+bgpcfgd
diff --git a/dockers/docker-fpm-quagga/supervisord.conf b/dockers/docker-fpm-quagga/supervisord.conf
index c4351fafefe9..7397a7428a08 100644
--- a/dockers/docker-fpm-quagga/supervisord.conf
+++ b/dockers/docker-fpm-quagga/supervisord.conf
@@ -3,6 +3,12 @@ logfile_maxbytes=1MB
 logfile_backups=2
 nodaemon=true
 
+[eventlistener:supervisor-proc-exit-listener]
+command=/usr/bin/supervisor-proc-exit-listener --container-name bgp
+events=PROCESS_STATE_EXITED
+autostart=true
+autorestart=unexpected
+
 [program:start.sh]
 command=/usr/bin/start.sh
 priority=1
@@ -25,7 +31,7 @@ stderr_logfile=syslog
 command=/usr/sbin/rsyslogd -n
 priority=3
 autostart=false
-autorestart=false
+autorestart=unexpected
 startsecs=0
 stdout_logfile=syslog
 stderr_logfile=syslog
diff --git a/files/build_templates/single_instance/bgp.service.j2 b/files/build_templates/single_instance/bgp.service.j2
index 7200a0e3ecf2..fdf9d9c78c04 100644
--- a/files/build_templates/single_instance/bgp.service.j2
+++ b/files/build_templates/single_instance/bgp.service.j2
@@ -3,12 +3,16 @@ Description=BGP container
 Requires=updategraph.service
 After=updategraph.service
 Before=ntp-config.service
+StartLimitIntervalSec=1200
+StartLimitBurst=3
 
 [Service]
 User={{ sonicadmin_user }}
 ExecStartPre=/usr/bin/{{docker_container_name}}.sh start
 ExecStart=/usr/bin/{{docker_container_name}}.sh wait
 ExecStop=/usr/bin/{{docker_container_name}}.sh stop
+Restart=always
+RestartSec=30
 
 [Install]
 WantedBy=multi-user.target
diff --git a/rules/docker-fpm-frr.mk b/rules/docker-fpm-frr.mk
index 1d918830ab55..517aff3ea9fc 100644
--- a/rules/docker-fpm-frr.mk
+++ b/rules/docker-fpm-frr.mk
@@ -26,6 +26,8 @@ $(DOCKER_FPM_FRR)_RUN_OPT += --privileged -t
 $(DOCKER_FPM_FRR)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 $(DOCKER_FPM_FRR)_RUN_OPT += -v /etc/sonic/frr:/etc/frr:rw
 
+$(DOCKER_FPM_FRR)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
+
 $(DOCKER_FPM_FRR)_BASE_IMAGE_FILES += vtysh:/usr/bin/vtysh
 $(DOCKER_FPM_FRR)_BASE_IMAGE_FILES += TSA:/usr/bin/TSA
 $(DOCKER_FPM_FRR)_BASE_IMAGE_FILES += TSB:/usr/bin/TSB
diff --git a/rules/docker-fpm-gobgp.mk b/rules/docker-fpm-gobgp.mk
index 3e1839d47099..03ec88e85b56 100644
--- a/rules/docker-fpm-gobgp.mk
+++ b/rules/docker-fpm-gobgp.mk
@@ -9,3 +9,4 @@ SONIC_DOCKER_IMAGES += $(DOCKER_FPM_GOBGP)
 $(DOCKER_FPM_GOBGP)_CONTAINER_NAME = bgp
 $(DOCKER_FPM_GOBGP)_RUN_OPT += --privileged -t
 $(DOCKER_FPM_GOBGP)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
+$(DOCKER_FPM_GOBPG)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
diff --git a/rules/docker-fpm-quagga.mk b/rules/docker-fpm-quagga.mk
index 2c78d1917b0e..5a384eedab2c 100644
--- a/rules/docker-fpm-quagga.mk
+++ b/rules/docker-fpm-quagga.mk
@@ -10,4 +10,6 @@ $(DOCKER_FPM_QUAGGA)_CONTAINER_NAME = bgp
 $(DOCKER_FPM_QUAGGA)_RUN_OPT += --privileged -t
 $(DOCKER_FPM_QUAGGA)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 
+$(DOCKER_FPM_QUAGGA)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT)
+
 $(DOCKER_FPM_QUAGGA)_BASE_IMAGE_FILES += vtysh:/usr/bin/vtysh