diff --git a/fpmsyncd/Makefile.am b/fpmsyncd/Makefile.am index bae2fd73a738..d0f27588be68 100644 --- a/fpmsyncd/Makefile.am +++ b/fpmsyncd/Makefile.am @@ -1,4 +1,4 @@ -INCLUDES = -I $(top_srcdir) -I $(FPM_PATH) +INCLUDES = -I $(top_srcdir) -I $(top_srcdir)/warmrestart -I $(FPM_PATH) bin_PROGRAMS = fpmsyncd @@ -8,9 +8,8 @@ else DBGFLAGS = -g endif -fpmsyncd_SOURCES = fpmsyncd.cpp fpmlink.cpp routesync.cpp +fpmsyncd_SOURCES = fpmsyncd.cpp fpmlink.cpp routesync.cpp $(top_srcdir)/warmrestart/warmRestartHelper.cpp $(top_srcdir)/warmrestart/warmRestartHelper.h fpmsyncd_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) fpmsyncd_CPPFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) fpmsyncd_LDADD = -lnl-3 -lnl-route-3 -lswsscommon - diff --git a/fpmsyncd/fpmsyncd.cpp b/fpmsyncd/fpmsyncd.cpp index d5a54f8fee31..42eba65f0f55 100644 --- a/fpmsyncd/fpmsyncd.cpp +++ b/fpmsyncd/fpmsyncd.cpp @@ -1,13 +1,24 @@ #include #include "logger.h" #include "select.h" +#include "selectabletimer.h" #include "netdispatcher.h" +#include "warmRestartHelper.h" #include "fpmsyncd/fpmlink.h" #include "fpmsyncd/routesync.h" + using namespace std; using namespace swss; + +/* + * Default warm-restart timer interval for routing-stack app. To be used only if + * no explicit value has been defined in configuration. + */ +const uint32_t DEFAULT_ROUTING_RESTART_INTERVAL = 120; + + int main(int argc, char **argv) { swss::Logger::linkToDbNative("fpmsyncd"); @@ -18,25 +29,75 @@ int main(int argc, char **argv) NetDispatcher::getInstance().registerMessageHandler(RTM_NEWROUTE, &sync); NetDispatcher::getInstance().registerMessageHandler(RTM_DELROUTE, &sync); - while (1) + while (true) { try { FpmLink fpm; Select s; + SelectableTimer warmStartTimer(timespec{0, 0}); - cout << "Waiting for connection..." << endl; + /* + * Pipeline should be flushed right away to deal with state pending + * from previous try/catch iterations. + */ + pipeline.flush(); + + cout << "Waiting for fpm-client connection..." << endl; fpm.accept(); cout << "Connected!" << endl; s.addSelectable(&fpm); + + /* If warm-restart feature is enabled, execute 'restoration' logic */ + bool warmStartEnabled = sync.m_warmStartHelper.checkAndStart(); + if (warmStartEnabled) + { + /* Obtain warm-restart timer defined for routing application */ + uint32_t warmRestartIval = sync.m_warmStartHelper.getRestartTimer(); + if (!warmRestartIval) + { + warmStartTimer.setInterval(timespec{DEFAULT_ROUTING_RESTART_INTERVAL, 0}); + } + else + { + warmStartTimer.setInterval(timespec{warmRestartIval, 0}); + } + + /* Execute restoration instruction and kick off warm-restart timer */ + if (sync.m_warmStartHelper.runRestoration()) + { + warmStartTimer.start(); + s.addSelectable(&warmStartTimer); + } + } + while (true) { Selectable *temps; - /* Reading FPM messages forever (and calling "readData" to read them) */ + + /* Reading FPM messages forever (and calling "readMe" to read them) */ s.select(&temps); - pipeline.flush(); - SWSS_LOG_DEBUG("Pipeline flushed"); + + /* + * Upon expiration of the warm-restart timer, proceed to run the + * reconciliation process and remove warm-restart timer from + * select() loop. + */ + if (warmStartEnabled && temps == &warmStartTimer) + { + SWSS_LOG_NOTICE("Warm-Restart timer expired."); + sync.m_warmStartHelper.reconcile(); + s.removeSelectable(&warmStartTimer); + + pipeline.flush(); + SWSS_LOG_DEBUG("Pipeline flushed"); + } + else if (!warmStartEnabled || sync.m_warmStartHelper.isReconciled()) + { + pipeline.flush(); + SWSS_LOG_DEBUG("Pipeline flushed"); + } } } catch (FpmLink::FpmConnectionClosedException &e) diff --git a/fpmsyncd/routesync.cpp b/fpmsyncd/routesync.cpp index 488410c9e3da..63805d9d0a6a 100644 --- a/fpmsyncd/routesync.cpp +++ b/fpmsyncd/routesync.cpp @@ -14,7 +14,8 @@ using namespace std; using namespace swss; RouteSync::RouteSync(RedisPipeline *pipeline) : - m_routeTable(pipeline, APP_ROUTE_TABLE_NAME, true) + m_routeTable(pipeline, APP_ROUTE_TABLE_NAME, true), + m_warmStartHelper(pipeline, &m_routeTable, APP_ROUTE_TABLE_NAME, "bgp", "bgp") { m_nl_sock = nl_socket_alloc(); nl_connect(m_nl_sock, NETLINK_ROUTE); @@ -38,10 +39,31 @@ void RouteSync::onMsg(int nlmsg_type, struct nl_object *obj) return; } + /* + * Upon arrival of a delete msg we could either push the change right away, + * or we could opt to defer it if we are going through a warm-reboot cycle. + */ + bool warmRestartInProgress = m_warmStartHelper.inProgress(); + if (nlmsg_type == RTM_DELROUTE) { - m_routeTable.del(destipprefix); - return; + if (!warmRestartInProgress) + { + m_routeTable.del(destipprefix); + return; + } + else + { + SWSS_LOG_INFO("Warm-Restart mode: Receiving delete msg: %s\n", + destipprefix); + + vector fvVector; + const KeyOpFieldsValuesTuple kfv = std::make_tuple(destipprefix, + DEL_COMMAND, + fvVector); + m_warmStartHelper.insertRefreshMap(kfv); + return; + } } else if (nlmsg_type != RTM_NEWROUTE) { @@ -118,8 +140,29 @@ void RouteSync::onMsg(int nlmsg_type, struct nl_object *obj) vector fvVector; FieldValueTuple nh("nexthop", nexthops); FieldValueTuple idx("ifname", ifnames); + fvVector.push_back(nh); fvVector.push_back(idx); - m_routeTable.set(destipprefix, fvVector); - SWSS_LOG_DEBUG("RoutTable set: %s %s %s\n", destipprefix, nexthops.c_str(), ifnames.c_str()); + + if (!warmRestartInProgress) + { + m_routeTable.set(destipprefix, fvVector); + SWSS_LOG_DEBUG("RouteTable set msg: %s %s %s\n", + destipprefix, nexthops.c_str(), ifnames.c_str()); + } + + /* + * During routing-stack restarting scenarios route-updates will be temporarily + * put on hold by warm-reboot logic. + */ + else + { + SWSS_LOG_INFO("Warm-Restart mode: RouteTable set msg: %s %s %s\n", + destipprefix, nexthops.c_str(), ifnames.c_str()); + + const KeyOpFieldsValuesTuple kfv = std::make_tuple(destipprefix, + SET_COMMAND, + fvVector); + m_warmStartHelper.insertRefreshMap(kfv); + } } diff --git a/fpmsyncd/routesync.h b/fpmsyncd/routesync.h index 43b6305287e8..1652bedee7b5 100644 --- a/fpmsyncd/routesync.h +++ b/fpmsyncd/routesync.h @@ -4,6 +4,8 @@ #include "dbconnector.h" #include "producerstatetable.h" #include "netmsg.h" +#include "warmRestartHelper.h" + namespace swss { @@ -16,10 +18,12 @@ class RouteSync : public NetMsg virtual void onMsg(int nlmsg_type, struct nl_object *obj); + WarmStartHelper m_warmStartHelper; + private: - ProducerStateTable m_routeTable; - struct nl_cache *m_link_cache; - struct nl_sock *m_nl_sock; + ProducerStateTable m_routeTable; + struct nl_cache *m_link_cache; + struct nl_sock *m_nl_sock; }; } diff --git a/tests/conftest.py b/tests/conftest.py index 0c5cdda970d6..d8f5e2f082f6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -290,6 +290,26 @@ def stop_swss(self): cmd += "supervisorctl stop {}; ".format(pname) self.runcmd(['sh', '-c', cmd]) + def start_zebra(dvs): + dvs.runcmd(['sh', '-c', 'supervisorctl start zebra']) + + # Let's give zebra a chance to connect to FPM. + time.sleep(5) + + def stop_zebra(dvs): + dvs.runcmd(['sh', '-c', 'pkill -x zebra']) + time.sleep(1) + + def start_fpmsyncd(dvs): + dvs.runcmd(['sh', '-c', 'supervisorctl start fpmsyncd']) + + # Let's give fpmsyncd a chance to connect to Zebra. + time.sleep(5) + + def stop_fpmsyncd(dvs): + dvs.runcmd(['sh', '-c', 'pkill -x fpmsyncd']) + time.sleep(1) + def init_asicdb_validator(self): self.asicdb = AsicDbValidator(self) @@ -334,13 +354,18 @@ def get_logs(self, modname=None): raise RuntimeError("Failed to unpack the archive.") os.system("chmod a+r -R log") - def add_log_marker(self): + def add_log_marker(self, file=None): marker = "=== start marker {} ===".format(datetime.now().isoformat()) - self.ctn.exec_run("logger {}".format(marker)) + + if file: + self.runcmd(['sh', '-c', "echo \"{}\" >> {}".format(marker, file)]) + else: + self.ctn.exec_run("logger {}".format(marker)) + return marker def SubscribeAppDbObject(self, objpfx): - r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.APP_DB) + r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.APPL_DB) pubsub = r.pubsub() pubsub.psubscribe("__keyspace@0__:%s*" % objpfx) return pubsub @@ -375,26 +400,40 @@ def CountSubscribedObjects(self, pubsub, ignore=None, timeout=10): return (nadd, ndel) def GetSubscribedAppDbObjects(self, pubsub, ignore=None, timeout=10): - r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.APP_DB) + r = redis.Redis(unix_socket_path=self.redis_sock, db=swsscommon.APPL_DB) addobjs = [] delobjs = [] idle = 0 + prev_key = None while True and idle < timeout: message = pubsub.get_message() if message: print message key = message['channel'].split(':', 1)[1] + # In producer/consumer_state_table scenarios, every entry will + # show up twice for every push/pop operation, so skip the second + # one to avoid double counting. + if key != None and key == prev_key: + continue + # Skip instructions with meaningless keys. To be extended in the + # future to other undesired keys. + if key == "ROUTE_TABLE_KEY_SET" or key == "ROUTE_TABLE_DEL_SET": + continue if ignore: fds = message['channel'].split(':') if fds[2] in ignore: continue + if message['data'] == 'hset': + (_, k) = key.split(':', 1) value=r.hgetall(key) - addobjs.append({'key':k, 'vals':value}) + addobjs.append({'key':json.dumps(k), 'vals':json.dumps(value)}) + prev_key = key elif message['data'] == 'del': - delobjs.append(key) + (_, k) = key.split(':', 1) + delobjs.append({'key':json.dumps(k)}) idle = 0 else: time.sleep(1) @@ -424,7 +463,8 @@ def GetSubscribedAsicDbObjects(self, pubsub, ignore=None, timeout=10): (_, t, k) = key.split(':', 2) addobjs.append({'type':t, 'key':k, 'vals':value}) elif message['data'] == 'del': - delobjs.append(key) + (_, t, k) = key.split(':', 2) + delobjs.append({'key':k}) idle = 0 else: time.sleep(1) diff --git a/tests/test_warm_reboot.py b/tests/test_warm_reboot.py index e5f2de6ed608..ace175872258 100644 --- a/tests/test_warm_reboot.py +++ b/tests/test_warm_reboot.py @@ -494,7 +494,7 @@ def test_swss_neighbor_syncup(dvs, testlog): # # Testcase 4: - # Stop neighsyncd, add even nummber of ipv4/ipv6 neighbor entries to each interface again, + # Stop neighsyncd, add even nummber of ipv4/ipv6 neighbor entries to each interface again, # Start neighsyncd # The neighsyncd is supposed to sync up the entries from kernel after warm restart # Check the timer is not retrieved from configDB since it is not configured @@ -708,6 +708,9 @@ def test_OrchagentWarmRestartReadyCheck(dvs, testlog): (exitcode, result) = dvs.runcmd("/usr/bin/orchagent_restart_check -n -s -w 500") assert result == "RESTARTCHECK failed\n" + # Cleaning previously pushed route-entry to ease life of subsequent testcases. + del_entry_tbl(appl_db, swsscommon.APP_ROUTE_TABLE_NAME, "2.2.2.0/24") + # recover for test cases after this one. dvs.stop_swss() dvs.start_swss() @@ -784,3 +787,658 @@ def test_swss_port_state_syncup(dvs, testlog): assert oper_status == "down" else: assert oper_status == "up" + + +############################################################################# +# # +# Routing Warm-Restart Testing # +# # +############################################################################# + + +def set_restart_timer(dvs, db, app_name, value): + create_entry_tbl( + db, + swsscommon.CFG_WARM_RESTART_TABLE_NAME, app_name, + [ + (app_name + "_timer", value), + ] + ) + + +# Temporary instruction to activate warm_restart. To be deleted once equivalent CLI +# function is pushed to sonic-utils. +def enable_warmrestart(dvs, db, app_name): + create_entry_tbl( + db, + swsscommon.CFG_WARM_RESTART_TABLE_NAME, app_name, + [ + ("enable", "true"), + ] + ) + + +################################################################################ +# +# Routing warm-restart testcases +# +################################################################################ + +def test_routing_WarmRestart(dvs, testlog): + + appl_db = swsscommon.DBConnector(swsscommon.APPL_DB, dvs.redis_sock, 0) + conf_db = swsscommon.DBConnector(swsscommon.CONFIG_DB, dvs.redis_sock, 0) + state_db = swsscommon.DBConnector(swsscommon.STATE_DB, dvs.redis_sock, 0) + + # Restart-timer to utilize during the following testcases + restart_timer = 10 + + + ############################################################################# + # + # Baseline configuration + # + ############################################################################# + + + # Defining create neighbor entries (4 ipv4 and 4 ip6, two each on each interface) in linux kernel + intfs = ["Ethernet0", "Ethernet4", "Ethernet8"] + + # Enable ipv6 on docker + dvs.runcmd("sysctl net.ipv6.conf.all.disable_ipv6=0") + + dvs.runcmd("ip -4 addr add 111.0.0.1/24 dev {}".format(intfs[0])) + dvs.runcmd("ip -6 addr add 1110::1/64 dev {}".format(intfs[0])) + dvs.runcmd("ip link set {} up".format(intfs[0])) + + dvs.runcmd("ip -4 addr add 122.0.0.1/24 dev {}".format(intfs[1])) + dvs.runcmd("ip -6 addr add 1220::1/64 dev {}".format(intfs[1])) + dvs.runcmd("ip link set {} up".format(intfs[1])) + + dvs.runcmd("ip -4 addr add 133.0.0.1/24 dev {}".format(intfs[2])) + dvs.runcmd("ip -6 addr add 1330::1/64 dev {}".format(intfs[2])) + dvs.runcmd("ip link set {} up".format(intfs[2])) + + time.sleep(1) + + # + # Setting peer's ip-addresses and associated neighbor-entries + # + ips = ["111.0.0.2", "122.0.0.2", "133.0.0.2"] + v6ips = ["1110::2", "1220::2", "1330::2"] + macs = ["00:00:00:00:11:02", "00:00:00:00:12:02", "00:00:00:00:13:02"] + + for i in range(len(ips)): + dvs.runcmd("ip neigh add {} dev {} lladdr {}".format(ips[i], intfs[i%2], macs[i])) + + for i in range(len(v6ips)): + dvs.runcmd("ip -6 neigh add {} dev {} lladdr {}".format(v6ips[i], intfs[i%2], macs[i])) + + time.sleep(1) + + # + # Defining baseline IPv4 non-ecmp route-entries + # + dvs.runcmd("ip route add 192.168.1.100/32 nexthop via 111.0.0.2") + dvs.runcmd("ip route add 192.168.1.200/32 nexthop via 122.0.0.2") + dvs.runcmd("ip route add 192.168.1.300/32 nexthop via 133.0.0.2") + + # + # Defining baseline IPv4 ecmp route-entries + # + dvs.runcmd("ip route add 192.168.1.1/32 nexthop via 111.0.0.2 nexthop via 122.0.0.2 nexthop via 133.0.0.2") + dvs.runcmd("ip route add 192.168.1.2/32 nexthop via 111.0.0.2 nexthop via 122.0.0.2 nexthop via 133.0.0.2") + dvs.runcmd("ip route add 192.168.1.3/32 nexthop via 111.0.0.2 nexthop via 122.0.0.2") + + # + # Defining baseline IPv6 non-ecmp route-entries + # + dvs.runcmd("ip -6 route add fc00:11:11::1/128 nexthop via 1110::2") + dvs.runcmd("ip -6 route add fc00:12:12::1/128 nexthop via 1220::2") + dvs.runcmd("ip -6 route add fc00:13:13::1/128 nexthop via 1330::2") + + # + # Defining baseline IPv6 ecmp route-entries + # + dvs.runcmd("ip -6 route add fc00:1:1::1/128 nexthop via 1110::2 nexthop via 1220::2 nexthop via 1330::2") + dvs.runcmd("ip -6 route add fc00:2:2::1/128 nexthop via 1110::2 nexthop via 1220::2 nexthop via 1330::2") + dvs.runcmd("ip -6 route add fc00:3:3::1/128 nexthop via 1110::2 nexthop via 1220::2") + + time.sleep(5) + + # Enabling some extra logging for troubleshooting purposes + dvs.runcmd("swssloglevel -l INFO -c fpmsyncd") + + # Subscribe to pubsub channels for routing-state associated to swss and sairedis dbs + pubsubAppDB = dvs.SubscribeAppDbObject("ROUTE_TABLE") + pubsubAsicDB = dvs.SubscribeAsicDbObject("SAI_OBJECT_TYPE_ROUTE_ENTRY") + + + ############################################################################# + # + # Testcase 1. Having routing-warm-reboot disabled, restart zebra and verify + # that the traditional/cold-boot logic is followed. + # + ############################################################################# + + # Restart zebra + dvs.stop_zebra() + dvs.start_zebra() + + time.sleep(5) + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "") + + # Verify that multiple changes are seen in swss and sairedis logs as there's + # no warm-reboot logic in place. + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) != 0 + + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) != 0 + + + ############################################################################# + # + # Testcase 2. Restart zebra and make no control-plane changes. + # For this and all subsequent test-cases routing-warm-reboot + # feature will be kept enabled. + # + ############################################################################# + + + # Enabling bgp warmrestart and setting restart timer. + # The following two instructions will be substituted by the commented ones + # once the later ones are added to sonic-utilities repo. + enable_warmrestart(dvs, conf_db, "bgp") + set_restart_timer(dvs, conf_db, "bgp", str(restart_timer)) + #dvs.runcmd("config warm_restart enable bgp") + #dvs.runcmd("config warm_restart bgp_timer {}".format(restart_timer)) + + time.sleep(1) + + # Restart zebra + dvs.stop_zebra() + dvs.start_zebra() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify swss changes -- none are expected this time + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 0 and len(delobjs) == 0 + + # Verify swss changes -- none are expected this time + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 0 and len(delobjs) == 0 + + + ############################################################################# + # + # Testcase 3. Restart zebra and add one new non-ecmp IPv4 prefix + # + ############################################################################# + + # Stop zebra + dvs.stop_zebra() + + # Add new prefix + dvs.runcmd("ip route add 192.168.100.0/24 nexthop via 111.0.0.2") + time.sleep(1) + + # Start zebra + dvs.start_zebra() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + rt_val = json.loads(addobjs[0]['vals']) + assert rt_key == "192.168.100.0/24" + assert rt_val == {"ifname": "Ethernet0", "nexthop": "111.0.0.2"} + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + assert rt_key['dest'] == "192.168.100.0/24" + + + ############################################################################# + # + # Testcase 4. Restart zebra and withdraw one non-ecmp IPv4 prefix + # + ############################################################################# + + + # Stop zebra + dvs.stop_zebra() + + # Delete prefix + dvs.runcmd("ip route del 192.168.100.0/24 nexthop via 111.0.0.2") + time.sleep(1) + + # Start zebra + dvs.start_zebra() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 0 and len(delobjs) == 1 + rt_key = json.loads(delobjs[0]['key']) + assert rt_key == "192.168.100.0/24" + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 0 and len(delobjs) == 1 + rt_key = json.loads(delobjs[0]['key']) + assert rt_key['dest'] == "192.168.100.0/24" + + + ############################################################################# + # + # Testcase 5. Restart zebra and add a new IPv4 ecmp-prefix + # + ############################################################################# + + + # Stop zebra + dvs.stop_zebra() + + # Add prefix + dvs.runcmd("ip route add 192.168.200.0/24 nexthop via 111.0.0.2 nexthop via 122.0.0.2 nexthop via 133.0.0.2") + time.sleep(1) + + # Start zebra + dvs.start_zebra() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + rt_val = json.loads(addobjs[0]['vals']) + assert rt_key == "192.168.200.0/24" + assert rt_val == {"ifname": "Ethernet0,Ethernet4,Ethernet8", "nexthop": "111.0.0.2,122.0.0.2,133.0.0.2"} + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + assert rt_key['dest'] == "192.168.200.0/24" + + + ############################################################################# + # + # Testcase 6. Restart zebra and delete one existing IPv4 ecmp-prefix. + # + ############################################################################# + + + # Stop zebra + dvs.stop_zebra() + + # Delete prefix + dvs.runcmd("ip route del 192.168.200.0/24 nexthop via 111.0.0.2 nexthop via 122.0.0.2 nexthop via 133.0.0.2") + time.sleep(1) + + # Start zebra + dvs.start_zebra() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 0 and len(delobjs) == 1 + rt_key = json.loads(delobjs[0]['key']) + assert rt_key == "192.168.200.0/24" + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 0 and len(delobjs) == 1 + rt_key = json.loads(delobjs[0]['key']) + assert rt_key['dest'] == "192.168.200.0/24" + + + ############################################################################# + # + # Testcase 7. Restart zebra and add one new path to an IPv4 ecmp-prefix + # + ############################################################################# + + + # Stop zebra + dvs.stop_zebra() + + # Add new path + dvs.runcmd("ip route del 192.168.1.3/32 nexthop via 111.0.0.2 nexthop via 122.0.0.2") + dvs.runcmd("ip route add 192.168.1.3/32 nexthop via 111.0.0.2 nexthop via 122.0.0.2 nexthop via 133.0.0.2") + time.sleep(1) + + # Start zebra + dvs.start_zebra() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + rt_val = json.loads(addobjs[0]['vals']) + assert rt_key == "192.168.1.3" + assert rt_val == {"ifname": "Ethernet0,Ethernet4,Ethernet8", "nexthop": "111.0.0.2,122.0.0.2,133.0.0.2"} + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + assert rt_key['dest'] == "192.168.1.3/32" + + + ############################################################################# + # + # Testcase 8. Restart zebra and delete one ecmp-path from an IPv4 ecmp-prefix. + # + ############################################################################# + + + # Stop zebra + dvs.stop_zebra() + + # Delete ecmp-path + dvs.runcmd("ip route del 192.168.1.3/32 nexthop via 111.0.0.2 nexthop via 122.0.0.2 nexthop via 133.0.0.2") + dvs.runcmd("ip route add 192.168.1.3/32 nexthop via 111.0.0.2 nexthop via 122.0.0.2") + time.sleep(1) + + # Start zebra + dvs.start_zebra() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + rt_val = json.loads(addobjs[0]['vals']) + assert rt_key == "192.168.1.3" + assert rt_val == {"ifname": "Ethernet0,Ethernet4", "nexthop": "111.0.0.2,122.0.0.2"} + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + assert rt_key['dest'] == "192.168.1.3/32" + + + ############################################################################# + # + # Testcase 9. Restart zebra and add one new non-ecmp IPv6 prefix + # + ############################################################################# + + + # Stop zebra + dvs.stop_zebra() + + # Add prefix + dvs.runcmd("ip -6 route add fc00:4:4::1/128 nexthop via 1110::2") + time.sleep(1) + + # Start zebra + dvs.start_zebra() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + rt_val = json.loads(addobjs[0]['vals']) + assert rt_key == "fc00:4:4::1" + assert rt_val == {"ifname": "Ethernet0", "nexthop": "1110::2"} + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + assert rt_key['dest'] == "fc00:4:4::1/128" + + + ############################################################################# + # + # Testcase 10. Restart zebra and withdraw one non-ecmp IPv6 prefix + # + ############################################################################# + + # Stop zebra + dvs.stop_zebra() + + # Delete prefix + dvs.runcmd("ip -6 route del fc00:4:4::1/128 nexthop via 1110::2") + time.sleep(1) + + # Start zebra + dvs.start_zebra() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 0 and len(delobjs) == 1 + rt_key = json.loads(delobjs[0]['key']) + assert rt_key == "fc00:4:4::1" + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 0 and len(delobjs) == 1 + rt_key = json.loads(delobjs[0]['key']) + assert rt_key['dest'] == "fc00:4:4::1/128" + + + ############################################################################# + # + # Testcase 11. Restart fpmsyncd and make no control-plane changes. + # + ############################################################################# + + + # Stop fpmsyncd + dvs.stop_fpmsyncd() + + # Start fpmsyncd + dvs.start_fpmsyncd() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify swss changes -- none are expected this time + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 0 and len(delobjs) == 0 + + # Verify sairedis changes -- none are expected this time + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 0 and len(delobjs) == 0 + + + ############################################################################# + # + # Testcase 12. Restart fpmsyncd and add one new non-ecmp IPv4 prefix + # + ############################################################################# + + + # Stop fpmsyncd + dvs.stop_fpmsyncd() + + # Add new prefix + dvs.runcmd("ip route add 192.168.100.0/24 nexthop via 111.0.0.2") + time.sleep(1) + + # Start fpmsyncd + dvs.start_fpmsyncd() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + rt_val = json.loads(addobjs[0]['vals']) + assert rt_key == "192.168.100.0/24" + assert rt_val == {"ifname": "Ethernet0", "nexthop": "111.0.0.2"} + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + assert rt_key['dest'] == "192.168.100.0/24" + + + ############################################################################# + # + # Testcase 13. Restart fpmsyncd and withdraw one non-ecmp IPv4 prefix + # + ############################################################################# + + + # Stop fpmsyncd + dvs.stop_fpmsyncd() + + # Delete prefix + dvs.runcmd("ip route del 192.168.100.0/24 nexthop via 111.0.0.2") + time.sleep(1) + + # Start fpmsyncd + dvs.start_fpmsyncd() + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 0 and len(delobjs) == 1 + rt_key = json.loads(delobjs[0]['key']) + assert rt_key == "192.168.100.0/24" + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 0 and len(delobjs) == 1 + rt_key = json.loads(delobjs[0]['key']) + assert rt_key['dest'] == "192.168.100.0/24" + + + ############################################################################# + # + # Testcase 14. Restart zebra and add/remove a new non-ecmp IPv4 prefix. As + # the 'delete' instruction would arrive after the 'add' one, no + # changes should be pushed down to SwSS. + # + ############################################################################# + + + # Restart zebra + dvs.stop_zebra() + dvs.start_zebra() + + # Add/delete new prefix + dvs.runcmd("ip route add 192.168.100.0/24 nexthop via 111.0.0.2") + time.sleep(1) + dvs.runcmd("ip route del 192.168.100.0/24 nexthop via 111.0.0.2") + time.sleep(1) + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify swss changes -- none are expected this time + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 0 and len(delobjs) == 0 + + # Verify swss changes -- none are expected this time + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 0 and len(delobjs) == 0 + + + ############################################################################# + # + # Testcase 15. Restart zebra and generate an add/remove/add for new non-ecmp + # IPv4 prefix. Verify that only the second 'add' instruction is + # honored and the corresponding update passed down to SwSS. + # + ############################################################################# + + + # Restart zebra + dvs.stop_zebra() + dvs.start_zebra() + + marker1 = dvs.add_log_marker("/var/log/swss/swss.rec") + marker2 = dvs.add_log_marker("/var/log/swss/sairedis.rec") + + # Add/delete new prefix + dvs.runcmd("ip route add 192.168.100.0/24 nexthop via 111.0.0.2") + time.sleep(1) + dvs.runcmd("ip route del 192.168.100.0/24 nexthop via 111.0.0.2") + time.sleep(1) + dvs.runcmd("ip route add 192.168.100.0/24 nexthop via 122.0.0.2") + time.sleep(1) + + # Verify FSM + swss_app_check_warmstart_state(state_db, "bgp", "restored") + time.sleep(restart_timer + 1) + swss_app_check_warmstart_state(state_db, "bgp", "reconciled") + + # Verify the changed prefix is seen in swss + (addobjs, delobjs) = dvs.GetSubscribedAppDbObjects(pubsubAppDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + rt_val = json.loads(addobjs[0]['vals']) + assert rt_key == "192.168.100.0/24" + assert rt_val == {"ifname": "Ethernet4", "nexthop": "122.0.0.2"} + + # Verify the changed prefix is seen in sairedis + (addobjs, delobjs) = dvs.GetSubscribedAsicDbObjects(pubsubAsicDB) + assert len(addobjs) == 1 and len(delobjs) == 0 + rt_key = json.loads(addobjs[0]['key']) + assert rt_key['dest'] == "192.168.100.0/24" + + diff --git a/warmrestart/warmRestartHelper.cpp b/warmrestart/warmRestartHelper.cpp new file mode 100644 index 000000000000..580e9f98a62b --- /dev/null +++ b/warmrestart/warmRestartHelper.cpp @@ -0,0 +1,378 @@ +#include +#include + +#include "warmRestartHelper.h" + + +using namespace swss; + + +WarmStartHelper::WarmStartHelper(RedisPipeline *pipeline, + ProducerStateTable *syncTable, + const std::string &syncTableName, + const std::string &dockerName, + const std::string &appName) : + m_restorationTable(pipeline, syncTableName, false), + m_syncTable(syncTable), + m_syncTableName(syncTableName), + m_dockName(dockerName), + m_appName(appName) +{ + WarmStart::initialize(appName, dockerName); +} + + +WarmStartHelper::~WarmStartHelper() +{ +} + + +void WarmStartHelper::setState(WarmStart::WarmStartState state) +{ + WarmStart::setWarmStartState(m_appName, state); + + /* Caching warm-restart FSM state in local member */ + m_state = state; +} + + +WarmStart::WarmStartState WarmStartHelper::getState(void) const +{ + return m_state; +} + + +/* + * To be called by each application to obtain the active/inactive state of + * warm-restart functionality, and proceed to initialize the FSM accordingly. + */ +bool WarmStartHelper::checkAndStart(void) +{ + bool enabled = WarmStart::checkWarmStart(m_appName, m_dockName); + + /* + * If warm-restart feature is enabled for this application, proceed to + * initialize its FSM, and clean any pending state that could be potentially + * held in ProducerState queues. + */ + if (enabled) + { + SWSS_LOG_NOTICE("Initializing Warm-Restart cycle for %s application.", + m_appName.c_str()); + + setState(WarmStart::INITIALIZED); + m_syncTable->clear(); + } + + /* Cleaning state from previous (unsuccessful) warm-restart attempts */ + m_restorationVector.clear(); + m_refreshMap.clear(); + + /* Keeping track of warm-reboot active/inactive state */ + m_enabled = enabled; + + return enabled; +} + + +bool WarmStartHelper::isReconciled(void) const +{ + return (m_state == WarmStart::RECONCILED); +} + + +bool WarmStartHelper::inProgress(void) const +{ + return (m_enabled && m_state != WarmStart::RECONCILED); +} + + +uint32_t WarmStartHelper::getRestartTimer(void) const +{ + return WarmStart::getWarmStartTimer(m_appName, m_dockName); +} + + +/* + * Invoked by warmStartHelper clients during initialization. All interested parties + * are expected to call this method to upload their associated redisDB state into + * a temporary buffer, which will eventually serve to resolve any conflict between + * 'old' and 'new' state. + */ +bool WarmStartHelper::runRestoration() +{ + SWSS_LOG_NOTICE("Warm-Restart: Initiating AppDB restoration process for %s " + "application.", m_appName.c_str()); + + m_restorationTable.getContent(m_restorationVector); + + /* + * If there's no AppDB state to restore, then alert callee right away to avoid + * iterating through the 'reconciliation' process. + */ + if (!m_restorationVector.size()) + { + SWSS_LOG_NOTICE("Warm-Restart: No records received from AppDB for %s " + "application.", m_appName.c_str()); + + setState(WarmStart::RECONCILED); + + return false; + } + + SWSS_LOG_NOTICE("Warm-Restart: Received %zu records from AppDB for %s " + "application.", + m_restorationVector.size(), + m_appName.c_str()); + + setState(WarmStart::RESTORED); + + SWSS_LOG_NOTICE("Warm-Restart: Completed AppDB restoration process for %s " + "application.", m_appName.c_str()); + + return true; +} + + +void WarmStartHelper::insertRefreshMap(const KeyOpFieldsValuesTuple &kfv) +{ + const std::string key = kfvKey(kfv); + + m_refreshMap[key] = kfv; +} + + +/* + * The reconciliation process takes place here. In essence, all we are doing + * is comparing the restored elements (old state) with the refreshed/new ones + * generated by the application once it completes its restart cycle. If a + * state-diff is found between these two, we will be honoring the refreshed + * one received from the application, and will proceed to push it down to AppDB. + */ +void WarmStartHelper::reconcile(void) +{ + SWSS_LOG_NOTICE("Warm-Restart: Initiating reconciliation process for %s " + "application.", m_appName.c_str()); + + assert(getState() == WarmStart::RESTORED); + + for (auto &restoredElem : m_restorationVector) + { + std::string restoredKey = kfvKey(restoredElem); + auto restoredFV = kfvFieldsValues(restoredElem); + + auto iter = m_refreshMap.find(restoredKey); + + /* + * If the restored element is not found in the refreshMap, we must + * push a delete operation for this entry. + */ + if (iter == m_refreshMap.end()) + { + SWSS_LOG_NOTICE("Warm-Restart reconciliation: deleting stale entry %s", + printKFV(restoredKey, restoredFV).c_str()); + + m_syncTable->del(restoredKey); + continue; + } + + /* + * If an explicit delete request is sent by the application, process it + * right away. + */ + else if (kfvOp(iter->second) == DEL_COMMAND) + { + SWSS_LOG_NOTICE("Warm-Restart reconciliation: deleting entry %s", + printKFV(restoredKey, restoredFV).c_str()); + + m_syncTable->del(restoredKey); + } + + /* + * If a matching entry is found in refreshMap, proceed to compare it + * with its restored counterpart. + */ + else + { + auto refreshedKey = kfvKey(iter->second); + auto refreshedFV = kfvFieldsValues(iter->second); + + if (compareAllFV(restoredFV, refreshedFV)) + { + SWSS_LOG_NOTICE("Warm-Restart reconciliation: updating entry %s", + printKFV(refreshedKey, refreshedFV).c_str()); + + m_syncTable->set(refreshedKey, refreshedFV); + } + else + { + SWSS_LOG_INFO("Warm-Restart reconciliation: no changes needed for " + "existing entry %s", + printKFV(refreshedKey, refreshedFV).c_str()); + } + } + + /* Deleting the just-processed restored entry from the refreshMap */ + m_refreshMap.erase(restoredKey); + } + + /* + * Iterate through all the entries left in the refreshMap, which correspond + * to brand-new entries to be pushed down to AppDB. + */ + for (auto &kfv : m_refreshMap) + { + auto refreshedKey = kfvKey(kfv.second); + auto refreshedOp = kfvOp(kfv.second); + auto refreshedFV = kfvFieldsValues(kfv.second); + + /* + * During warm-reboot, apps could receive an 'add' and a 'delete' for an + * entry that does not exist in AppDB. In these cases we must prevent the + * 'delete' from being pushed down to AppDB, so we are handling this case + * differently than the 'add' one. + */ + if(refreshedOp == DEL_COMMAND) + { + SWSS_LOG_NOTICE("Warm-Restart reconciliation: discarding non-existing" + " entry %s\n", + refreshedKey.c_str()); + } + else + { + SWSS_LOG_NOTICE("Warm-Restart reconciliation: introducing new entry %s", + printKFV(refreshedKey, refreshedFV).c_str()); + + m_syncTable->set(refreshedKey, refreshedFV); + } + } + + /* Clearing pending kfv's from refreshMap */ + m_refreshMap.clear(); + + /* Clearing restoration vector */ + m_restorationVector.clear(); + + setState(WarmStart::RECONCILED); + + SWSS_LOG_NOTICE("Warm-Restart: Concluded reconciliation process for %s " + "application.", m_appName.c_str()); +} + + +/* + * Compare all field-value-tuples within two vectors. + * + * Example: v1 {nexthop: 10.1.1.1, ifname: eth1} + * v2 {nexthop: 10.1.1.2, ifname: eth2} + * + * Returns: + * + * 'false' : If the content of both 'fields' and 'values' fully match + * 'true' : No full-match is found + */ +bool WarmStartHelper::compareAllFV(const std::vector &v1, + const std::vector &v2) +{ + std::unordered_map v1Map((v1.begin()), v1.end()); + + /* Iterate though all v2 tuples to check if their content match v1 ones */ + for (auto &v2fv : v2) + { + auto v1Iter = v1Map.find(v2fv.first); + /* + * The sizes of both tuple-vectors should always match within any + * given application. In other words, all fields within v1 should be + * also present in v2. + * + * To make this possible, every application should continue relying on a + * uniform schema to create/generate information. For example, fpmsyncd + * will be always expected to push FieldValueTuples with "nexthop" and + * "ifname" fields; neighsyncd is expected to make use of "family" and + * "neigh" fields, etc. The existing reconciliation logic will rely on + * this assumption. + */ + assert(v1Iter != v1Map.end()); + + if (compareOneFV(v1Map[fvField(*v1Iter)], fvValue(v2fv))) + { + return true; + } + } + + return false; +} + + +/* + * Compare the values of a single field-value within two different KFVs. + * + * Example: s1 {nexthop: 10.1.1.1, 10.1.1.2} + * s2 {nexthop: 10.1.1.2, 10.1.1.1} + * + * Example: s1 {Ethernet1, Ethernet2} + * s2 {Ethernet2, Ethernet1} + * + * Returns: + * + * 'false' : If the content of both strings fully matches + * 'true' : No full-match is found + */ +bool WarmStartHelper::compareOneFV(const std::string &s1, const std::string &s2) +{ + if (s1.size() != s2.size()) + { + return true; + } + + std::vector splitValuesS1 = tokenize(s1, ','); + std::vector splitValuesS2 = tokenize(s2, ','); + + if (splitValuesS1.size() != splitValuesS2.size()) + { + return true; + } + + std::sort(splitValuesS1.begin(), splitValuesS1.end()); + std::sort(splitValuesS2.begin(), splitValuesS2.end()); + + for (size_t i = 0; i < splitValuesS1.size(); i++) + { + if (splitValuesS1[i] != splitValuesS2[i]) + { + return true; + } + } + + return false; +} + + +/* + * Helper method to print KFVs in a friendly fashion. + * + * Example: + * + * 192.168.1.0/30 { nexthop: 10.2.2.1,10.1.2.1 | ifname: Ethernet116,Ethernet112 } + */ +const std::string WarmStartHelper::printKFV(const std::string &key, + const std::vector &fv) +{ + std::string res; + + res = key + " { "; + + for (size_t i = 0; i < fv.size(); ++i) + { + res += fv[i].first + ": " + fv[i].second; + + if (i != fv.size() - 1) + { + res += " | "; + } + } + + res += " } "; + + return res; +} diff --git a/warmrestart/warmRestartHelper.h b/warmrestart/warmRestartHelper.h new file mode 100644 index 000000000000..75af3c4b9dc0 --- /dev/null +++ b/warmrestart/warmRestartHelper.h @@ -0,0 +1,83 @@ +#ifndef __WARMRESTART_HELPER__ +#define __WARMRESTART_HELPER__ + + +#include +#include +#include +#include + +#include "dbconnector.h" +#include "producerstatetable.h" +#include "netmsg.h" +#include "table.h" +#include "tokenize.h" +#include "warm_restart.h" + + +namespace swss { + + +class WarmStartHelper { + public: + + WarmStartHelper(RedisPipeline *pipeline, + ProducerStateTable *syncTable, + const std::string &syncTableName, + const std::string &dockerName, + const std::string &appName); + + ~WarmStartHelper(); + + /* fvVector type to be used to host AppDB restored elements */ + using kfvVector = std::vector; + + /* + * kfvMap type to be utilized to store all the new/refresh state coming + * from the restarting applications. + */ + using kfvMap = std::unordered_map; + + void setState(WarmStart::WarmStartState state); + + WarmStart::WarmStartState getState(void) const; + + bool checkAndStart(void); + + bool isReconciled(void) const; + + bool inProgress(void) const; + + uint32_t getRestartTimer(void) const; + + bool runRestoration(void); + + void insertRefreshMap(const KeyOpFieldsValuesTuple &kfv); + + void reconcile(void); + + const std::string printKFV(const std::string &key, + const std::vector &fv); + + private: + + bool compareAllFV(const std::vector &left, + const std::vector &right); + + bool compareOneFV(const std::string &v1, const std::string &v2); + + ProducerStateTable *m_syncTable; // producer-table to sync/push state to + Table m_restorationTable; // redis table to import current-state from + kfvVector m_restorationVector; // buffer struct to hold old state + kfvMap m_refreshMap; // buffer struct to hold new state + WarmStart::WarmStartState m_state; // cached value of warmStart's FSM state + bool m_enabled; // warm-reboot enabled/disabled status + std::string m_syncTableName; // producer-table-name to sync/push state to + std::string m_dockName; // sonic-docker requesting warmStart services + std::string m_appName; // sonic-app requesting warmStart services +}; + + +} + +#endif