Skip to content

Commit

Permalink
* [saidump]
Browse files Browse the repository at this point in the history
•	Saidump for DNX-SAI sonic-net/sonic-buildimage#13561

Solution and modification:
To use the redis-db SAVE option to save the snapshot of DB each time and recover later, instead of looping through each entry in the table and saving it.

(1) Updated sonic-buildimage/build_debian.sh, to install Python library rdbtools into the host.
(2) Updated sonic-buildimage/src/sonic-sairedis/saidump/saidump.cpp, add a new option -r, which updates the rdbtools's output-JSON files' format.
(3) Add a new script file: files/scripts/saidump.sh, to do the below steps
  For each ASIC0, such as ASIC0,

  #1. Save the Redis data.
  sudo sonic-db-cli -n asic$1 SAVE > /dev/null

  #2. Move dump files to /var/run/redisX/
  docker exec database$1 sh -c "mv /var/lib/redis/dump.rdb /var/run/redis$1/"

  sonic-net#3. Run rdb command to convert the dump files into JSON files
  sudo python /usr/local/bin/rdb --command json  /var/run/redis$1/dump.rdb | sudo tee /var/run/redis$1/dump.json > /dev/null

  sonic-net#4. Run saidump -r to update the JSON files' format as same as the saidump before. Then we can get the saidump result in standard output.
  docker exec syncd$1 sh -c "saidump -r /var/run/redis$1/dump.json"

  sonic-net#5. clear
  sudo rm -f /var/run/redis$1/dump.rdb
  sudo rm -f /var/run/redis$1/dump.json

(4) Update sonic-buildimage/src/sonic-utilities/scripts/generate_dump, replace saidump with saidump.sh
  • Loading branch information
JunhongMao committed Sep 1, 2023
1 parent 4c2527f commit 5ee5c12
Showing 1 changed file with 184 additions and 3 deletions.
187 changes: 184 additions & 3 deletions saidump/saidump.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
#include <string>
#include <set>
#include <sstream>
#include <iostream>
#include <fstream>
#include <regex>

extern "C" {
#include <sai.h>
Expand All @@ -10,18 +13,26 @@ extern "C" {
#include "swss/table.h"
#include "meta/sai_serialize.h"
#include "sairediscommon.h"
#include "swss/json.h"
#include "swss/json.hpp"

#include <getopt.h>

// TODO split to multiple cpp

using namespace swss;
using namespace std;
using json = nlohmann::json;

// 100 MB
const int64_t RDB_FILE_MAX_SIZE = 1024 * 1024 * 100;

struct CmdOptions
{
bool skipAttributes;
bool dumpTempView;
bool dumpGraph;
string rdb_file;
};

CmdOptions g_cmdOptions;
Expand All @@ -31,11 +42,13 @@ void printUsage()
{
SWSS_LOG_ENTER();

std::cout << "Usage: saidump [-t] [-g] [-h]" << std::endl;
std::cout << "Usage: saidump [-t] [-g] [-r] [-h]" << std::endl;
std::cout << " -t --tempView:" << std::endl;
std::cout << " Dump temp view" << std::endl;
std::cout << " -g --dumpGraph:" << std::endl;
std::cout << " Dump current graph" << std::endl;
std::cout << " -r --rdb:" << std::endl;
std::cout << " Dump by parsing the Redis persistence file dump.rdb that is generated by SAVE command." << std::endl;
std::cout << " -h --help:" << std::endl;
std::cout << " Print out this message" << std::endl;
}
Expand All @@ -49,14 +62,15 @@ CmdOptions handleCmdLine(int argc, char **argv)
options.dumpTempView = false;
options.dumpGraph = false;

const char* const optstring = "gth";
const char* const optstring = "gtr:h";

while (true)
{
static struct option long_options[] =
{
{ "dumpGraph", no_argument, 0, 'g' },
{ "tempView", no_argument, 0, 't' },
{ "rdb", required_argument, 0, 'r' },
{ "help", no_argument, 0, 'h' },
{ 0, 0, 0, 0 }
};
Expand All @@ -82,6 +96,11 @@ CmdOptions handleCmdLine(int argc, char **argv)
options.dumpTempView = true;
break;

case 'r':
SWSS_LOG_WARN("Dumping from dump.rdb");
options.rdb_file = string(optarg);
break;

case 'h':
printUsage();
exit(EXIT_SUCCESS);
Expand Down Expand Up @@ -399,7 +418,159 @@ void dumpGraph(const TableDump& td)
std::cout << "}" << std::endl;
}

int main(int argc, char ** argv)
/*
preprocess the input json file to make make sure it's a valid json file.
*/
int pre_process_file(string file_name)
{
ifstream input_file(file_name);

if (!input_file.is_open())
{
cerr << "Failed to open the input file." << endl;
return SAI_STATUS_FAILURE;
}

input_file.seekg(0, ios::end); // Move to the end of the file
int64_t file_size = input_file.tellg(); // Get the current position

if (file_size >= RDB_FILE_MAX_SIZE)
{
SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024);

// Close the input file
input_file.close();
return SAI_STATUS_FAILURE;
}
input_file.seekg(0, ios::beg); // Move to the begin of the file

// Read the content of the input file into a string
string content((istreambuf_iterator<char>(input_file)),
istreambuf_iterator<char>());


input_file.close();

content = regex_replace(content, regex("\\},\\{\\r"), ",");

//erase the 1st and last char.
if (content.length() >= 2 && content[0] == '[' && content[content.length()-1] == ']') {
content.erase(0, 1);
content.erase(content.length() - 1);
}

ofstream outputFile(file_name);

if (!outputFile.is_open())
{
cerr << "Failed to open the output file." << endl;
return SAI_STATUS_FAILURE;
}

outputFile << content;
outputFile.close();

return SAI_STATUS_SUCCESS;
}

int dump_from_redis_rdb_file(string file_name)
{
ifstream input_file(file_name);
if (!input_file.is_open())
{
SWSS_LOG_NOTICE("The file %s is not exsisted for dumping from redis dump.rdb file", file_name);
return SAI_STATUS_FAILURE;
}

input_file.seekg(0, ios::end); // Move to the end of the file
int64_t file_size = input_file.tellg(); // Get the current position
if (file_size >= RDB_FILE_MAX_SIZE)
{
SWSS_LOG_ERROR("Get %s's size failure or its size %ld >= %ld MB.\n", file_name.c_str(), file_size, RDB_FILE_MAX_SIZE / 1024 / 1024);

// Close the input file
input_file.close();
return SAI_STATUS_FAILURE;
}
input_file.seekg(0, ios::beg); // Move to the begin of the file

try
{
// Parse the JSON data from the file (validation)
nlohmann::json jsonData;
input_file >> jsonData;
input_file.close();

SWSS_LOG_DEBUG("%s\n","JSON file is valid.\n");

for (json::iterator it = jsonData.begin(); it != jsonData.end(); ++it)
{
json jj_key = it.key();

string keystr = jj_key;
string item_name = keystr;
size_t pos = keystr.find_first_of(":");
if (pos != string::npos)
{
if("ASIC_STATE" != keystr.substr(0, pos)) // filter out non ASIC_STATE
{
continue;
}
item_name = keystr.substr(pos + 1);
if (item_name.find(":") != string::npos)
{
item_name.replace(item_name.find_first_of(":"), 1, " ");
}
}
else
{
continue;
}

cout << item_name << " " << endl;

json jj = it.value();

if (!(*it).is_object())
{
continue;
}

TableMap map;
for (json::iterator itt = jj.begin(); itt != jj.end(); ++itt)
{
if (itt.key() == "NULL")
{
continue;
}
map[itt.key()] = itt.value();
}

size_t indent = 4;
size_t max_len = get_max_attr_len(map);
string str_indent = pad_string("", indent);

for (const auto&field: map)
{
stringstream ss;
ss << str_indent << pad_string(field.first, max_len) << " : ";
ss << field.second;
cout << ss.str() << std::endl;
}
cout << endl;
}
return SAI_STATUS_SUCCESS;
}
catch (exception &ex)
{
input_file.close();
cerr << "JSON file:" << file_name << " is invalid." << endl;
cerr << "JSON parsing error: " << ex.what() << endl;
}
return SAI_STATUS_FAILURE;
}

int main(int argc, char **argv)
{
swss::Logger::getInstance().setMinPrio(swss::Logger::SWSS_DEBUG);

Expand All @@ -411,6 +582,16 @@ int main(int argc, char ** argv)

g_cmdOptions = handleCmdLine(argc, argv);

if (g_cmdOptions.rdb_file.size() > 0)
{
if (SAI_STATUS_FAILURE == pre_process_file(g_cmdOptions.rdb_file))
{
return EXIT_FAILURE;
}
dump_from_redis_rdb_file(g_cmdOptions.rdb_file);
return EXIT_SUCCESS;
}

swss::DBConnector db("ASIC_DB", 0);

std::string table = ASIC_STATE_TABLE;
Expand Down

0 comments on commit 5ee5c12

Please sign in to comment.