Skip to content

Commit

Permalink
Calculate chunkserver left space of each physical pool
Browse files Browse the repository at this point in the history
  • Loading branch information
@SeanHai
SeanHai committed Jan 6, 2021
1 parent 7edddfc commit 6870e5d
Show file tree
Hide file tree
Showing 8 changed files with 162 additions and 92 deletions.
30 changes: 29 additions & 1 deletion src/tools/mds_client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -771,7 +771,7 @@ int MDSClient::GetCopySetsInCluster(std::vector<CopysetInfo>* copysets) {

int MDSClient::ListServersInCluster(std::vector<ServerInfo>* servers) {
assert(servers != nullptr);
// 先列出逻辑池
// list physicalpools
std::vector<PhysicalPoolInfo> phyPools;
if (ListPhysicalPoolsInCluster(&phyPools) != 0) {
std::cout << "ListPhysicalPoolsInCluster fail!" << std::endl;
Expand Down Expand Up @@ -812,6 +812,34 @@ int MDSClient::ListChunkServersInCluster(
return 0;
}

int MDSClient::ListChunkServersInCluster(std::map<PoolIdType,
std::vector<ChunkServerInfo>>* chunkservers) {
assert(chunkservers != nullptr);
std::vector<ServerInfo> servers;
if (ListServersInCluster(&servers) != 0) {
std::cout << "ListServersInCluster fail!" << std::endl;
return -1;
}

for (const auto& server : servers) {
std::vector<ChunkServerInfo> chunkserverList;
if (ListChunkServersOnServer(server.serverid(),
&chunkserverList) != 0) {
std::cout << "ListChunkServersOnServer fail!" << std::endl;
return -1;
}

auto iter = chunkservers->find(server.physicalpoolid());
if (iter != chunkservers->end()) {
iter->second.insert(iter->second.end(), chunkserverList.begin(),
chunkserverList.end());
} else {
chunkservers->emplace(server.physicalpoolid(), chunkserverList);
}
}
return 0;
}

int MDSClient::GetListenAddrFromDummyPort(const std::string& dummyAddr,
std::string* listenAddr) {
assert(listenAddr != nullptr);
Expand Down
8 changes: 8 additions & 0 deletions src/tools/mds_client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,14 @@ class MDSClient {
virtual int ListChunkServersInCluster(
std::vector<ChunkServerInfo>* chunkservers);

/**
* @brief list all the chunkservers with poolid in cluster
* @param[out] chunkservers chunkserver info
* @return succeed return 0; failed return -1;
*/
virtual int ListChunkServersInCluster(std::map<PoolIdType,
std::vector<ChunkServerInfo>>* chunkservers);

/**
* @brief 获取mds的某个metric的值
* @param metricName metric的名字
Expand Down
152 changes: 81 additions & 71 deletions src/tools/status_tool.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,56 +683,64 @@ int StatusTool::PrintChunkserverStatus(bool checkLeftSize) {
ret = -1;
}
}
std::vector<ChunkServerInfo> chunkservers;
res = mdsClient_->ListChunkServersInCluster(&chunkservers);
std::map<PoolIdType, std::vector<ChunkServerInfo>> poolChunkservers;
res = mdsClient_->ListChunkServersInCluster(&poolChunkservers);
if (res != 0) {
std::cout << "ListChunkServersInCluster fail!" << std::endl;
return -1;
}
uint64_t total = 0;
uint64_t online = 0;
uint64_t offline = 0;
std::vector<uint64_t> chunkLeftSize;
std::vector<uint64_t> walSegmentLeftSize;
std::map<PoolIdType, std::vector<uint64_t>> poolChunkLeftSize;
std::map<PoolIdType, std::vector<uint64_t>> poolWalSegmentLeftSize;
std::vector<ChunkServerIdType> offlineCs;
// 获取chunkserver的online状态
for (const auto& chunkserver : chunkservers) {
total++;
std::string csAddr = chunkserver.hostip()
+ ":" + std::to_string(chunkserver.port());
if (copysetCheckCore_->CheckChunkServerOnline(csAddr)) {
online++;
} else {
offline++;
offlineCs.emplace_back(chunkserver.chunkserverid());
}
if (!checkLeftSize) {
continue;
}
std::string metricName = GetCSLeftChunkName(csAddr);
uint64_t chunkNum;
MetricRet res = metricClient_->GetMetricUint(csAddr,
metricName, &chunkNum);
if (res != MetricRet::kOK) {
std::cout << "Get left chunk size of chunkserver " << csAddr
<< " fail!" << std::endl;
ret = -1;
continue;
}
uint64_t size = chunkNum * FLAGS_chunkSize;
chunkLeftSize.emplace_back(size / mds::kGB);
// walfilepool left size
metricName = GetCSLeftWalSegmentName(csAddr);
uint64_t walSegmentNum;
res = metricClient_->GetMetricUint(csAddr, metricName, &walSegmentNum);
if (res != MetricRet::kOK) {
std::cout << "Get left wal segment size of chunkserver " << csAddr
<< " fail!" << std::endl;
ret = -1;
continue;
for (const auto& poolChunkserver : poolChunkservers) {
std::vector<uint64_t> chunkLeftSize;
std::vector<uint64_t> walSegmentLeftSize;
for (const auto& chunkserver : poolChunkserver.second) {
total++;
std::string csAddr = chunkserver.hostip()
+ ":" + std::to_string(chunkserver.port());
if (copysetCheckCore_->CheckChunkServerOnline(csAddr)) {
online++;
} else {
offline++;
offlineCs.emplace_back(chunkserver.chunkserverid());
}
if (!checkLeftSize) {
continue;
}
std::string metricName = GetCSLeftChunkName(csAddr);
uint64_t chunkNum;
MetricRet res = metricClient_->GetMetricUint(csAddr,
metricName, &chunkNum);
if (res != MetricRet::kOK) {
std::cout << "Get left chunk size of chunkserver " << csAddr
<< " fail!" << std::endl;
ret = -1;
continue;
}
uint64_t size = chunkNum * FLAGS_chunkSize;
chunkLeftSize.emplace_back(size / mds::kGB);
// walfilepool left size
metricName = GetCSLeftWalSegmentName(csAddr);
uint64_t walSegmentNum;
res = metricClient_->GetMetricUint(csAddr, metricName,
&walSegmentNum);
if (res != MetricRet::kOK) {
std::cout << "Get left wal segment size of chunkserver "
<< csAddr << " fail!" << std::endl;
ret = -1;
continue;
}
size = walSegmentNum * FLAGS_walSegmentSize;
walSegmentLeftSize.emplace_back(size / mds::kGB);
}
size = walSegmentNum * FLAGS_walSegmentSize;
walSegmentLeftSize.emplace_back(size / mds::kGB);
poolChunkLeftSize.emplace(poolChunkserver.first, chunkLeftSize);
poolWalSegmentLeftSize.emplace(poolChunkserver.first,
walSegmentLeftSize);
}
// 获取offline chunkserver的恢复状态
std::vector<ChunkServerIdType> offlineRecover;
Expand Down Expand Up @@ -773,46 +781,48 @@ int StatusTool::PrintChunkserverStatus(bool checkLeftSize) {
return ret;
}

PrintCsLeftSizeStatistics("chunkfilepool", chunkLeftSize);
PrintCsLeftSizeStatistics("walfilepool", walSegmentLeftSize);
PrintCsLeftSizeStatistics("chunkfilepool", poolChunkLeftSize);
PrintCsLeftSizeStatistics("walfilepool", poolWalSegmentLeftSize);
return ret;
}

void StatusTool::PrintCsLeftSizeStatistics(
const std::string& name,
const std::vector<uint64_t>& leftSize) {
if (leftSize.empty()) {
void StatusTool::PrintCsLeftSizeStatistics(const std::string& name,
const std::map<PoolIdType,
std::vector<uint64_t>>& poolLeftSize) {
if (poolLeftSize.empty()) {
std::cout << "No " << name << " left size found!" << std::endl;
return;
}
uint64_t min = leftSize[0];
uint64_t max = leftSize[0];
double sum = 0;
for (const auto& size : leftSize) {
sum += size;
if (size < min) {
min = size;
for (const auto& leftSize : poolLeftSize) {
uint64_t min = leftSize.second[0];
uint64_t max = leftSize.second[0];
double sum = 0;
for (const auto& size : leftSize.second) {
sum += size;
if (size < min) {
min = size;
}
if (size > max) {
max = size;
}
}
if (size > max) {
max = size;
uint64_t range = max - min;
double avg = sum / leftSize.second.size();
sum = 0;
for (const auto& size : leftSize.second) {
sum += (size - avg) * (size - avg);
}
}
uint64_t range = max - min;
double avg = sum / leftSize.size();
sum = 0;
for (const auto& size : leftSize) {
sum += (size - avg) * (size - avg);
}

double var = sum / leftSize.size();
std:: cout.setf(std::ios::fixed);
std::cout<< std::setprecision(2);
std::cout<< name;
std::cout << " left size: min = " << min << "GB"
<< ", max = " << max << "GB"
<< ", average = " << avg << "GB"
<< ", range = " << range << "GB"
<< ", variance = " << var << std::endl;
double var = sum / leftSize.second.size();
std:: cout.setf(std::ios::fixed);
std::cout<< std::setprecision(2);
std::cout<< "pool" << leftSize.first << " " << name;
std::cout << " left size: min = " << min << "GB"
<< ", max = " << max << "GB"
<< ", average = " << avg << "GB"
<< ", range = " << range << "GB"
<< ", variance = " << var << std::endl;
}
}

int StatusTool::GetPoolsInCluster(std::vector<PhysicalPoolInfo>* phyPools,
Expand Down
3 changes: 2 additions & 1 deletion src/tools/status_tool.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,8 @@ class StatusTool : public CurveTool {
int PrintClientStatus();
int ClientListCmd();
void PrintCsLeftSizeStatistics(const std::string& name,
const std::vector<uint64_t>& leftSize);
const std::map<PoolIdType,
std::vector<uint64_t>>& poolLeftSize);
int PrintSnapshotCloneStatus();

/**
Expand Down
4 changes: 3 additions & 1 deletion test/tools/curve_cli_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ using ::testing::Return;
using ::testing::Invoke;
using ::testing::DoAll;
using ::testing::SetArgPointee;
using ::testing::An;

DECLARE_int32(timeout_ms);
DECLARE_int32(max_retry);
Expand Down Expand Up @@ -287,7 +288,8 @@ TEST_F(CurveCliTest, DoSnapshotAll) {
EXPECT_CALL(*mdsClient_, Init(_))
.Times(2)
.WillRepeatedly(Return(0));
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::vector<ChunkServerInfo>*>()))
.Times(2)
.WillRepeatedly(DoAll(SetArgPointee<0>(chunkservers),
Return(0)));
Expand Down
4 changes: 3 additions & 1 deletion test/tools/mock/mock_mds_client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,9 @@ class MockMDSClient : public MDSClient {
MOCK_METHOD1(GetCopySetsInCluster, int(std::vector<CopysetInfo>*));
MOCK_METHOD1(ListServersInCluster, int(std::vector<ServerInfo>*));
MOCK_METHOD1(ListChunkServersInCluster,
int(std::vector<ChunkServerInfo>*));
int(std::vector<ChunkServerInfo>*));
MOCK_METHOD1(ListChunkServersInCluster, int(std::map<PoolIdType,
std::vector<ChunkServerInfo>>*));
MOCK_METHOD2(GetMetric, int(const std::string&, uint64_t*));
MOCK_CONST_METHOD0(GetMdsAddrVec, const std::vector<std::string>&());
MOCK_METHOD0(GetCurrentMds, std::vector<std::string>());
Expand Down
34 changes: 23 additions & 11 deletions test/tools/status_tool_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
* Author: charisu
*/
#include <gtest/gtest.h>
#include <map>
#include "src/tools/status_tool.h"
#include "test/tools/mock/mock_namespace_tool_core.h"
#include "test/tools/mock/mock_copyset_check_core.h"
Expand Down Expand Up @@ -333,7 +334,8 @@ TEST_F(StatusToolTest, ChunkServerCmd) {
.WillOnce(Return(0));

// 正常情况，有一个chunkserver的UnhealthyRatio大于0
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::vector<ChunkServerInfo>*>()))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(chunkservers),
Return(0)));
Expand All @@ -349,14 +351,16 @@ TEST_F(StatusToolTest, ChunkServerCmd) {

// 只显示offline的
FLAGS_offline = true;
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::vector<ChunkServerInfo>*>()))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(chunkservers),
Return(0)));
ASSERT_EQ(0, statusTool.RunCommand("chunkserver-list"));

// 只显示unhealthy ratio大于0的
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::vector<ChunkServerInfo>*>()))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(chunkservers),
Return(0)));
Expand All @@ -373,15 +377,17 @@ TEST_F(StatusToolTest, ChunkServerCmd) {
ASSERT_EQ(0, statusTool.RunCommand("chunkserver-list"));

// list chunkserver失败
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::vector<ChunkServerInfo>*>()))
.Times(1)
.WillOnce(Return(-1));
ASSERT_EQ(-1, statusTool.RunCommand("chunkserver-list"));

// FLAGS_checkCSAlive为true的时候，会发送rpc检查chunkserver在线状态
FLAGS_checkHealth = false;
FLAGS_checkCSAlive = true;
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::vector<ChunkServerInfo>*>()))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(chunkservers),
Return(0)));
Expand Down Expand Up @@ -426,11 +432,13 @@ TEST_F(StatusToolTest, StatusCmdCommon) {
{"127.0.0.1:2383", true}};

ChunkServerInfo csInfo;
std::vector<ChunkServerInfo> chunkservers;
std::vector<ChunkServerInfo> chunkserverList;
std::map<PoolIdType, std::vector<ChunkServerInfo>> chunkservers;
for (uint64_t i = 1; i <= 3; ++i) {
GetCsInfoForTest(&csInfo, i, i == 1);
chunkservers.emplace_back(csInfo);
chunkserverList.emplace_back(csInfo);
}
chunkservers.emplace(1, chunkserverList);

// 设置Init的期望
EXPECT_CALL(*mdsClient_, Init(_, _))
Expand Down Expand Up @@ -525,7 +533,8 @@ TEST_F(StatusToolTest, StatusCmdCommon) {
.WillRepeatedly(SetArgPointee<0>(onlineState));

// 4、设置chunkserver status的输出
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::map<PoolIdType, std::vector<ChunkServerInfo>>*>()))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(chunkservers),
Return(0)));
Expand All @@ -542,7 +551,8 @@ TEST_F(StatusToolTest, StatusCmdCommon) {
ASSERT_EQ(0, statusTool.RunCommand("status"));

// 5、设置chunkserver status的输出
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::map<PoolIdType, std::vector<ChunkServerInfo>>*>()))
.Times(1)
.WillOnce(DoAll(SetArgPointee<0>(chunkservers),
Return(0)));
Expand Down Expand Up @@ -652,7 +662,8 @@ TEST_F(StatusToolTest, StatusCmdError) {
// 4、获取chunkserver version失败并ListChunkServersInCluster失败
EXPECT_CALL(*versionTool_, GetAndCheckChunkServerVersion(_, _))
.WillOnce(Return(-1));
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::map<PoolIdType, std::vector<ChunkServerInfo>>*>()))
.Times(1)
.WillOnce(Return(-1));
ASSERT_EQ(-1, statusTool.RunCommand("status"));
Expand All @@ -674,7 +685,8 @@ TEST_F(StatusToolTest, StatusCmdError) {
.WillOnce(DoAll(SetArgPointee<0>("0.0.1"),
SetArgPointee<1>(failedList),
Return(0)));
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(_))
EXPECT_CALL(*mdsClient_, ListChunkServersInCluster(
An<std::map<PoolIdType, std::vector<ChunkServerInfo>>*>()))
.Times(1)
.WillOnce(Return(-1));
ASSERT_EQ(-1, statusTool.RunCommand("chunkserver-status"));
Expand Down
Loading

0 comments on commit 6870e5d

Please sign in to comment.