Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[native] Remove spill stats reporting from presto native #22751

Merged
merged 1 commit into from
May 15, 2024
Merged

[native] Remove spill stats reporting from presto native #22751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 1 addition & 48 deletions presto-native-execution/presto_cpp/main/PeriodicTaskManager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ static constexpr size_t kTaskPeriodCleanOldTasks{60'000'000}; // 60 seconds.
static constexpr size_t kConnectorPeriodGlobalCounters{
60'000'000}; // 60 seconds.
static constexpr size_t kOsPeriodGlobalCounters{2'000'000}; // 2 seconds
static constexpr size_t kSpillStatsUpdateIntervalUs{60'000'000}; // 60 seconds
// Every 1 minute we print endpoint latency counters.
static constexpr size_t kHttpEndpointLatencyPeriodGlobalCounters{
60'000'000}; // 60 seconds.
Expand Down Expand Up @@ -114,6 +113,7 @@ void PeriodicTaskManager::start() {
opts.arbitrator = arbitrator_->kind() == "NOOP" ? nullptr : arbitrator_;
opts.allocator = memoryAllocator_;
opts.cache = asyncDataCache_;
opts.spillMemoryPool = velox::memory::spillMemoryPool();
velox::startPeriodicStatsReporter(opts);

// If executors are null, don't bother starting this task.
Expand All @@ -134,8 +134,6 @@ void PeriodicTaskManager::start() {

addOperatingSystemStatsUpdateTask();

addSpillStatsUpdateTask();

if (SystemConfig::instance()->enableHttpEndpointLatencyFilter()) {
addHttpEndpointLatencyStatsTask();
}
Expand Down Expand Up @@ -393,51 +391,6 @@ void PeriodicTaskManager::addOperatingSystemStatsUpdateTask() {
"os_counters");
}

void PeriodicTaskManager::addSpillStatsUpdateTask() {
addTask(
[this]() { updateSpillStatsTask(); },
kSpillStatsUpdateIntervalUs,
"spill_stats");
}

void PeriodicTaskManager::updateSpillStatsTask() {
const auto updatedSpillStats = velox::common::globalSpillStats();
VELOX_CHECK_GE(updatedSpillStats, lastSpillStats_);
const auto deltaSpillStats = updatedSpillStats - lastSpillStats_;
REPORT_IF_NOT_ZERO(kCounterSpillRuns, deltaSpillStats.spillRuns);
REPORT_IF_NOT_ZERO(kCounterSpilledFiles, deltaSpillStats.spilledFiles);
REPORT_IF_NOT_ZERO(kCounterSpilledRows, deltaSpillStats.spilledRows);
REPORT_IF_NOT_ZERO(kCounterSpilledBytes, deltaSpillStats.spilledBytes);
REPORT_IF_NOT_ZERO(kCounterSpillFillTimeUs, deltaSpillStats.spillFillTimeUs);
REPORT_IF_NOT_ZERO(kCounterSpillSortTimeUs, deltaSpillStats.spillSortTimeUs);
REPORT_IF_NOT_ZERO(
kCounterSpillSerializationTimeUs,
deltaSpillStats.spillSerializationTimeUs);
REPORT_IF_NOT_ZERO(kCounterSpillWrites, deltaSpillStats.spillWrites);
REPORT_IF_NOT_ZERO(
kCounterSpillFlushTimeUs, deltaSpillStats.spillFlushTimeUs);
REPORT_IF_NOT_ZERO(
kCounterSpillWriteTimeUs, deltaSpillStats.spillWriteTimeUs);
REPORT_IF_NOT_ZERO(
kCounterSpillMaxLevelExceeded,
deltaSpillStats.spillMaxLevelExceededCount);

if (!deltaSpillStats.empty()) {
LOG(INFO) << "Updated spill stats: " << updatedSpillStats.toString();
LOG(INFO) << "Spill stats change:" << deltaSpillStats.toString();
}

const auto spillMemoryStats = velox::memory::spillMemoryPool()->stats();
LOG(INFO) << "Spill memory usage: current["
<< velox::succinctBytes(spillMemoryStats.currentBytes) << "] peak["
<< velox::succinctBytes(spillMemoryStats.peakBytes) << "]";
RECORD_METRIC_VALUE(kCounterSpillMemoryBytes, spillMemoryStats.currentBytes);
RECORD_HISTOGRAM_METRIC_VALUE(
kCounterSpillPeakMemoryBytes, spillMemoryStats.peakBytes);

lastSpillStats_ = updatedSpillStats;
}

void PeriodicTaskManager::printHttpEndpointLatencyStats() {
const auto latencyMetrics =
http::filters::HttpEndpointLatencyFilter::retrieveLatencies();
Expand Down
19 changes: 0 additions & 19 deletions presto-native-execution/presto_cpp/main/PeriodicTaskManager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
#include <folly/experimental/FunctionScheduler.h>
#include <folly/experimental/ThreadedRepeatingFunctionRunner.h>
#include "velox/common/memory/Memory.h"
#include "velox/exec/Spill.h"
#include "velox/exec/Task.h"

namespace folly {
Expand Down Expand Up @@ -112,9 +111,6 @@ class PeriodicTaskManager {
void addOperatingSystemStatsUpdateTask();
void updateOperatingSystemStats();

void addSpillStatsUpdateTask();
void updateSpillStatsTask();

// Adds task that periodically prints http endpoint latency metrics.
void addHttpEndpointLatencyStatsTask();
void printHttpEndpointLatencyStats();
Expand All @@ -135,28 +131,13 @@ class PeriodicTaskManager {
std::shared_ptr<velox::connector::Connector>>& connectors_;
PrestoServer* const server_;

// Cache related stats
int64_t lastMemoryCacheHits_{0};
int64_t lastMemoryCacheHitsBytes_{0};
int64_t lastMemoryCacheInserts_{0};
int64_t lastMemoryCacheEvictions_{0};
int64_t lastMemoryCacheEvictionChecks_{0};
int64_t lastMemoryCacheStalls_{0};
int64_t lastMemoryCacheAllocClocks_{0};
int64_t lastMemoryCacheAgedOuts_{0};
int64_t lastSsdCacheCheckpointsWritten_{0};
int64_t lastSsdCacheCheckpointsRead_{0};
int64_t lastSsdCacheRegionsEvicted_{0};

// Operating system related stats.
int64_t lastUserCpuTimeUs_{0};
int64_t lastSystemCpuTimeUs_{0};
int64_t lastSoftPageFaults_{0};
int64_t lastHardPageFaults_{0};
int64_t lastVoluntaryContextSwitches_{0};
int64_t lastForcedContextSwitches_{0};
// Renabled this after update velox.
velox::common::SpillStats lastSpillStats_;

// NOTE: declare last since the threads access other members of `this`.
folly::FunctionScheduler oneTimeRunner_;
Expand Down
22 changes: 0 additions & 22 deletions presto-native-execution/presto_cpp/main/common/Counters.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,28 +110,6 @@ void registerPrestoMetrics() {
62l * 1024 * 1024 * 1024, // max bucket value: 62GB
100);

/// ================== Disk Spilling Counters =================

DEFINE_METRIC(kCounterSpillRuns, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSpilledFiles, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSpilledRows, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSpilledBytes, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSpillFillTimeUs, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSpillSortTimeUs, facebook::velox::StatType::SUM);
DEFINE_METRIC(
kCounterSpillSerializationTimeUs, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSpillWrites, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSpillFlushTimeUs, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSpillWriteTimeUs, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSpillMemoryBytes, facebook::velox::StatType::AVG);
DEFINE_HISTOGRAM_METRIC(
kCounterSpillPeakMemoryBytes,
1l * 512 * 1024 * 1024,
0,
20l * 1024 * 1024 * 1024, // max bucket value: 20GB
100);
DEFINE_METRIC(kCounterSpillMaxLevelExceeded, facebook::velox::StatType::SUM);

// NOTE: Metrics type exporting for file handle cache counters are in
// PeriodicTaskManager because they have dynamic names. The following counters
// have their type exported there:
Expand Down
44 changes: 0 additions & 44 deletions presto-native-execution/presto_cpp/main/common/Counters.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,50 +138,6 @@ constexpr folly::StringPiece kCounterOsNumVoluntaryContextSwitches{
constexpr folly::StringPiece kCounterOsNumForcedContextSwitches{
"presto_cpp.os_num_forced_context_switches"};

/// ================== Disk Spilling Counters =================

/// The number of times that spilling runs on a velox operator.
constexpr folly::StringPiece kCounterSpillRuns{"presto_cpp.spill_run_count"};
/// The number of spilled files.
constexpr folly::StringPiece kCounterSpilledFiles{
"presto_cpp.spilled_file_count"};
/// The number of spilled rows.
constexpr folly::StringPiece kCounterSpilledRows{
"presto_cpp.spilled_row_count"};
/// The number of bytes spilled to disks.
///
/// NOTE: if compression is enabled, this counts the compressed bytes.
constexpr folly::StringPiece kCounterSpilledBytes{"presto_cpp.spilled_bytes"};
/// The time spent on filling rows for spilling.
constexpr folly::StringPiece kCounterSpillFillTimeUs{
"presto_cpp.spill_fill_time_us"};
/// The time spent on sorting rows for spilling.
constexpr folly::StringPiece kCounterSpillSortTimeUs{
"presto_cpp.spill_sort_time_us"};
/// The time spent on serializing rows for spilling.
constexpr folly::StringPiece kCounterSpillSerializationTimeUs{
"presto_cpp.spill_serialization_time_us"};
/// The number of disk writes to spill rows.
constexpr folly::StringPiece kCounterSpillWrites{
"presto_cpp.spill_write_count"};
/// The time spent on copy out serialized rows for disk write. If compression
/// is enabled, this includes the compression time.
constexpr folly::StringPiece kCounterSpillFlushTimeUs{
"presto_cpp.spill_flush_time_us"};
/// The time spent on writing spilled rows to disk.
constexpr folly::StringPiece kCounterSpillWriteTimeUs{
"presto_cpp.spill_write_time_us"};
/// The number of times that a spillable operator exceeds the max spill level
/// limit that can't spill.
constexpr folly::StringPiece kCounterSpillMaxLevelExceeded{
"presto_cpp.spill_exceeded_max_level_count"};
/// The current spilling memory usage in bytes.
constexpr folly::StringPiece kCounterSpillMemoryBytes{
"presto_cpp.spill_memory_bytes"};
/// The peak spilling memory usage in bytes.
constexpr folly::StringPiece kCounterSpillPeakMemoryBytes{
"presto_cpp.spill_peak_memory_bytes"};

/// ================== HiveConnector Counters ==================
/// Format template strings use 'constexpr std::string_view' to be 'fmt::format'
/// compatible.
Expand Down
2 changes: 1 addition & 1 deletion presto-native-execution/velox
Open in desktop
Submodule velox updated 60 files
+6 −0 velox/common/base/Counters.cpp
+6 −0 velox/common/base/Counters.h
+17 −0 velox/common/base/PeriodicStatsReporter.cpp
+8 −2 velox/common/base/PeriodicStatsReporter.h
+151 −4 velox/common/base/tests/StatsReporterTest.cpp
+3 −0 velox/common/caching/AsyncDataCache.cpp
+35 −20 velox/common/caching/SsdCache.cpp
+18 −9 velox/common/caching/SsdCache.h
+12 −13 velox/common/caching/SsdFile.cpp
+33 −10 velox/common/caching/SsdFile.h
+52 −7 velox/common/caching/tests/AsyncDataCacheTest.cpp
+1 −1 velox/common/caching/tests/SsdFileTest.cpp
+13 −1 velox/common/io/IoStatistics.cpp
+7 −1 velox/common/io/IoStatistics.h
+4 −0 velox/connectors/Connector.h
+1 −1 velox/connectors/hive/HiveConnectorSplit.h
+3 −1 velox/connectors/hive/iceberg/CMakeLists.txt
+6 −0 velox/docs/functions/spark/misc.rst
+6 −0 velox/docs/monitoring/metrics.rst
+0 −1 velox/dwio/common/Options.h
+4 −4 velox/dwio/dwrf/test/CacheInputTest.cpp
+3 −0 velox/dwio/dwrf/writer/DictionaryEncodingUtils.h
+2 −0 velox/dwio/dwrf/writer/IntegerDictionaryEncoder.h
+1 −1 velox/exec/Driver.cpp
+1 −0 velox/exec/Driver.h
+5 −1 velox/exec/FilterProject.cpp
+6 −0 velox/exec/TableScan.cpp
+1 −0 velox/exec/TableScan.h
+6 −0 velox/exec/Task.cpp
+7 −3 velox/exec/Task.h
+5 −0 velox/exec/fuzzer/CMakeLists.txt
+1 −1 velox/exec/fuzzer/JoinFuzzer.cpp
+0 −0 velox/exec/fuzzer/JoinFuzzer.h
+1 −1 velox/exec/fuzzer/JoinFuzzerRunner.h
+1 −6 velox/exec/tests/CMakeLists.txt
+43 −16 velox/exec/tests/DriverTest.cpp
+1 −1 velox/exec/tests/JoinFuzzerTest.cpp
+0 −22 velox/exec/tests/TableWriteTest.cpp
+7 −1 velox/expression/EvalCtx.cpp
+11 −1 velox/expression/EvalCtx.h
+8 −3 velox/expression/SimpleFunctionRegistry.cpp
+32 −10 velox/expression/SimpleFunctionRegistry.h
+9 −0 velox/expression/tests/SimpleFunctionTest.cpp
+16 −5 velox/functions/Registerer.h
+12 −11 velox/functions/lib/aggregates/tests/utils/AggregationTestBase.cpp
+1 −1 velox/functions/prestosql/URLFunctions.cpp
+64 −39 velox/functions/prestosql/URLFunctions.h
+5 −4 velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h
+2 −1 velox/functions/sparksql/CMakeLists.txt
+71 −0 velox/functions/sparksql/InputFileName.cpp
+22 −0 velox/functions/sparksql/InputFileName.h
+7 −0 velox/functions/sparksql/Register.cpp
+1 −0 velox/functions/sparksql/tests/CMakeLists.txt
+82 −0 velox/functions/sparksql/tests/InputFileNameTest.cpp
+1 −1 velox/functions/tests/CMakeLists.txt
+35 −4 velox/functions/tests/FunctionRegistryTest.cpp
+14 −14 velox/vector/ComplexVector.h
+14 −18 velox/vector/ConstantVector.h
+31 −41 velox/vector/FlatVector.h
+9 −9 velox/vector/SimpleVector.h
Loading