Skip to content

Commit

Permalink
[native] Remove memory related counters
Browse files Browse the repository at this point in the history
  • Loading branch information
@tanjialiang
tanjialiang committed May 12, 2024
1 parent d9114d3 commit 00655a0
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 298 deletions.
110 changes: 0 additions & 110 deletions presto-native-execution/presto_cpp/main/common/Counters.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,11 +90,6 @@ void registerPrestoMetrics() {
kCounterNumBlockedWaitForSpillDrivers, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterNumBlockedYieldDrivers, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterNumStuckDrivers, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterMappedMemoryBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterAllocatedMemoryBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterMmapRawAllocBytesSmall, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMmapExternalMappedBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterTotalPartitionedOutputBuffer, facebook::velox::StatType::AVG);
DEFINE_METRIC(
Expand All @@ -115,111 +110,6 @@ void registerPrestoMetrics() {
62l * 1024 * 1024 * 1024, // max bucket value: 62GB
100);

/// ================== AsyncDataCache Counters ==================

DEFINE_METRIC(kCounterCacheMaxAgeSecs, facebook::velox::StatType::AVG);

DEFINE_METRIC(kCounterMemoryCacheNumEntries, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumEmptyEntries, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumSharedEntries, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumExclusiveEntries, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumPrefetchedEntries, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheTotalTinyBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheTotalLargeBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheTotalTinyPaddingBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheTotalLargePaddingBytes,
facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheTotalPrefetchBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheSumEvictScore, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumCumulativeHit, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterMemoryCacheNumHit, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheCumulativeHitBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterMemoryCacheHitBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumCumulativeNew, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterMemoryCacheNumNew, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumCumulativeEvict, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterMemoryCacheNumEvict, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumCumulativeEvictChecks,
facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumEvictChecks, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumCumulativeWaitExclusive,
facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumWaitExclusive, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumCumulativeAllocClocks,
facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumAllocClocks, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumCumulativeAgedOutEntries,
facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterMemoryCacheNumAgedOutEntries, facebook::velox::StatType::AVG);

/// ================== SsdCache Counters ==================

DEFINE_METRIC(kCounterSsdCacheCachedEntries, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterSsdCacheCachedRegions, facebook::velox::StatType::AVG);
DEFINE_METRIC(kCounterSsdCacheCachedBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeReadEntries, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeReadBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeWrittenEntries, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeWrittenBytes, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeAgedOutEntries, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeAgedOutRegions, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeOpenSsdErrors, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeOpenCheckpointErrors,
facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeOpenLogErrors, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeDeleteCheckpointErrors,
facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeGrowFileErrors, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeWriteSsdErrors, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeWriteCheckpointErrors,
facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeReadSsdErrors, facebook::velox::StatType::AVG);
DEFINE_METRIC(
kCounterSsdCacheCumulativeReadCheckpointErrors,
facebook::velox::StatType::AVG);

DEFINE_METRIC(
kCounterSsdCacheCheckpointsRead, facebook::velox::StatType::SUM);
DEFINE_METRIC(
kCounterSsdCacheCheckpointsWritten, facebook::velox::StatType::SUM);
DEFINE_METRIC(kCounterSsdCacheRegionsEvicted, facebook::velox::StatType::SUM);

/// ================== Disk Spilling Counters =================

DEFINE_METRIC(kCounterSpillRuns, facebook::velox::StatType::SUM);
Expand Down
191 changes: 3 additions & 188 deletions presto-native-execution/presto_cpp/main/common/Counters.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,12 @@ constexpr folly::StringPiece kCounterHttpClientPrestoExchangeNumOnBody{
/// PrestoExchangeSource.
constexpr folly::StringPiece kCounterHttpClientPrestoExchangeOnBodyBytes{
"presto_cpp.http.client.presto_exchange_source.on_body_bytes"};

/// SerializedPage size in bytes from PrestoExchangeSource.
constexpr folly::StringPiece kCounterPrestoExchangeSerializedPageSize{
"presto_cpp.presto_exchange_source.serialized_page_size"};
/// Peak number of bytes queued in PrestoExchangeSource waiting for consume.
constexpr folly::StringPiece kCounterExchangeSourcePeakQueuedBytes{
"presto_cpp.exchange_source_peak_queued_bytes"};

constexpr folly::StringPiece kCounterNumQueryContexts{
"presto_cpp.num_query_contexts"};
Expand Down Expand Up @@ -136,35 +138,6 @@ constexpr folly::StringPiece kCounterOsNumVoluntaryContextSwitches{
constexpr folly::StringPiece kCounterOsNumForcedContextSwitches{
"presto_cpp.os_num_forced_context_switches"};

/// ================== Memory Counters =================

/// Number of bytes currently mapped in MemoryAllocator. These bytes represent
/// the bytes that are either currently being allocated or were in the past
/// allocated, not yet been returned back to the operating system, in the
/// form of 'Allocation' or 'ContiguousAllocation'.
constexpr folly::StringPiece kCounterMappedMemoryBytes{
"presto_cpp.mapped_memory_bytes"};
/// Number of bytes currently allocated (used) from MemoryAllocator in the form
/// of 'Allocation' or 'ContiguousAllocation'.
constexpr folly::StringPiece kCounterAllocatedMemoryBytes{
"presto_cpp.allocated_memory_bytes"};
/// Number of bytes currently mapped in MmapAllocator, in the form of
/// 'ContiguousAllocation'.
///
/// NOTE: This applies only to MmapAllocator
constexpr folly::StringPiece kCounterMmapExternalMappedBytes{
"presto_cpp.mmap_external_mapped_bytes"};
/// Number of bytes currently allocated from MmapAllocator directly from raw
/// allocateBytes() interface, and internally allocated by malloc. Only small
/// chunks of memory are delegated to malloc
///
/// NOTE: This applies only to MmapAllocator
constexpr folly::StringPiece kCounterMmapRawAllocBytesSmall{
"presto_cpp.mmap_raw_alloc_bytes_small"};
/// Peak number of bytes queued in PrestoExchangeSource waiting for consume.
constexpr folly::StringPiece kCounterExchangeSourcePeakQueuedBytes{
"presto_cpp.exchange_source_peak_queued_bytes"};

/// ================== Disk Spilling Counters =================

/// The number of times that spilling runs on a velox operator.
Expand Down Expand Up @@ -209,164 +182,6 @@ constexpr folly::StringPiece kCounterSpillMemoryBytes{
constexpr folly::StringPiece kCounterSpillPeakMemoryBytes{
"presto_cpp.spill_peak_memory_bytes"};

/// ================== AsyncDataCache Counters ==================

/// Max possible age of AsyncDataCache and SsdCache entries since the raw file
/// was opened to load the cache.
constexpr folly::StringPiece kCounterCacheMaxAgeSecs{
"presto_cpp.cache_max_age_secs"};

/// Total number of cache entries.
constexpr folly::StringPiece kCounterMemoryCacheNumEntries{
"presto_cpp.memory_cache_num_entries"};
/// Total number of cache entries that do not cache anything.
constexpr folly::StringPiece kCounterMemoryCacheNumEmptyEntries{
"presto_cpp.memory_cache_num_empty_entries"};
/// Total number of cache entries that are pinned for shared access.
constexpr folly::StringPiece kCounterMemoryCacheNumSharedEntries{
"presto_cpp.memory_cache_num_shared_entries"};
/// Total number of cache entries that are pinned for exclusive access.
constexpr folly::StringPiece kCounterMemoryCacheNumExclusiveEntries{
"presto_cpp.memory_cache_num_exclusive_entries"};
/// Total number of cache entries that are being or have been prefetched but
/// have not been hit.
constexpr folly::StringPiece kCounterMemoryCacheNumPrefetchedEntries{
"presto_cpp.memory_cache_num_prefetched_entries"};
/// Total number of bytes of the cached data that is much smaller than a
/// 'MappedMemory' page (AsyncDataCacheEntry::kTinyDataSize).
constexpr folly::StringPiece kCounterMemoryCacheTotalTinyBytes{
"presto_cpp.memory_cache_total_tiny_bytes"};
/// Total number of bytes of the cached data excluding
/// 'kCounterMemoryCacheTotalTinyBytes'.
constexpr folly::StringPiece kCounterMemoryCacheTotalLargeBytes{
"presto_cpp.memory_cache_total_large_bytes"};
/// Total unused capacity bytes in 'kCounterMemoryCacheTotalTinyBytes'.
constexpr folly::StringPiece kCounterMemoryCacheTotalTinyPaddingBytes{
"presto_cpp.memory_cache_total_tiny_padding_bytes"};
/// Total unused capacity bytes in 'kCounterMemoryCacheTotalLargeBytes'.
constexpr folly::StringPiece kCounterMemoryCacheTotalLargePaddingBytes{
"presto_cpp.memory_cache_total_large_padding_bytes"};
/// Total bytes of cache entries in prefetch state.
constexpr folly::StringPiece kCounterMemoryCacheTotalPrefetchBytes{
"presto_cpp.memory_cache_total_prefetched_bytes"};
/// Sum of scores of evicted entries. This serves to infer an average lifetime
/// for entries in cache.
constexpr folly::StringPiece kCounterMemoryCacheSumEvictScore{
"presto_cpp.memory_cache_sum_evict_score"};
/// Cumulative number of hits (saved IO). The first hit to a prefetched entry
/// does not count.
constexpr folly::StringPiece kCounterMemoryCacheNumCumulativeHit{
"presto_cpp.memory_cache_num_cumulative_hit"};
/// Number of hits (saved IO) since last counter retrieval. The first hit to a
/// prefetched entry does not count.
constexpr folly::StringPiece kCounterMemoryCacheNumHit{
"presto_cpp.memory_cache_num_hit"};
/// Cumulative amount of hit bytes (saved IO). The first hit to a prefetched
/// entry does not count.
constexpr folly::StringPiece kCounterMemoryCacheCumulativeHitBytes{
"presto_cpp.memory_cache_cumulative_hit_bytes"};
/// Amount of hit bytes (saved IO) since last counter retrieval. The first hit
/// to a prefetched entry does not count.
constexpr folly::StringPiece kCounterMemoryCacheHitBytes{
"presto_cpp.memory_cache_hit_bytes"};
/// Cumulative number of new entries created.
constexpr folly::StringPiece kCounterMemoryCacheNumCumulativeNew{
"presto_cpp.memory_cache_num_cumulative_new"};
/// Number of new entries created since last counter retrieval.
constexpr folly::StringPiece kCounterMemoryCacheNumNew{
"presto_cpp.memory_cache_num_new"};
/// Cumulative number of times a valid entry was removed in order to make space.
constexpr folly::StringPiece kCounterMemoryCacheNumCumulativeEvict{
"presto_cpp.memory_cache_num_cumulative_evict"};
/// Number of times a valid entry was removed in order to make space, since last
/// counter retrieval.
constexpr folly::StringPiece kCounterMemoryCacheNumEvict{
"presto_cpp.memory_cache_num_evict"};
/// Cumulative number of entries considered for evicting.
constexpr folly::StringPiece kCounterMemoryCacheNumCumulativeEvictChecks{
"presto_cpp.memory_cache_num_cumulative_evict_checks"};
/// Number of entries considered for evicting, since last counter retrieval.
constexpr folly::StringPiece kCounterMemoryCacheNumEvictChecks{
"presto_cpp.memory_cache_num_evict_checks"};
/// Cumulative number of times a user waited for an entry to transit from
/// exclusive to shared mode.
constexpr folly::StringPiece kCounterMemoryCacheNumCumulativeWaitExclusive{
"presto_cpp.memory_cache_num_cumulative_wait_exclusive"};
/// Number of times a user waited for an entry to transit from exclusive to
/// shared mode, since last counter retrieval.
constexpr folly::StringPiece kCounterMemoryCacheNumWaitExclusive{
"presto_cpp.memory_cache_num_wait_exclusive"};
/// Cumulative clocks spent in allocating or freeing memory for backing cache
/// entries.
constexpr folly::StringPiece kCounterMemoryCacheNumCumulativeAllocClocks{
"presto_cpp.memory_cache_num_cumulative_alloc_clocks"};
/// Clocks spent in allocating or freeing memory for backing cache entries,
/// since last counter retrieval
constexpr folly::StringPiece kCounterMemoryCacheNumAllocClocks{
"presto_cpp.memory_cache_num_alloc_clocks"};
/// Cumulative number of AsyncDataCache entries that are aged out and evicted
/// given configured TTL.
constexpr folly::StringPiece kCounterMemoryCacheNumCumulativeAgedOutEntries{
"presto_cpp.memory_cache_num_cumulative_aged_out_entries"};
/// Number of AsyncDataCache entries that are aged out and evicted
/// given configured TTL.
constexpr folly::StringPiece kCounterMemoryCacheNumAgedOutEntries{
"presto_cpp.memory_cache_num_aged_out_entries"};

/// ================== SsdCache Counters ==================

/// Number of regions currently cached by SSD.
constexpr folly::StringPiece kCounterSsdCacheCachedRegions{
"presto_cpp.ssd_cache_cached_regions"};
/// Number of entries currently cached by SSD.
constexpr folly::StringPiece kCounterSsdCacheCachedEntries{
"presto_cpp.ssd_cache_cached_entries"};
/// Total bytes currently cached by SSD.
constexpr folly::StringPiece kCounterSsdCacheCachedBytes{
"presto_cpp.ssd_cache_cached_bytes"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeReadEntries{
"presto_cpp.ssd_cache_cumulative_read_entries"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeReadBytes{
"presto_cpp.ssd_cache_cumulative_read_bytes"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeWrittenEntries{
"presto_cpp.ssd_cache_cumulative_written_entries"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeWrittenBytes{
"presto_cpp.ssd_cache_cumulative_written_bytes"};
/// Cumulative number of SsdCache entries that are aged out and evicted given
/// configured TTL.
constexpr folly::StringPiece kCounterSsdCacheCumulativeAgedOutEntries{
"presto_cpp.ssd_cache_cumulative_aged_out_entries"};
/// Cumulative number of SsdCache regions that are aged out and evicted given
/// configured TTL.
constexpr folly::StringPiece kCounterSsdCacheCumulativeAgedOutRegions{
"presto_cpp.ssd_cache_cumulative_aged_out_regions"};

constexpr folly::StringPiece kCounterSsdCacheCumulativeOpenSsdErrors{
"presto_cpp.ssd_cache_cumulative_open_ssd_errors"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeOpenCheckpointErrors{
"presto_cpp.ssd_cache_cumulative_open_checkpoint_errors"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeOpenLogErrors{
"presto_cpp.ssd_cache_cumulative_open_log_errors"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeDeleteCheckpointErrors{
"presto_cpp.ssd_cache_cumulative_delete_checkpoint_errors"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeGrowFileErrors{
"presto_cpp.ssd_cache_cumulative_grow_file_errors"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeWriteSsdErrors{
"presto_cpp.ssd_cache_cumulative_write_ssd_errors"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeWriteCheckpointErrors{
"presto_cpp.ssd_cache_cumulative_write_checkpoint_errors"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeReadSsdErrors{
"presto_cpp.ssd_cache_cumulative_read_ssd_errors"};
constexpr folly::StringPiece kCounterSsdCacheCumulativeReadCheckpointErrors{
"presto_cpp.ssd_cache_cumulative_read_checkpoint_errors"};

constexpr folly::StringPiece kCounterSsdCacheCheckpointsRead{
"presto_cpp.ssd_cache_checkpoints_read"};
constexpr folly::StringPiece kCounterSsdCacheCheckpointsWritten{
"presto_cpp.ssd_cache_checkpoints_written"};
constexpr folly::StringPiece kCounterSsdCacheRegionsEvicted{
"presto_cpp.ssd_cache_regions_evicted"};

/// ================== HiveConnector Counters ==================
/// Format template strings use 'constexpr std::string_view' to be 'fmt::format'
/// compatible.
Expand Down

0 comments on commit 00655a0

Please sign in to comment.