Skip to content

Commit

Permalink
Rationalise postgres configuration. Closes #2471
Browse files Browse the repository at this point in the history
  • Loading branch information
@kaidaguerre
kaidaguerre committed Oct 3, 2022
1 parent 9130a45 commit e2d12b3
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 91 deletions.
126 changes: 39 additions & 87 deletions pkg/constants/postgresql_conf.go
View file
Original file line number Diff line number Diff line change
@@ -1,109 +1,61 @@
package constants

const PostgresqlConfContent = `# -----------------------------
const PostgresqlConfContent = `
# -----------------------------
# PostgreSQL configuration file
# -----------------------------
#
# DO NOT EDIT THIS FILE
# it will be overwritten on server start
# DO NOT EDIT THIS FILE!
# It is overwritten each time Steampipe starts.
#
# In the rare case that postgres.conf customization is required, modifications
# or additions should be placed in the 'postgres.conf.d' folder as a config
# include file. For example: 'postgres.conf.d/01-custom-settings.conf'.
# See https://www.postgresql.org/docs/current/config-setting.html#CONFIG-INCLUDES
#
#------------------------------------------------------------------------------
# STEAMPIPE CONFIG ADDITIONS
#------------------------------------------------------------------------------
# First, use Steampipe's default settings for Postgres.
include = 'steampipe.conf'
# settings in this directory will override any steampipe provided values from the
# config file above
include_dir = 'postgresql.conf.d' # includes files ending in .conf inside this dir
# Second, allow users to customize Postgres settings with custom '.conf' files
# created in the 'postgresql.conf.d' directory. Use with care, these settings
# overwrite any 'steampipe.conf' settings above.
include_dir = 'postgresql.conf.d'
`

const SteampipeConfContent = `
# Custom settings for Steampipe
#
# DO NOT EDIT THIS FILE
# it will be overwritten on server start
#
# Modification or additions to postgres config should be placed in the
# postgresql.conf.d folder with a name alphabetically following this
# file - for example 01-custom-settings.conf
# ------------------------------------------
# Steampipe's default Postgres configuration
# ------------------------------------------
#
# see documentation on this behavior in the postgresql docs:
# https://www.postgresql.org/docs/11/config-setting.html#CONFIG-INCLUDES
bgwriter_lru_maxpages=0
effective_cache_size=64kB
fsync=off
full_page_writes=off
maintenance_work_mem=1024kB
password_encryption=scram-sha-256
random_page_cost=0.01
seq_page_cost=0.01
# If the shared buffers are too small then large tables in memory can create
# "no unpinned buffers available" errors.
# DO NOT EDIT THIS FILE!
# It is overwritten each time Steampipe starts.
#
# In that case, set shared_buffers in an overriding config file
# shared_buffers=128kB
# If synchronous_commit=off then the setup process can fail because the
# installation of the foreign server is not committed before the DB shutsdown.
# Steampipe does very few commits in general, so leaving this on will have
# very little impact on performance.
# In the rare case that postgres.conf customization is required, modifications
# or additions should be placed in the 'postgres.conf.d' folder as a config
# include file. For example: 'postgres.conf.d/01-custom-settings.conf'.
# See https://www.postgresql.org/docs/current/config-setting.html#CONFIG-INCLUDES
#
# In that case, set synchronous_commit in an overriding config file
# synchronous_commit=off
temp_buffers=800kB
# Steampipe is run in many different systems and regions, so use UTC for all
# timestamps by default - both in SQL responses and log entries.
timezone=UTC
track_activities=off
track_counts = on
wal_buffers=32kB
work_mem=64kB
jit=off
max_locks_per_transaction = 2048
# postgres log collection
log_connections=on
log_disconnections=on
log_min_duration_statement=1000
log_destination=stderr
log_statement=none
log_min_error_statement=error
logging_collector=on
log_filename='database-%Y-%m-%d.log'
log_timezone=UTC
# Make the database log consistent with our plugin logs in both name and daily
# rotation frequency. These will appear in '~/.steampipe/logs' and are cleared
# after 7 days by the Steampipe CLI.
log_filename='database-%Y-%m-%d.log'
#------------------------------------------------------------------------------
# AUTOVACUUM PARAMETERS
#------------------------------------------------------------------------------
autovacuum = on # Enable autovacuum subprocess
# Postgres log messages sent to stderr should be redirected to the log file.
logging_collector=on
# all params left as default:
#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze
#autovacuum_analyze_threshold = 50 # min number of row updates before analyze
#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum
#autovacuum_max_workers = 3 # max number of autovacuum subprocesses
#autovacuum_multixact_freeze_max_age = 400000000
# maximum multixact age
# before forced vacuum
#autovacuum_naptime = 1min # time between autovacuum runs
#autovacuum_vacuum_cost_delay = 2ms # default vacuum cost delay for
# autovacuum, in milliseconds;
# -1 means use vacuum_cost_delay
#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for
# autovacuum, -1 means use
# vacuum_cost_limit
#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum
#autovacuum_vacuum_threshold = 50 # min number of row updates before vacuum
#autovacuum_work_mem = -1 # qthe maximum memory to be used by each autovacuum worker process.
#log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and
# their durations, > 0 logs only
# actions running at least this number
# of milliseconds.
# Connection logging is fast, low volume and helpful to troubleshoot issues
# around plugin startup or failure.
log_connections=on
log_disconnections=on
`
# Logging of slow queries (> 5 secs) is helpful when reviewing environments or
# troubleshooting with users.
log_min_duration_statement=5000
`
14 changes: 10 additions & 4 deletions pkg/db/db_local/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ func initDatabase() error {

// initdb sometimes fail due to invalid locale settings, to avoid this we update
// the locale settings to use 'C' only for the initdb process to complete, and
// then return back to the existing locale settings of the user.
// then return to the existing locale settings of the user.
// set LC_ALL env variable to override current locale settings
err := os.Setenv("LC_ALL", "C")
if err != nil {
Expand All @@ -447,12 +447,18 @@ func initDatabase() error {
initDBExecutable := getInitDbBinaryExecutablePath()
initDbProcess := exec.Command(
initDBExecutable,
// Steampipe runs Postgres as a local, embedded database so trust local
// users to login without a password.
fmt.Sprintf("--auth=%s", "trust"),
// Ensure the name of the database superuser is consistent across installs.
// By default it would be based on the user running the install of this
// embedded database.
fmt.Sprintf("--username=%s", constants.DatabaseSuperUser),
// Postgres data should placed under the Steampipe install directory.
fmt.Sprintf("--pgdata=%s", getDataLocation()),
// Ensure the encoding is consistent across installs. By default it would
// be based on the system locale.
fmt.Sprintf("--encoding=%s", "UTF-8"),
fmt.Sprintf("--wal-segsize=%d", 1),
"--debug",
)

log.Printf("[TRACE] initdb start: %s", initDbProcess.String())
Expand All @@ -463,7 +469,7 @@ func initDatabase() error {
return runError
}

// unset LC_ALL to return back to original locale settings
// unset LC_ALL to return to original locale settings
err = os.Unsetenv("LC_ALL")
if err != nil {
log.Printf("[TRACE] failed to return back to original locale settings:\n %s", err.Error())
Expand Down

0 comments on commit e2d12b3

Please sign in to comment.