astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-22 05:30:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

Set the maximum open connections limit in PG and MySQL target configs (#10558)

Browse Source 
As the bulk/recursive delete will require multiple connections to open at an instance,
The default open connections limit will be reached which results in the following error

```FATAL:  sorry, too many clients already```

By setting the open connections to a reasonable value - `2`, We ensure that the max open connections
will not be exhausted and lie under bounds.

The queries are simple inserts/updates/deletes which is operational and sufficient with the
the maximum open connection limit is 2.

Fixes #10553

Allow user configuration for MaxOpenConnections
This commit is contained in:
Praveen raj Mani
2020-09-25 10:50:30 +05:30
committed by GitHub
parent 37a5d5d7a0
commit b880796aef
9 changed files with 198 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/event/target/postgresql.go
+48 -32
View File
@@ -84,43 +84,46 @@ const (
// Postgres constants
const (
PostgresFormat = "format"
PostgresConnectionString = "connection_string"
PostgresTable = "table"
PostgresHost = "host"
PostgresPort = "port"
PostgresUsername = "username"
PostgresPassword = "password"
PostgresDatabase = "database"
PostgresQueueDir = "queue_dir"
PostgresQueueLimit = "queue_limit"
PostgresFormat = "format"
PostgresConnectionString = "connection_string"
PostgresTable = "table"
PostgresHost = "host"
PostgresPort = "port"
PostgresUsername = "username"
PostgresPassword = "password"
PostgresDatabase = "database"
PostgresQueueDir = "queue_dir"
PostgresQueueLimit = "queue_limit"
PostgresMaxOpenConnections = "max_open_connections"
EnvPostgresEnable = "MINIO_NOTIFY_POSTGRES_ENABLE"
EnvPostgresFormat = "MINIO_NOTIFY_POSTGRES_FORMAT"
EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING"
EnvPostgresTable = "MINIO_NOTIFY_POSTGRES_TABLE"
EnvPostgresHost = "MINIO_NOTIFY_POSTGRES_HOST"
EnvPostgresPort = "MINIO_NOTIFY_POSTGRES_PORT"
EnvPostgresUsername = "MINIO_NOTIFY_POSTGRES_USERNAME"
EnvPostgresPassword = "MINIO_NOTIFY_POSTGRES_PASSWORD"
EnvPostgresDatabase = "MINIO_NOTIFY_POSTGRES_DATABASE"
EnvPostgresQueueDir = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR"
EnvPostgresQueueLimit = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT"
EnvPostgresEnable = "MINIO_NOTIFY_POSTGRES_ENABLE"
EnvPostgresFormat = "MINIO_NOTIFY_POSTGRES_FORMAT"
EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING"
EnvPostgresTable = "MINIO_NOTIFY_POSTGRES_TABLE"
EnvPostgresHost = "MINIO_NOTIFY_POSTGRES_HOST"
EnvPostgresPort = "MINIO_NOTIFY_POSTGRES_PORT"
EnvPostgresUsername = "MINIO_NOTIFY_POSTGRES_USERNAME"
EnvPostgresPassword = "MINIO_NOTIFY_POSTGRES_PASSWORD"
EnvPostgresDatabase = "MINIO_NOTIFY_POSTGRES_DATABASE"
EnvPostgresQueueDir = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR"
EnvPostgresQueueLimit = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT"
EnvPostgresMaxOpenConnections = "MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS"
)
// PostgreSQLArgs - PostgreSQL target arguments.
type PostgreSQLArgs struct {
Enable bool `json:"enable"`
Format string `json:"format"`
ConnectionString string `json:"connectionString"`
Table string `json:"table"`
Host xnet.Host `json:"host"` // default: localhost
Port string `json:"port"` // default: 5432
Username string `json:"username"` // default: user running minio
Password string `json:"password"` // default: no password
Database string `json:"database"` // default: same as user
QueueDir string `json:"queueDir"`
QueueLimit uint64 `json:"queueLimit"`
Enable bool `json:"enable"`
Format string `json:"format"`
ConnectionString string `json:"connectionString"`
Table string `json:"table"`
Host xnet.Host `json:"host"` // default: localhost
Port string `json:"port"` // default: 5432
Username string `json:"username"` // default: user running minio
Password string `json:"password"` // default: no password
Database string `json:"database"` // default: same as user
QueueDir string `json:"queueDir"`
QueueLimit uint64 `json:"queueLimit"`
MaxOpenConnections int `json:"maxOpenConnections"`
}
// Validate PostgreSQLArgs fields
@@ -160,6 +163,10 @@ func (p PostgreSQLArgs) Validate() error {
}
}
if p.MaxOpenConnections < 0 {
return errors.New("maxOpenConnections cannot be less than zero")
}
return nil
}
@@ -195,6 +202,10 @@ func (target *PostgreSQLTarget) IsActive() (bool, error) {
return false, err
}
target.db = db
if target.args.MaxOpenConnections > 0 {
// Set the maximum connections limit
target.db.SetMaxOpenConns(target.args.MaxOpenConnections)
}
}
if err := target.db.Ping(); err != nil {
if IsConnErr(err) {
@@ -391,6 +402,11 @@ func NewPostgreSQLTarget(id string, args PostgreSQLArgs, doneCh <-chan struct{},
}
target.db = db
if args.MaxOpenConnections > 0 {
// Set the maximum connections limit
target.db.SetMaxOpenConns(args.MaxOpenConnections)
}
var store Store
if args.QueueDir != "" {