Support access format for database notification targets (#3953)

* Add configuration parameter "format" for db targets and perform configuration migration. * Add PostgreSQL `access` format: This causes Minio to append all events to the configured table. Prefix, suffix and event filters continue to be supported for this mode too. * Update documentation for PostgreSQL notification target. * Add MySQL `access` format: It is very similar to the same format for PostgreSQL. * Update MySQL notification documentation.
2026-07-22 13:40:22 +03:00 · 2017-03-27 23:57:25 +05:30
parent 6e63904048
commit a099319e66
14 changed files with 637 additions and 211 deletions
@@ -14,11 +14,19 @@
 * limitations under the License.
 */

-// MySQL Notifier implementation. A table with a specific
-// structure (column names, column types, and primary key/uniqueness
-// constraint) is used. The user may set the table name in the
-// configuration. A sample SQL command that creates a command with the
-// required structure is:
+// MySQL Notifier implementation. Two formats, "namespace" and
+// "access" are supported.
+//
+// * Namespace format
+//
+// On each create or update object event in Minio Object storage
+// server, a row is created or updated in the table in MySQL. On each
+// object removal, the corresponding row is deleted from the table.
+//
+// A table with a specific structure (column names, column types, and
+// primary key/uniqueness constraint) is used. The user may set the
+// table name in the configuration. A sample SQL command that creates
+// a command with the required structure is:
 //
 //     CREATE TABLE myminio (
 //         key_name VARCHAR(2048),
@@ -30,10 +38,18 @@
 // here. The implementation has been tested with MySQL Ver 14.14
 // Distrib 5.7.17.
 //
-// On each create or update object event in Minio Object storage
-// server, a row is created or updated in the table in MySQL. On
-// each object removal, the corresponding row is deleted from the
-// table.
+// * Access format
+//
+// On each event, a row is appended to the configured table. There is
+// no deletion or modification of existing rows.
+//
+// A different table schema is used for this format. A sample SQL
+// commant that creates a table with the required structure is:
+//
+// CREATE TABLE myminio (
+//     event_time TIMESTAMP WITH TIME ZONE NOT NULL,
+//     event_data JSONB
+// );

 package cmd

@@ -42,29 +58,53 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"time"

 	"github.com/Sirupsen/logrus"
 	"github.com/go-sql-driver/mysql"
 )

 const (
-	upsertRowMySQL = `INSERT INTO %s (key_name, value)
+	// Queries for format=namespace mode.
+	upsertRowForNSMySQL = `INSERT INTO %s (key_name, value)
 VALUES (?, ?)
 ON DUPLICATE KEY UPDATE value=VALUES(value);
 `
-	deleteRowMySQL = ` DELETE FROM %s
+	deleteRowForNSMySQL = ` DELETE FROM %s
 WHERE key_name = ?;`
-	createTableMySQL = `CREATE TABLE %s (
+	createTableForNSMySQL = `CREATE TABLE %s (
    key_name VARCHAR(2048),
    value JSON,
    PRIMARY KEY (key_name)
 );`
+
+	// Queries for format=access mode.
+	insertRowForAccessMySQL = `INSERT INTO %s (event_time, event_data)
+VALUES (?, ?);`
+	createTableForAccessMySQL = `CREATE TABLE %s (
+    event_time DATETIME NOT NULL,
+    event_data JSON
+);`
+
+	// Query to check if a table already exists.
 	tableExistsMySQL = `SELECT 1 FROM %s;`
 )

+func makeMySQLError(msg string, a ...interface{}) error {
+	s := fmt.Sprintf(msg, a...)
+	return fmt.Errorf("MySQL Notifier Error: %s", s)
+}
+
+var (
+	myNFormatError = makeMySQLError(`"format" value is invalid - it must be one of "%s" or "%s".`, formatNamespace, formatAccess)
+	myNTableError  = makeMySQLError("Table was not specified in the configuration.")
+)
+
 type mySQLNotify struct {
 	Enable bool `json:"enable"`

+	Format string `json:"format"`
+
 	// pass data-source-name connection string in config
 	// directly. This string is formatted according to
 	// https://github.com/go-sql-driver/mysql#dsn-data-source-name
@@ -86,14 +126,16 @@ func (m *mySQLNotify) Validate() error {
 	if !m.Enable {
 		return nil
 	}
+	if m.Format != formatNamespace && m.Format != formatAccess {
+		return myNFormatError
+	}
 	if m.DsnString == "" {
 		if _, err := checkURL(m.Host); err != nil {
 			return err
 		}
 	}
 	if m.Table == "" {
-		return fmt.Errorf(
-			"MySQL Notifier Error: Table was not specified in configuration")
+		return myNTableError
 	}
 	return nil
 }
@@ -101,6 +143,7 @@ func (m *mySQLNotify) Validate() error {
 type mySQLConn struct {
 	dsnStr        string
 	table         string
+	format        string
 	preparedStmts map[string]*sql.Stmt
 	*sql.DB
 }
@@ -126,30 +169,32 @@ func dialMySQL(msql mySQLNotify) (mySQLConn, error) {

 	db, err := sql.Open("mysql", dsnStr)
 	if err != nil {
-		return mySQLConn{}, fmt.Errorf(
-			"MySQL Notifier Error: Connection opening failure (dsnStr=%s): %v",
-			dsnStr, err,
-		)
+		return mySQLConn{}, makeMySQLError(
+			"Connection opening failure (dsnStr=%s): %v",
+			dsnStr, err)
 	}

 	// ping to check that server is actually reachable.
 	err = db.Ping()
 	if err != nil {
-		return mySQLConn{}, fmt.Errorf(
-			"MySQL Notifier Error: Ping to server failed with: %v",
-			err,
-		)
+		return mySQLConn{}, makeMySQLError(
+			"Ping to server failed with: %v", err)
 	}

 	// check that table exists - if not, create it.
 	_, err = db.Exec(fmt.Sprintf(tableExistsMySQL, msql.Table))
 	if err != nil {
+		createStmt := createTableForNSMySQL
+		if msql.Format == formatAccess {
+			createStmt = createTableForAccessMySQL
+		}
+
 		// most likely, table does not exist. try to create it:
-		_, errCreate := db.Exec(fmt.Sprintf(createTableMySQL, msql.Table))
+		_, errCreate := db.Exec(fmt.Sprintf(createStmt, msql.Table))
 		if errCreate != nil {
 			// failed to create the table. error out.
-			return mySQLConn{}, fmt.Errorf(
-				"MySQL Notifier Error: 'Select' failed with %v, then 'Create Table' failed with %v",
+			return mySQLConn{}, makeMySQLError(
+				"'Select' failed with %v, then 'Create Table' failed with %v",
 				err, errCreate,
 			)
 		}
@@ -157,19 +202,33 @@ func dialMySQL(msql mySQLNotify) (mySQLConn, error) {

 	// create prepared statements
 	stmts := make(map[string]*sql.Stmt)
-	// insert or update statement
-	stmts["upsertRow"], err = db.Prepare(fmt.Sprintf(upsertRowMySQL, msql.Table))
-	if err != nil {
-		return mySQLConn{},
-			fmt.Errorf("MySQL Notifier Error: create UPSERT prepared statement failed with: %v", err)
-	}
-	stmts["deleteRow"], err = db.Prepare(fmt.Sprintf(deleteRowMySQL, msql.Table))
-	if err != nil {
-		return mySQLConn{},
-			fmt.Errorf("MySQL Notifier Error: create DELETE prepared statement failed with: %v", err)
-	}
+	switch msql.Format {
+	case formatNamespace:
+		// insert or update statement
+		stmts["upsertRow"], err = db.Prepare(fmt.Sprintf(upsertRowForNSMySQL,
+			msql.Table))
+		if err != nil {
+			return mySQLConn{},
+				makeMySQLError("create UPSERT prepared statement failed with: %v", err)
+		}
+		// delete statement
+		stmts["deleteRow"], err = db.Prepare(fmt.Sprintf(deleteRowForNSMySQL,
+			msql.Table))
+		if err != nil {
+			return mySQLConn{},
+				makeMySQLError("create DELETE prepared statement failed with: %v", err)
+		}
+	case formatAccess:
+		// insert statement
+		stmts["insertRow"], err = db.Prepare(fmt.Sprintf(insertRowForAccessMySQL,
+			msql.Table))
+		if err != nil {
+			return mySQLConn{}, makeMySQLError(
+				"create INSERT prepared statement failed with: %v", err)
+		}

-	return mySQLConn{dsnStr, msql.Table, stmts, db}, nil
+	}
+	return mySQLConn{dsnStr, msql.Table, msql.Format, stmts, db}, nil
 }

 func newMySQLNotify(accountID string) (*logrus.Logger, error) {
@@ -210,35 +269,66 @@ func (myC mySQLConn) Fire(entry *logrus.Entry) error {
 		return nil
 	}

-	// Check for event delete
-	if eventMatch(entryEventType, []string{"s3:ObjectRemoved:*"}) {
-		// delete row from the table
-		_, err := myC.preparedStmts["deleteRow"].Exec(entry.Data["Key"])
-		if err != nil {
-			return fmt.Errorf(
-				"Error deleting event with key = %v - got mysql error - %v",
-				entry.Data["Key"], err,
-			)
-		}
-	} else {
-		// json encode the value for the row
+	jsonEncoder := func(d interface{}) ([]byte, error) {
 		value, err := json.Marshal(map[string]interface{}{
-			"Records": entry.Data["Records"],
+			"Records": d,
 		})
 		if err != nil {
-			return fmt.Errorf(
-				"Unable to encode event %v to JSON - got error - %v",
-				entry.Data["Records"], err,
-			)
+			return nil, makeMySQLError(
+				"Unable to encode event %v to JSON: %v", d, err)
+		}
+		return value, nil
+	}
+
+	switch myC.format {
+	case formatNamespace:
+		// Check for event delete
+		if eventMatch(entryEventType, []string{"s3:ObjectRemoved:*"}) {
+			// delete row from the table
+			_, err := myC.preparedStmts["deleteRow"].Exec(entry.Data["Key"])
+			if err != nil {
+				return makeMySQLError(
+					"Error deleting event with key = %v - got mysql error - %v",
+					entry.Data["Key"], err,
+				)
+			}
+		} else {
+			value, err := jsonEncoder(entry.Data["Records"])
+			if err != nil {
+				return err
+			}
+
+			// upsert row into the table
+			_, err = myC.preparedStmts["upsertRow"].Exec(entry.Data["Key"], value)
+			if err != nil {
+				return makeMySQLError(
+					"Unable to upsert event with Key=%v and Value=%v - got mysql error - %v",
+					entry.Data["Key"], entry.Data["Records"], err,
+				)
+			}
+		}
+	case formatAccess:
+		// eventTime is taken from the first entry in the
+		// records.
+		events, ok := entry.Data["Records"].([]NotificationEvent)
+		if !ok {
+			return makeMySQLError("unable to extract event time due to conversion error of entry.Data[\"Records\"]=%v", entry.Data["Records"])
+		}
+		eventTime, err := time.Parse(timeFormatAMZ, events[0].EventTime)
+		if err != nil {
+			return makeMySQLError("unable to parse event time \"%s\": %v",
+				events[0].EventTime, err)
 		}

-		// upsert row into the table
-		_, err = myC.preparedStmts["upsertRow"].Exec(entry.Data["Key"], value)
+		value, err := jsonEncodeEventData(entry.Data["Records"])
 		if err != nil {
-			return fmt.Errorf(
-				"Unable to upsert event with Key=%v and Value=%v - got mysql error - %v",
-				entry.Data["Key"], entry.Data["Records"], err,
-			)
+			return err
+		}
+
+		_, err = myC.preparedStmts["insertRow"].Exec(eventTime, value)
+		if err != nil {
+			return makeMySQLError("Unable to insert event with value=%v: %v",
+				value, err)
 		}
 	}