Add bucket metadata state client/handler (Fixes #3022) (#3152)

- Adds an interface to update in-memory bucket metadata state called
  BucketMetaState - this interface has functions to:
     - update bucket notification configuration,
     - bucket listener configuration,
     - bucket policy configuration, and
     - send bucket event

- This interface is implemented by `localBMS` a type for manipulating
  local node in-memory bucket metadata, and by `remoteBMS` a type for
  manipulating remote node in-memory bucket metadata.

- The remote node interface, makes an RPC call, but the local node
  interface does not - it updates in-memory bucket state directly.

- Rename mkPeersFromEndpoints to makeS3Peers and refactored it.

- Use arrayslice instead of map in s3Peers struct

- `s3Peers.SendUpdate` now receives an arrayslice of peer indexes to
  send the request to, with a special nil value slice indicating that
  all peers should be sent the update.

- `s3Peers.SendUpdate` now returns an arrayslice of errors, representing
  errors from peers when sending an update. The array positions
  correspond to peer array s3Peers.peers

Improve globalS3Peers:

- Make isDistXL a global `globalIsDistXL` and remove from s3Peers

- Make globalS3Peers an array of (address, bucket-meta-state) pairs.

- Fix code and tests.

cmd/utils_test.go

@@ -229,13 +229,14 @@ func TestLocalAddress(t *testing.T) {
 	globalMinioPort = "9000"
 	globalMinioHost = ""
 	testCases := []struct {
+		isDistXL     bool
 		srvCmdConfig serverCmdConfig
 		localAddr    string
 	}{
 		// Test 1 - local address is found.
 		{
+			isDistXL: true,
 			srvCmdConfig: serverCmdConfig{
-				isDistXL: true,
 				endpoints: []*url.URL{{
 					Scheme: "http",
 					Host:   "localhost:9000",
@@ -258,9 +259,9 @@ func TestLocalAddress(t *testing.T) {
 		},
 		// Test 2 - local address is everything.
 		{
+			isDistXL: false,
 			srvCmdConfig: serverCmdConfig{
 				serverAddr: net.JoinHostPort("", globalMinioPort),
-				isDistXL:   false,
 				endpoints: []*url.URL{{
 					Path: "/mnt/disk1",
 				}, {
@@ -275,8 +276,8 @@ func TestLocalAddress(t *testing.T) {
 		},
 		// Test 3 - local address is not found.
 		{
+			isDistXL: true,
 			srvCmdConfig: serverCmdConfig{
-				isDistXL: true,
 				endpoints: []*url.URL{{
 					Scheme: "http",
 					Host:   "1.1.1.1:9000",
@@ -301,9 +302,9 @@ func TestLocalAddress(t *testing.T) {
 		// name is specified in the --address option on the
 		// server command line.
 		{
+			isDistXL: false,
 			srvCmdConfig: serverCmdConfig{
 				serverAddr: "play.minio.io:9000",
-				isDistXL:   false,
 				endpoints: []*url.URL{{
 					Path: "/mnt/disk1",
 				}, {
@@ -320,6 +321,7 @@ func TestLocalAddress(t *testing.T) {
 
 	// Validates fetching local address.
 	for i, testCase := range testCases {
+		globalIsDistXL = testCase.isDistXL
 		localAddr := getLocalAddress(testCase.srvCmdConfig)
 		if localAddr != testCase.localAddr {
 			t.Fatalf("Test %d: Expected %s, got %s", i+1, testCase.localAddr, localAddr)