Reduce JWT overhead for internode tokens (#13738)

Since JWT tokens remain valid for up to 15 minutes, we 
don't have to regenerate tokens for every call.

Cache tokens for matching access+secret+audience 
for up to 15 seconds.

```
BenchmarkAuthenticateNode/uncached-32         	  270567	      4179 ns/op	    2961 B/op	      33 allocs/op
BenchmarkAuthenticateNode/cached-32           	 7684824	       157.5 ns/op	      48 B/op	       1 allocs/op
```

Reduces internode call allocations a great deal.

cmd/jwt.go

@@ -25,6 +25,7 @@ import (
 
 	jwtgo "github.com/golang-jwt/jwt/v4"
 	jwtreq "github.com/golang-jwt/jwt/v4/request"
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/minio/minio/internal/auth"
 	xjwt "github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
@@ -81,6 +82,35 @@ func authenticateJWTUsersWithCredentials(credentials auth.Credentials, expiresAt
 	return jwt.SignedString([]byte(serverCred.SecretKey))
 }
 
+// cachedAuthenticateNode will cache authenticateNode results for given values up to ttl.
+func cachedAuthenticateNode(ttl time.Duration) func(accessKey, secretKey, audience string) (string, error) {
+	type key struct {
+		accessKey, secretKey, audience string
+	}
+	type value struct {
+		created time.Time
+		res     string
+		err     error
+	}
+	cache, err := lru.NewARC(100)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return authenticateNode
+	}
+	return func(accessKey, secretKey, audience string) (string, error) {
+		k := key{accessKey: accessKey, secretKey: secretKey, audience: audience}
+		v, ok := cache.Get(k)
+		if ok {
+			if val, ok := v.(*value); ok && time.Since(val.created) < ttl {
+				return val.res, val.err
+			}
+		}
+		s, err := authenticateNode(accessKey, secretKey, audience)
+		cache.Add(k, &value{created: time.Now(), res: s, err: err})
+		return s, err
+	}
+}
+
 func authenticateNode(accessKey, secretKey, audience string) (string, error) {
 	claims := xjwt.NewStandardClaims()
 	claims.SetExpiry(UTCNow().Add(defaultInterNodeJWTExpiry))
@@ -152,9 +182,14 @@ func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, bool, error) {
 	return claims, owner, nil
 }
 
-func newAuthToken(audience string) string {
-	cred := globalActiveCred
-	token, err := authenticateNode(cred.AccessKey, cred.SecretKey, audience)
-	logger.CriticalIf(GlobalContext, err)
-	return token
+// newCachedAuthToken returns a token that is cached up to 15 seconds.
+// If globalActiveCred is updated it is reflected at once.
+func newCachedAuthToken() func(audience string) string {
+	fn := cachedAuthenticateNode(15 * time.Second)
+	return func(audience string) string {
+		cred := globalActiveCred
+		token, err := fn(cred.AccessKey, cred.SecretKey, audience)
+		logger.CriticalIf(GlobalContext, err)
+		return token
+	}
 }