Fix precendence bug in S3Select SQL IN clauses (#18708)

Fixes a precendence issue in SQL Select where `a in b and c = 3` was parsed as `a
in (b and c = 3)`.

Fixes #18682

.dockerignore

@@ -1,6 +1,5 @@
 .git
 .github
-docs
 default.etcd
 *.gz
 *.tar.gz
@@ -8,3 +7,11 @@ default.etcd
 *.zip
 browser/node_modules
 node_modules
+docs/debugging/s3-verify/s3-verify
+docs/debugging/xl-meta/xl-meta
+docs/debugging/s3-check-md5/s3-check-md5
+docs/debugging/hash-set/hash-set
+docs/debugging/healing-bin/healing-bin
+docs/debugging/inspect/inspect
+docs/debugging/pprofgoparser/pprofgoparser
+docs/debugging/reorder-disks/reorder-disks

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,6 @@
 ## Community Contribution License
 All community contributions in this pull request are licensed to the project maintainers
-under the terms of the [Apache 2 license] (https://www.apache.org/licenses/LICENSE-2.0). 
+under the terms of the [Apache 2 license](https://www.apache.org/licenses/LICENSE-2.0). 
 By creating this pull request I represent that I have the right to license the 
 contributions to the project maintainers under the Apache 2 license.
 

.github/workflows/go-cross.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/go-fips.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/go-healing.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/go-lint.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/go.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/helm-lint.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/iam-integrations.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/mint.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+      - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -37,7 +38,11 @@ jobs:
 
       - name: build-minio
         run: |
-          TAG="minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+          TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+
+      - name: multipart uploads test
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}"
 
       - name: compress and encrypt
         run: |
@@ -59,7 +64,14 @@ jobs:
              docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
              docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
           done
-          docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
+
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true
+          for volume in $(docker volume ls -q | grep minio); do
+            docker volume rm ${volume} || true
+          done
+
+          docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}
           docker system prune -f || true
           docker volume prune -f || true
           docker volume rm $(docker volume ls -q -f dangling=true) || true

.github/workflows/mint/minio-compress-encrypt.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
   expose:
     - "9000"
@@ -16,10 +16,10 @@ x-minio-common: &minio-common
     MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on"
     MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
   healthcheck:
-    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    test: ["CMD", "mc", "ready", "local"]
-    interval: 30s
+    interval: 5s
-    timeout: 20s
+    timeout: 5s
-    retries: 3
+    retries: 5
 
 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access

.github/workflows/mint/minio-erasure.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" edata{1...4}
   expose:
     - "9000"
@@ -13,10 +13,10 @@ x-minio-common: &minio-common
     MINIO_ROOT_PASSWORD: "minio123"
     MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
   healthcheck:
-    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    test: ["CMD", "mc", "ready", "local"]
-    interval: 30s
+    interval: 5s
-    timeout: 20s
+    timeout: 5s
-    retries: 3
+    retries: 5
 
 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access

.github/workflows/mint/minio-pools.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
   expose:
     - "9000"
@@ -13,10 +13,10 @@ x-minio-common: &minio-common
     MINIO_ROOT_PASSWORD: "minio123"
     MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
   healthcheck:
-    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    test: ["CMD", "mc", "ready", "local"]
-    interval: 30s
+    interval: 5s
-    timeout: 20s
+    timeout: 5s
-    retries: 3
+    retries: 5
 
 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access

.github/workflows/multipart/docker-compose-site1.yaml

@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site1-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site1-minio1:
+    <<: *minio-common
+    hostname: site1-minio1
+    volumes:
+      - site1-data1-1:/data1
+      - site1-data1-2:/data2
+
+  site1-minio2:
+    <<: *minio-common
+    hostname: site1-minio2
+    volumes:
+      - site1-data2-1:/data1
+      - site1-data2-2:/data2
+
+  site1-minio3:
+    <<: *minio-common
+    hostname: site1-minio3
+    volumes:
+      - site1-data3-1:/data1
+      - site1-data3-2:/data2
+
+  site1-minio4:
+    <<: *minio-common
+    hostname: site1-minio4
+    volumes:
+      - site1-data4-1:/data1
+      - site1-data4-2:/data2
+
+  site1-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site1-nginx
+    volumes:
+      - ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9001:9001"
+    depends_on:
+      - site1-minio1
+      - site1-minio2
+      - site1-minio3
+      - site1-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site1-data1-1:
+  site1-data1-2:
+  site1-data2-1:
+  site1-data2-2:
+  site1-data3-1:
+  site1-data3-2:
+  site1-data4-1:
+  site1-data4-2:

.github/workflows/multipart/docker-compose-site2.yaml

@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site2-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site2-minio1:
+    <<: *minio-common
+    hostname: site2-minio1
+    volumes:
+      - site2-data1-1:/data1
+      - site2-data1-2:/data2
+
+  site2-minio2:
+    <<: *minio-common
+    hostname: site2-minio2
+    volumes:
+      - site2-data2-1:/data1
+      - site2-data2-2:/data2
+
+  site2-minio3:
+    <<: *minio-common
+    hostname: site2-minio3
+    volumes:
+      - site2-data3-1:/data1
+      - site2-data3-2:/data2
+
+  site2-minio4:
+    <<: *minio-common
+    hostname: site2-minio4
+    volumes:
+      - site2-data4-1:/data1
+      - site2-data4-2:/data2
+
+  site2-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site2-nginx
+    volumes:
+      - ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9002:9002"
+    depends_on:
+      - site2-minio1
+      - site2-minio2
+      - site2-minio3
+      - site2-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site2-data1-1:
+  site2-data1-2:
+  site2-data2-1:
+  site2-data2-2:
+  site2-data3-1:
+  site2-data3-2:
+  site2-data4-1:
+  site2-data4-2:

.github/workflows/multipart/migrate.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+set -x
+
+## change working directory
+cd .github/workflows/multipart/
+
+function cleanup() {
+	docker-compose -f docker-compose-site1.yaml rm -s -f || true
+	docker-compose -f docker-compose-site2.yaml rm -s -f || true
+	for volume in $(docker volume ls -q | grep minio); do
+		docker volume rm ${volume} || true
+	done
+
+	docker system prune -f || true
+	docker volume prune -f || true
+	docker volume rm $(docker volume ls -q -f dangling=true) || true
+}
+
+cleanup
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+fi
+
+(
+	cd /tmp
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+)
+
+export RELEASE=RELEASE.2023-08-29T23-07-35Z
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+sleep 30s
+
+./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
+./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4
+
+./mc ready site1/
+./mc ready site2/
+
+./mc admin replicate add site1 site2
+./mc mb site1/testbucket/
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+s3-check-md5 -h
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+## we do not need to fail here, since we are going to test
+## upgrading to master, healing and being able to recover
+## the last version.
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads ${failed_count_site1}"
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads ${failed_count_site2}"
+fi
+
+export RELEASE=${1}
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+./mc ready site1/
+./mc ready site2/
+
+for i in $(seq 1 10); do
+	# mc admin heal -r --remove when used against a LB endpoint
+	# behaves flaky, let this run 10 times before giving up
+	./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null
+	./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
+done
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+cleanup
+
+## change working directory
+cd ../../../

.github/workflows/multipart/nginx-site1.conf

@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site1-minio1:9000;
+        server site1-minio2:9000;
+        server site1-minio3:9000;
+        server site1-minio4:9000;
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

.github/workflows/multipart/nginx-site2.conf

@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site2-minio1:9000;
+        server site2-minio2:9000;
+        server site2-minio3:9000;
+        server site2-minio4:9000;
+    }
+
+    server {
+        listen       9002;
+        listen  [::]:9002;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

.github/workflows/replication.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -35,6 +36,12 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-decom
 
+      - name: Test Config File
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-configfile
+
       - name: Test Replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

.github/workflows/root-disable.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/run-mint.sh

@@ -16,7 +16,7 @@ docker volume rm $(docker volume ls -f dangling=true) || true
 cd .github/workflows/mint
 
 docker-compose -f minio-${MODE}.yaml up -d
-sleep 5m
+sleep 30s
 
 docker system prune -f || true
 docker volume prune -f || true

.github/workflows/shfmt.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 permissions:
   contents: read

.github/workflows/upgrade-ci-cd.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/vulncheck.yml

@@ -3,6 +3,7 @@ on:
   pull_request:
     branches:
       - master
+
   push:
     branches:
       - master
@@ -20,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21.0
+          go-version: 1.21.5
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.gitignore

@@ -32,6 +32,8 @@ minio.RELEASE*
 mc
 nancy
 inspects/*
+.bin/
+*.gz
 docs/debugging/s3-verify/s3-verify
 docs/debugging/xl-meta/xl-meta
 docs/debugging/s3-check-md5/s3-check-md5
@@ -39,5 +41,5 @@ docs/debugging/hash-set/hash-set
 docs/debugging/healing-bin/healing-bin
 docs/debugging/inspect/inspect
 docs/debugging/pprofgoparser/pprofgoparser
-.bin/
+docs/debugging/reorder-disks/reorder-disks
-*.gz
+docs/debugging/populate-hard-links/populate-hardlinks

.golangci.yml

@@ -13,7 +13,6 @@ linters:
   enable:
     - durationcheck
     - gocritic
-    - gofmt
     - gofumpt
     - goimports
     - gomodguard
@@ -30,5 +29,8 @@ linters:
 issues:
   exclude-use-default: false
   exclude:
+    - "empty-block:"
+    - "unused-parameter:"
+    - "dot-imports:"
     - should have a package comment
     - error strings should not be capitalized or end with punctuation or a newline

Dockerfile

@@ -1,8 +1,6 @@
 FROM minio/minio:latest
 
-ENV PATH=/opt/bin:$PATH
+COPY ./minio /usr/bin/minio
-
-COPY ./minio /opt/bin/minio
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]

Dockerfile.hotfix

@@ -1,4 +1,31 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
+FROM golang:1.21-alpine as build
+
+ARG TARGETARCH
+ARG RELEASE
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+# Install curl and minisign
+RUN apk add -U --no-cache ca-certificates && \
+    apk add -U --no-cache curl && \
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
+
+# Download minio binary and signature file
+RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
+    curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    chmod +x /go/bin/minio
+
+# Download mc binary and signature file
+RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    chmod +x /go/bin/mc
+
+# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
+RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
+    minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
+
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest
 
 ARG RELEASE
 
@@ -17,33 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
     MINIO_CONFIG_ENV_FILE=config.env \
-    PATH=/opt/bin:$PATH
+    MC_CONFIG_DIR=/tmp/.mc
+
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/mc /usr/bin/mc
 
-COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE
-
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
-RUN \
-     microdnf clean all && \
-     microdnf update --nodocs && \
-     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
-     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
-     microdnf clean all && \
-     chmod +x /opt/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     /usr/bin/verify-minio.sh && \
-     microdnf clean all
 
 EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
 VOLUME ["/data"]
 
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 CMD ["minio"]

Dockerfile.release

@@ -1,6 +1,31 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
+FROM golang:1.21-alpine as build
 
 ARG TARGETARCH
+ARG RELEASE
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+# Install curl and minisign
+RUN apk add -U --no-cache ca-certificates && \
+    apk add -U --no-cache curl && \
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
+
+# Download minio binary and signature file
+RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    chmod +x /go/bin/minio
+
+# Download mc binary and signature file
+RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    chmod +x /go/bin/mc
+
+# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
+RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
+    minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
+
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest
 
 ARG RELEASE
 
@@ -19,35 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
     MINIO_CONFIG_ENV_FILE=config.env \
-    PATH=/opt/bin:$PATH
+    MC_CONFIG_DIR=/tmp/.mc
+
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/mc /usr/bin/mc
 
-COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE
-
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
-RUN \
-     microdnf clean all && \
-     microdnf update --nodocs && \
-     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
-     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
-     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /opt/bin/mc && \
-     microdnf clean all && \
-     chmod +x /opt/bin/minio && \
-     chmod +x /opt/bin/mc && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     /usr/bin/verify-minio.sh && \
-     microdnf clean all
 
 EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
 VOLUME ["/data"]
 
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 CMD ["minio"]

Dockerfile.release.fips

@@ -1,6 +1,25 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
+FROM golang:1.21-alpine as build
 
 ARG TARGETARCH
+ARG RELEASE
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+# Install curl and minisign
+RUN apk add -U --no-cache ca-certificates && \
+    apk add -U --no-cache curl && \
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
+
+# Download minio binary and signature file
+RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
+    chmod +x /go/bin/minio
+
+# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
+RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
+
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest
 
 ARG RELEASE
 
@@ -18,34 +37,17 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
-    MINIO_CONFIG_ENV_FILE=config.env \
+    MINIO_CONFIG_ENV_FILE=config.env
-    PATH=/opt/bin:$PATH
+
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/minio /usr/bin/minio
 
-COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE
-
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
-RUN \
-     microdnf clean all && \
-     microdnf update --nodocs && \
-     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
-     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /opt/bin/minio.minisig && \
-     microdnf clean all && \
-     chmod +x /opt/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     /usr/bin/verify-minio.sh && \
-     microdnf clean all
 
 EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
 VOLUME ["/data"]
 
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 CMD ["minio"]

Dockerfile.release.old_cpu

@@ -0,0 +1,61 @@
+FROM golang:1.21-alpine as build
+
+ARG TARGETARCH
+ARG RELEASE
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+# Install curl and minisign
+RUN apk add -U --no-cache ca-certificates && \
+    apk add -U --no-cache curl && \
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
+
+# Download minio binary and signature file
+RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    chmod +x /go/bin/minio
+
+# Download mc binary and signature file
+RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    chmod +x /go/bin/mc
+
+# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
+RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
+    minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
+
+FROM registry.access.redhat.com/ubi8/ubi-micro:latest
+
+ARG RELEASE
+
+LABEL name="MinIO" \
+      vendor="MinIO Inc <dev@min.io>" \
+      maintainer="MinIO Inc <dev@min.io>" \
+      version="${RELEASE}" \
+      release="${RELEASE}" \
+      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
+      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
+
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
+    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
+    MINIO_CONFIG_ENV_FILE=config.env \
+    MC_CONFIG_DIR=/tmp/.mc
+
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/mc /usr/bin/mc
+
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+VOLUME ["/data"]
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+CMD ["minio"]

Makefile

@@ -6,9 +6,8 @@ GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)
 
 VERSION ?= $(shell git describe --tags)
-TAG ?= "minio/minio:$(VERSION)"
+TAG ?= "quay.io/minio/minio:$(VERSION)"
 
-GOLANGCI_VERSION = v1.51.2
 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
 GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
 
@@ -23,8 +22,8 @@ help: ## print this help
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION)
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@6ac204f0b4d48d17ab4fa442134c7fba13127a4e
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio
@@ -45,22 +44,25 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
 
 check: test
-test: verifiers build ## builds minio, runs linters, tests
+test: verifiers build build-debugging ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
 
-test-root-disable: install
+test-root-disable: install-race
 	@echo "Running minio root lockdown tests"
 	@env bash $(PWD)/buildscripts/disable-root.sh
 
-test-decom: install
+test-decom: install-race
 	@echo "Running minio decom tests"
 	@env bash $(PWD)/docs/distributed/decom.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
 	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
 
-test-upgrade: build
+test-configfile: install-race
+	@env bash $(PWD)/docs/distributed/distributed-from-config-file.sh
+
+test-upgrade: install-race
 	@echo "Running minio upgrade tests"
 	@(env bash $(PWD)/buildscripts/minio-upgrade.sh)
 
@@ -86,18 +88,18 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
 
-test-replication: install test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
+test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 
-test-site-replication-ldap: install ## verify automatic site replication
+test-site-replication-ldap: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with LDAP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
 
-test-site-replication-oidc: install ## verify automatic site replication
+test-site-replication-oidc: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with OIDC)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)
 
-test-site-replication-minio: install ## verify automatic site replication
+test-site-replication-minio: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
 
@@ -128,6 +130,9 @@ verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)
 
+build-debugging:
+	@(env bash $(PWD)/docs/debugging/build.sh)
+
 build: checks ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"
 	@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@@ -138,14 +143,23 @@ hotfix-vars:
 	$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
 	$(eval TAG := "minio/minio:$(VERSION)")
 
-hotfix: hotfix-vars install ## builds minio binary with hotfix tags
+hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
-	@mv -f ./minio ./minio.$(VERSION)
+	@wget -q -c https://github.com/minio/pkger/releases/download/v2.2.0/pkger_2.2.0_linux_amd64.deb
-	@minisign -qQSm ./minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
+	@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.0/linux-systemd/distributed/minio.service
-	@sha256sum < ./minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio.$(VERSION).sha256sum
+	@sudo apt install ./pkger_2.2.0_linux_amd64.deb --yes
+	@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
+	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio
+	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)
+	@minisign -qQSm minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
+	@sha256sum < minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION).sha256sum
+	@cp -af minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)* minio-release/$(GOOS)-$(GOARCH)/archive/
+	@pkger -r $(VERSION) --ignore
 
 hotfix-push: hotfix
-	@scp -q -r minio.$(VERSION)* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
-	@scp -q -r minio.$(VERSION)* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
+	@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
+	@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
 	@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.$(VERSION)"
 
 docker-hotfix-push: docker-hotfix
@@ -159,6 +173,12 @@ docker: build ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -q --no-cache -t $(TAG) . -f Dockerfile
 
+install-race: checks ## builds minio to $(PWD)
+	@echo "Building minio binary to './minio'"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
+	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
+
 install: build ## builds minio and installs it to $GOPATH/bin.
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
@@ -174,3 +194,6 @@ clean: ## cleanup all generated assets
 	@rm -rvf build
 	@rm -rvf release
 	@rm -rvf .verify*
+	@rm -rvf minio-release
+	@rm -rvf minio.RELEASE*.hotfix.*
+	@rm -rvf pkger_*.deb

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
 	export CGO_ENABLED=0
 
 	## List of architectures and OS to test coss compilation.
-	SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips"
+	SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
 }
 
 function _build() {

buildscripts/disable-root.sh

@@ -56,6 +56,8 @@ done
 
 set +e
 
+sleep 10
+
 ./mc ls minioadm/
 if [ $? -ne 0 ]; then
 	echo "listing failed, 'minioadmin' should be enabled"

buildscripts/rewrite-old-new.sh

@@ -87,8 +87,8 @@ function verify_rewrite() {
 		exit 1
 	fi
 
-	go build ./docs/debugging/s3-check-md5/
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-versions \
 		-access-key minio \
@@ -113,7 +113,7 @@ function verify_rewrite() {
 	go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
 	sleep 1
 
-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-versions \
 		-access-key minio \

buildscripts/unaligned-healing.sh

@@ -82,8 +82,8 @@ function start_minio_16drive() {
 	rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
 	sleep 10
 
-	go build ./docs/debugging/s3-check-md5/
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-access-key minio \
 		-secret-key minio123 \
@@ -111,7 +111,7 @@ function start_minio_16drive() {
 		exit 1
 	fi
 
-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-access-key minio \
 		-secret-key minio123 \
@@ -134,7 +134,7 @@ function start_minio_16drive() {
 
 	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
 
-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-access-key minio \
 		-secret-key minio123 \

buildscripts/upgrade-tests/compose.yml

@@ -9,11 +9,6 @@ x-minio-common: &minio-common
   expose:
     - "9000"
     - "9001"
-  healthcheck:
-    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-    interval: 30s
-    timeout: 20s
-    retries: 3
 
 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access

cmd/acl-handlers.go

@@ -25,7 +25,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // Data types used for returning dummy access control

cmd/admin-bucket-handlers.go

@@ -41,8 +41,7 @@ import (
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
-	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 const (
@@ -58,7 +57,7 @@ const (
 func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketQuotaAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -95,7 +94,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		Quota:     data,
 		UpdatedAt: updatedAt,
 	}
-	if quotaConfig.Quota == 0 {
+	if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
 		bucketMeta.Quota = nil
 	}
 
@@ -110,7 +109,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetBucketQuotaAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -148,7 +147,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	update := r.Form.Get("update") == "true"
 
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketTargetAction)
 	if objectAPI == nil {
 		return
 	}
@@ -291,7 +290,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 	arnType := vars["type"]
 
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetBucketTargetAction)
 	if objectAPI == nil {
 		return
 	}
@@ -325,7 +324,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 	arn := vars["arn"]
 
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketTargetAction)
 	if objectAPI == nil {
 		return
 	}
@@ -365,7 +364,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 
 	bucket := pathClean(r.Form.Get("bucket"))
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportBucketMetadataAction)
 	if objectAPI == nil {
 		return
 	}
@@ -393,7 +392,8 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	// of bucket metadata
 	zipWriter := zip.NewWriter(w)
 	defer zipWriter.Close()
-	rawDataFn := func(r io.Reader, filename string, sz int) error {
+
+	rawDataFn := func(r io.Reader, filename string, sz int) {
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
 			name:    filename,
 			size:    int64(sz),
@@ -402,20 +402,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 			isDir:   false,
 			sys:     nil,
 		})
-		if zerr != nil {
+		if zerr == nil {
-			logger.LogIf(ctx, zerr)
-			return nil
-		}
 			header.Method = zip.Deflate
 			zwriter, zerr := zipWriter.CreateHeader(header)
-		if zerr != nil {
+			if zerr == nil {
-			logger.LogIf(ctx, zerr)
+				io.Copy(zwriter, r)
-			return nil
 			}
-		if _, err := io.Copy(zwriter, r); err != nil {
-			logger.LogIf(ctx, err)
 		}
-		return nil
 	}
 
 	cfgFiles := []string{
@@ -447,10 +440,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
 			case bucketLifecycleConfig:
 				config, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
 				if err != nil {
@@ -466,10 +456,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
 			case bucketQuotaConfigFile:
 				config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
 				if err != nil {
@@ -484,10 +471,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
 			case bucketSSEConfig:
 				config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
 				if err != nil {
@@ -502,10 +486,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
 			case bucketTaggingConfig:
 				config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
 				if err != nil {
@@ -520,10 +501,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
 			case objectLockConfig:
 				config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
 				if err != nil {
@@ -539,10 +517,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
 			case bucketVersioningConfig:
 				config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
 				if err != nil {
@@ -558,10 +533,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
 			case bucketReplicationConfig:
 				config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
 				if err != nil {
@@ -576,11 +548,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
 			case bucketTargetsFile:
 				config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
 				if err != nil {
@@ -596,10 +564,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
 			}
 		}
 	}
@@ -647,7 +612,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 	ctx := r.Context()
 
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ImportBucketMetadataAction)
 	if objectAPI == nil {
 		return
 	}
@@ -853,7 +818,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
+			bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)
 				continue
@@ -1021,7 +986,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ReplicationDiff)
 	if objectAPI == nil {
 		return
 	}
@@ -1083,7 +1048,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ReplicationDiff)
 	if objectAPI == nil {
 		return
 	}

cmd/admin-handler-utils.go

@@ -27,14 +27,14 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // validateAdminReq will validate request against and return whether it is allowed.
 // If any of the supplied actions are allowed it will be successful.
 // If nil ObjectLayer is returned, the operation is not permitted.
 // When nil ObjectLayer has been returned an error has always been sent to w.
-func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
+func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...policy.AdminAction) (ObjectLayer, auth.Credentials) {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil {
@@ -78,7 +78,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 
 	var apiErr APIError
 	switch e := err.(type) {
-	case iampolicy.Error:
+	case policy.Error:
 		apiErr = APIError{
 			Code:           "XMinioMalformedIAMPolicy",
 			Description:    e.Error(),

cmd/admin-handlers-config-kv.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/etcd"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
@@ -38,14 +37,14 @@ import (
 	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -149,7 +148,7 @@ type setConfigResult struct {
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -242,7 +241,7 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -288,7 +287,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -321,7 +320,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -367,7 +366,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -405,7 +404,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -430,7 +429,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -482,7 +481,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -500,8 +499,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 			switch hkv.Key {
 			case config.EtcdSubSys:
 				off = !etcd.Enabled(item.Config)
-			case config.CacheSubSys:
-				off = !cache.Enabled(item.Config)
 			case config.StorageClassSubSys:
 				off = !storageclass.Enabled(item.Config)
 			case config.PolicyPluginSubSys:

cmd/admin-handlers-idp-config.go

@@ -33,12 +33,12 @@ import (
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/ldap"
-	"github.com/minio/pkg/ldap"
+	"github.com/minio/pkg/v2/policy"
 )
 
 func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -121,7 +121,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
 
 	// IDP config is not dynamic. Sanity check.
 	if dynamic {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), "", r.URL)
 		return
 	}
 
@@ -220,7 +220,7 @@ func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *ht
 func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -273,7 +273,7 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
 func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -332,7 +332,7 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}

cmd/admin-handlers-idp-ldap.go

@@ -19,13 +19,17 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"net/http"
+	"strings"
 
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
@@ -50,7 +54,7 @@ func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r
 	// Check authorization.
 
 	objectAPI, cred := validateAdminReq(ctx, w, r,
-		iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
+		policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -96,7 +100,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 
 	// Check authorization.
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction)
 	if objectAPI == nil {
 		return
 	}
@@ -175,3 +179,267 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 
 	writeSuccessResponseJSON(w, encryptedData)
 }
+
+// AddServiceAccountLDAP adds a new service account for provided LDAP username or DN
+//
+// PUT /minio/admin/v3/idp/ldap/add-service-account
+func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.Request) {
+	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
+	if APIError.Code != "" {
+		writeErrorResponseJSON(ctx, w, APIError, r.URL)
+		return
+	}
+
+	// fail if ldap is not enabled
+	if !globalIAMSys.LDAPConfig.Enabled() {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("LDAP not enabled")), r.URL)
+	}
+
+	// Find the user for the request sender (as it may be sent via a service
+	// account or STS account):
+	requestorUser := cred.AccessKey
+	requestorParentUser := cred.AccessKey
+	requestorGroups := cred.Groups
+	requestorIsDerivedCredential := false
+	if cred.IsServiceAccount() || cred.IsTemp() {
+		requestorParentUser = cred.ParentUser
+		requestorIsDerivedCredential = true
+	}
+
+	// Check if we are creating svc account for request sender.
+	isSvcAccForRequestor := false
+	if targetUser == requestorUser || targetUser == requestorParentUser {
+		isSvcAccForRequestor = true
+	}
+
+	var (
+		targetGroups []string
+		err          error
+	)
+
+	// If we are creating svc account for request sender, ensure
+	// that targetUser is a real user (i.e. not derived
+	// credentials).
+	if isSvcAccForRequestor {
+		if requestorIsDerivedCredential {
+			if requestorParentUser == "" {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
+					errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL)
+				return
+			}
+			targetUser = requestorParentUser
+		}
+		targetGroups = requestorGroups
+
+		// Deny if the target user is not LDAP
+		isLDAP, err := globalIAMSys.LDAPConfig.DoesUsernameExist(targetUser)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if isLDAP == "" {
+			err := errors.New("Specified user does not exist on LDAP server")
+			APIErr := errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err)
+			writeErrorResponseJSON(ctx, w, APIErr, r.URL)
+			return
+		}
+
+		// In case of LDAP/OIDC we need to set `opts.claims` to ensure
+		// it is associated with the LDAP/OIDC user properly.
+		for k, v := range cred.Claims {
+			if k == expClaim {
+				continue
+			}
+			opts.claims[k] = v
+		}
+	} else {
+		isDN := globalIAMSys.LDAPConfig.IsLDAPUserDN(targetUser)
+
+		opts.claims[ldapUserN] = targetUser // simple username
+		targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
+		if err != nil {
+			// if not found, check if DN
+			if strings.Contains(err.Error(), "not found") && isDN {
+				// warn user that DNs are not allowed
+				err = fmt.Errorf("Must use short username to add service account. %w", err)
+			}
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		opts.claims[ldapUser] = targetUser // DN
+	}
+
+	newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	createResp := madmin.AddServiceAccountResp{
+		Credentials: madmin.Credentials{
+			AccessKey:  newCred.AccessKey,
+			SecretKey:  newCred.SecretKey,
+			Expiration: newCred.Expiration,
+		},
+	}
+
+	data, err := json.Marshal(createResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+
+	// Call hook for cluster-replication if the service account is not for a
+	// root user.
+	if newCred.ParentUser != globalActiveCred.AccessKey {
+		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Create: &madmin.SRSvcAccCreate{
+					Parent:        newCred.ParentUser,
+					AccessKey:     newCred.AccessKey,
+					SecretKey:     newCred.SecretKey,
+					Groups:        newCred.Groups,
+					Name:          newCred.Name,
+					Description:   newCred.Description,
+					Claims:        opts.claims,
+					SessionPolicy: createReq.Policy,
+					Status:        auth.AccountOn,
+					Expiration:    createReq.Expiration,
+				},
+			},
+			UpdatedAt: updatedAt,
+		}))
+	}
+}
+
+// ListAccessKeysLDAP - GET /minio/admin/v3/idp/ldap/list-access-keys
+func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	userDN := r.Form.Get("userDN")
+
+	// If listing is requested for a specific user (who is not the request
+	// sender), check that the user has permissions.
+	if userDN != "" && userDN != cred.ParentUser {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListServiceAccountsAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	} else {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListServiceAccountsAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+			DenyOnly:        true,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+		userDN = cred.AccessKey
+		if cred.ParentUser != "" {
+			userDN = cred.ParentUser
+		}
+	}
+
+	targetAccount, err := globalIAMSys.LDAPConfig.DoesUsernameExist(userDN)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	} else if userDN == "" {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
+		return
+	}
+
+	listType := r.Form.Get("listType")
+	if listType != "sts-only" && listType != "svcacc-only" && listType != "" {
+		// default to both
+		listType = ""
+	}
+
+	var serviceAccounts []auth.Credentials
+	var stsKeys []auth.Credentials
+
+	if listType == "" || listType == "sts-only" {
+		stsKeys, err = globalIAMSys.ListSTSAccounts(ctx, targetAccount)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+	if listType == "" || listType == "svcacc-only" {
+		serviceAccounts, err = globalIAMSys.ListServiceAccounts(ctx, targetAccount)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	var serviceAccountList []madmin.ServiceAccountInfo
+	var stsKeyList []madmin.ServiceAccountInfo
+
+	for _, svc := range serviceAccounts {
+		expiryTime := svc.Expiration
+		serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{
+			AccessKey:  svc.AccessKey,
+			Expiration: &expiryTime,
+		})
+	}
+	for _, sts := range stsKeys {
+		expiryTime := sts.Expiration
+		stsKeyList = append(stsKeyList, madmin.ServiceAccountInfo{
+			AccessKey:  sts.AccessKey,
+			Expiration: &expiryTime,
+		})
+	}
+
+	listResp := madmin.ListAccessKeysLDAPResp{
+		ServiceAccounts: serviceAccountList,
+		STSKeys:         stsKeyList,
+	}
+
+	data, err := json.Marshal(listResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}

cmd/admin-handlers-pools.go

@@ -22,11 +22,12 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 var (
@@ -37,7 +38,7 @@ var (
 func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -66,17 +67,29 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
 
 	pools := strings.Split(v, ",")
 	poolIndices := make([]int, 0, len(pools))
 
 	for _, pool := range pools {
-		idx := globalEndpoints.GetPoolIdx(pool)
+		var idx int
+		if byID {
+			var err error
+			idx, err = strconv.Atoi(pool)
+			if err != nil {
+				// We didn't find any matching pools, invalid input
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+				return
+			}
+		} else {
+			idx = globalEndpoints.GetPoolIdx(pool)
 			if idx == -1 {
 				// We didn't find any matching pools, invalid input
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
 				return
 			}
+		}
 		var pool *erasureSets
 		for pidx := range z.serverPools {
 			if pidx == idx {
@@ -113,7 +126,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -132,8 +145,17 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
+	idx := -1
+
+	if byID {
+		if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
+			idx = i
+		}
+	} else {
+		idx = globalEndpoints.GetPoolIdx(v)
+	}
 
-	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
 		// We didn't find any matching pools, invalid input
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
@@ -159,7 +181,7 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -178,8 +200,17 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
+	idx := -1
+
+	if byID {
+		if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
+			idx = i
+		}
+	} else {
+		idx = globalEndpoints.GetPoolIdx(v)
+	}
 
-	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
 		apiErr := toAdminAPIErr(ctx, errInvalidArgument)
 		apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)
@@ -200,7 +231,7 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -233,7 +264,7 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -304,7 +335,7 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -344,7 +375,7 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
 	if objectAPI == nil {
 		return
 	}

cmd/admin-handlers-site-replication.go

@@ -31,17 +31,17 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
-	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
 func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
 	if objectAPI == nil {
 		return
 	}
@@ -52,7 +52,8 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
+	opts := getSRAddOptions(r)
+	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites, opts)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -68,6 +69,11 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseJSON(w, body)
 }
 
+func getSRAddOptions(r *http.Request) (opts madmin.SRAddOptions) {
+	opts.ReplicateILMExpiry = r.Form.Get("replicateILMExpiry") == "true"
+	return
+}
+
 // SRPeerJoin - PUT /minio/admin/v3/site-replication/join
 //
 // used internally to tell current cluster to enable SR with
@@ -75,7 +81,7 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
 	if objectAPI == nil {
 		return
 	}
@@ -97,7 +103,7 @@ func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
 	if objectAPI == nil {
 		return
 	}
@@ -144,7 +150,7 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
 	if objectAPI == nil {
 		return
 	}
@@ -163,7 +169,7 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 		if item.Policy == nil {
 			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
 		} else {
-			policy, perr := iampolicy.ParseConfig(bytes.NewReader(item.Policy))
+			policy, perr := policy.ParseConfig(bytes.NewReader(item.Policy))
 			if perr != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
 				return
@@ -192,11 +198,11 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 	}
 }
 
-// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
+// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/peer/bucket-meta
 func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
 	if objectAPI == nil {
 		return
 	}
@@ -220,7 +226,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		if item.Policy == nil {
 			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
 		} else {
-			bktPolicy, berr := policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
+			bktPolicy, berr := policy.ParseBucketPolicyConfig(bytes.NewReader(item.Policy), item.Bucket)
 			if berr != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
 				return
@@ -253,6 +259,8 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
 	case madmin.SRBucketMetaTypeSSEConfig:
 		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
+	case madmin.SRBucketMetaLCConfig:
+		err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -265,7 +273,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
 	if objectAPI == nil {
 		return
 	}
@@ -285,7 +293,7 @@ func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
 	if objectAPI == nil {
 		return
 	}
@@ -322,7 +330,7 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
 func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
 	if objectAPI == nil {
 		return
 	}
@@ -334,6 +342,7 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
 		opts.Users = true
 		opts.Policies = true
 		opts.Groups = true
+		opts.ILMExpiryRules = true
 	}
 	info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI, opts)
 	if err != nil {
@@ -351,7 +360,7 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
 func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
 	if objectAPI == nil {
 		return
 	}
@@ -373,7 +382,7 @@ func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http
 func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
 	if objectAPI == nil {
 		return
 	}
@@ -383,7 +392,9 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site)
+
+	opts := getSREditOptions(r)
+	status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site, opts)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -398,13 +409,19 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, body)
 }
 
+func getSREditOptions(r *http.Request) (opts madmin.SREditOptions) {
+	opts.DisableILMExpiryReplication = r.Form.Get("disableILMExpiryReplication") == "true"
+	opts.EnableILMExpiryReplication = r.Form.Get("enableILMExpiryReplication") == "true"
+	return
+}
+
 // SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit
 //
 // used internally to tell current cluster to update endpoint for peer
 func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
 	if objectAPI == nil {
 		return
 	}
@@ -422,12 +439,37 @@ func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// SRStateEdit - PUT /minio/admin/v3/site-replication/state/edit
+//
+// used internally to tell current cluster to update site replication state
+func (a adminAPIHandlers) SRStateEdit(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var state madmin.SRStateEditReq
+	if err := parseJSONBody(ctx, r.Body, &state, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if err := globalSiteReplicationSys.PeerStateEditReq(ctx, state); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+}
+
 func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
 	q := r.Form
 	opts.Buckets = q.Get("buckets") == "true"
 	opts.Policies = q.Get("policies") == "true"
 	opts.Groups = q.Get("groups") == "true"
 	opts.Users = q.Get("users") == "true"
+	opts.ILMExpiryRules = q.Get("ilm-expiry-rules") == "true"
+	opts.PeerState = q.Get("peer-state") == "true"
 	opts.Entity = madmin.GetSREntityType(q.Get("entity"))
 	opts.EntityValue = q.Get("entityvalue")
 	opts.ShowDeleted = q.Get("showDeleted") == "true"
@@ -439,7 +481,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
 func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
 	if objectAPI == nil {
 		return
 	}
@@ -470,7 +512,7 @@ func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.R
 func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
 	if objectAPI == nil {
 		return
 	}
@@ -492,7 +534,7 @@ func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationResyncAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationResyncAction)
 	if objectAPI == nil {
 		return
 	}
@@ -538,7 +580,7 @@ func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.
 
 	connectTime := time.Now()
 	for {
-		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
 		atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n))
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)

cmd/admin-handlers-users-race_test.go

@@ -32,7 +32,7 @@ import (
 
 	"github.com/minio/madmin-go/v3"
 	minio "github.com/minio/minio-go/v7"
-	"github.com/minio/pkg/sync/errgroup"
+	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
 func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {

cmd/admin-handlers-users.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -34,14 +35,14 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.DeleteUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -85,7 +86,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -119,7 +120,7 @@ func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -186,10 +187,10 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 		checkDenyOnly = true
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.GetUserAdminAction,
+		Action:          policy.GetUserAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -218,7 +219,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -288,7 +289,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -315,7 +316,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListGroupsAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -339,7 +340,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -382,7 +383,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, policy.EnableUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -470,10 +471,10 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		checkDenyOnly = true
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.CreateUserAdminAction,
+		Action:          policy.CreateUserAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -544,34 +545,41 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	args := policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.ListTemporaryAccountsAdminAction,
+		Action:          policy.ListTemporaryAccountsAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
-	}) {
+	}
+
+	if !globalIAMSys.IsAllowed(args) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 		return
 	}
 
-	stsAccount, policy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey)
+	stsAccount, sessionPolicy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	var stsAccountPolicy iampolicy.Policy
+	var stsAccountPolicy policy.Policy
 
-	if policy != nil {
+	if sessionPolicy != nil {
-		stsAccountPolicy = *policy
+		stsAccountPolicy = *sessionPolicy
 	} else {
-		policiesNames, err := globalIAMSys.PolicyDBGet(stsAccount.ParentUser, false)
+		policiesNames, err := globalIAMSys.PolicyDBGet(stsAccount.ParentUser, stsAccount.Groups...)
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
+		if len(policiesNames) == 0 {
+			policySet, _ := args.GetPolicies(iamPolicyClaimNameOpenID())
+			policiesNames = policySet.ToSlice()
+		}
+
 		stsAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
 	}
 
@@ -584,7 +592,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 	infoResp := madmin.TemporaryAccountInfoResp{
 		ParentUser:    stsAccount.ParentUser,
 		AccountStatus: stsAccount.Status,
-		ImpliedPolicy: policy == nil,
+		ImpliedPolicy: sessionPolicy == nil,
 		Policy:        string(policyJSON),
 		Expiration:    &stsAccount.Expiration,
 	}
@@ -606,72 +614,17 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 
 // AddServiceAccount - PUT /minio/admin/v3/add-service-account
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
+	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
-
+	if APIError.Code != "" {
-	// Get current object layer instance.
+		writeErrorResponseJSON(ctx, w, APIError, r.URL)
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
-	if s3Err != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
-		return
-	}
-
-	password := cred.SecretKey
-	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
-		return
-	}
-
-	var createReq madmin.AddServiceAccountReq
-	if err = json.Unmarshal(reqBytes, &createReq); err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
-		return
-	}
-
-	// service account access key cannot have space characters beginning and end of the string.
-	if hasSpaceBE(createReq.AccessKey) {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
-		return
-	}
-
-	if err := createReq.Validate(); err != nil {
-		// Since this validation would happen client side as well, we only send
-		// a generic error message here.
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
 		return
 	}
 
 	var (
-		targetUser   string
 		targetGroups []string
+		err          error
 	)
 
-	// If the request did not set a TargetUser, the service account is
-	// created for the request sender.
-	targetUser = createReq.TargetUser
-	if targetUser == "" {
-		targetUser = cred.AccessKey
-	}
-
-	description := createReq.Description
-	if description == "" {
-		description = createReq.Comment
-	}
-	opts := newServiceAccountOpts{
-		accessKey:   createReq.AccessKey,
-		secretKey:   createReq.SecretKey,
-		name:        createReq.Name,
-		description: description,
-		expiration:  createReq.Expiration,
-		claims:      make(map[string]interface{}),
-	}
-
 	// Find the user for the request sender (as it may be sent via a service
 	// account or STS account):
 	requestorUser := cred.AccessKey
@@ -705,23 +658,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	// that targetUser is a real user (i.e. not derived
 	// credentials).
 	if isSvcAccForRequestor {
-		// Check if adding service account is explicitly denied.
-		//
-		// This allows turning off service accounts for request sender,
-		// if there is no deny statement this call is implicitly enabled.
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     requestorUser,
-			Groups:          requestorGroups,
-			Action:          iampolicy.CreateServiceAccountAdminAction,
-			ConditionValues: getConditionValues(r, "", cred),
-			IsOwner:         owner,
-			Claims:          cred.Claims,
-			DenyOnly:        true,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
-
 		if requestorIsDerivedCredential {
 			if requestorParentUser == "" {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
@@ -740,24 +676,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 			}
 			opts.claims[k] = v
 		}
-	} else {
+	} else if globalIAMSys.LDAPConfig.Enabled() {
-		// Need permission if we are creating a service account for a
-		// user <> to the request sender
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     requestorUser,
-			Groups:          requestorGroups,
-			Action:          iampolicy.CreateServiceAccountAdminAction,
-			ConditionValues: getConditionValues(r, "", cred),
-			IsOwner:         owner,
-			Claims:          cred.Claims,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
-
 		// In case of LDAP we need to resolve the targetUser to a DN and
 		// query their groups:
-		if globalIAMSys.LDAPConfig.Enabled() {
 		opts.claims[ldapUserN] = targetUser // simple username
 		targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
 		if err != nil {
@@ -765,7 +686,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 			return
 		}
 		opts.claims[ldapUser] = targetUser // username DN
-		}
 
 		// NOTE: if not using LDAP, then internal IDP or open ID is
 		// being used - in the former, group info is enforced when
@@ -773,16 +693,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// latter, a group notion is not supported.
 	}
 
-	var sp *iampolicy.Policy
-	if len(createReq.Policy) > 0 {
-		sp, err = iampolicy.ParseConfig(bytes.NewReader(createReq.Policy))
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-	}
-
-	opts.sessionPolicy = sp
 	newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -803,7 +713,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	encryptedData, err := madmin.EncryptData(password, data)
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -864,10 +774,10 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.UpdateServiceAccountAdminAction,
+		Action:          policy.UpdateServiceAccountAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -903,13 +813,16 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	var sp *iampolicy.Policy
+	var sp *policy.Policy
 	if len(updateReq.NewPolicy) > 0 {
-		sp, err = iampolicy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
+		sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
+		if sp.Version == "" && len(sp.Statements) == 0 {
+			sp = nil
+		}
 	}
 	opts := updateServiceAccountOpts{
 		secretKey:     updateReq.NewSecretKey,
@@ -970,16 +883,16 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	svcAccount, policy, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
+	svcAccount, sessionPolicy, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.ListServiceAccountsAdminAction,
+		Action:          policy.ListServiceAccountsAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -995,12 +908,15 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		}
 	}
 
-	var svcAccountPolicy iampolicy.Policy
+	// if session policy is nil or empty, then it is implied policy
+	impliedPolicy := sessionPolicy == nil || (sessionPolicy.Version == "" && len(sessionPolicy.Statements) == 0)
 
-	if policy != nil {
+	var svcAccountPolicy policy.Policy
-		svcAccountPolicy = *policy
+
+	if !impliedPolicy {
+		svcAccountPolicy = *sessionPolicy
 	} else {
-		policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, false)
+		policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, svcAccount.Groups...)
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1024,7 +940,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		Name:          svcAccount.Name,
 		Description:   svcAccount.Description,
 		AccountStatus: svcAccount.Status,
-		ImpliedPolicy: policy == nil,
+		ImpliedPolicy: impliedPolicy,
 		Policy:        string(policyJSON),
 		Expiration:    expiration,
 	}
@@ -1067,10 +983,10 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 	// sender), check that the user has permissions.
 	user := r.Form.Get("user")
 	if user != "" && user != cred.AccessKey {
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.ListServiceAccountsAdminAction,
+			Action:          policy.ListServiceAccountsAdminAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -1146,12 +1062,16 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// We do not care if service account is readable or not at this point,
 	// since this is a delete call we shall allow it to be deleted if possible.
-	svcAccount, _, _ := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
+	svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
+	if errors.Is(err, errNoSuchServiceAccount) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
+		return
+	}
 
-	adminPrivilege := globalIAMSys.IsAllowed(iampolicy.Args{
+	adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.RemoveServiceAccountAdminAction,
+		Action:          policy.RemoveServiceAccountAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -1192,7 +1112,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	writeSuccessNoContent(w)
 }
 
-// AccountInfoHandler returns usage
+// AccountInfoHandler returns usage, permissions and other bucket metadata for incoming us
 func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -1219,10 +1139,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	enablePrefixUsage := r.Form.Get("prefix-usage") == "true"
 
 	isAllowedAccess := func(bucketName string) (rd, wr bool) {
-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.ListBucketAction,
+			Action:          policy.ListBucketAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
@@ -1232,10 +1152,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			rd = true
 		}
 
-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.GetBucketLocationAction,
+			Action:          policy.GetBucketLocationAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
@@ -1245,10 +1165,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			rd = true
 		}
 
-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.PutObjectAction,
+			Action:          policy.PutObjectAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
@@ -1261,12 +1181,30 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	// Load the latest calculated data usage
+	bucketStorageCache.Once.Do(func() {
-	dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
+		// Set this to 10 secs since its enough, as scanner
+		// does not update the bucket usage values frequently.
+		bucketStorageCache.TTL = 10 * time.Second
+
+		// Rely on older value if usage loading fails from disk.
+		bucketStorageCache.Relax = true
+		bucketStorageCache.Update = func() (interface{}, error) {
+			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
+			defer done()
+
+			return loadDataUsageFromBackend(ctx, objectAPI)
+		}
+	})
+
+	var dataUsageInfo DataUsageInfo
+	v, _ := bucketStorageCache.Get()
+	if v != nil {
+		dataUsageInfo, _ = v.(DataUsageInfo)
+	}
 
 	// If etcd, dns federation configured list buckets from etcd.
-	var buckets []BucketInfo
 	var err error
+	var buckets []BucketInfo
 	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && !IsErrIgnored(err,
@@ -1285,7 +1223,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return buckets[i].Name < buckets[j].Name
 		})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1298,14 +1236,14 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		accountName = cred.ParentUser
 	}
 
-	roleArn := iampolicy.Args{Claims: cred.Claims}.GetRoleArn()
+	roleArn := policy.Args{Claims: cred.Claims}.GetRoleArn()
-	policySetFromClaims, hasPolicyClaim := iampolicy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
+	policySetFromClaims, hasPolicyClaim := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
-	var effectivePolicy iampolicy.Policy
+	var effectivePolicy policy.Policy
 
 	var buf []byte
 	switch {
 	case accountName == globalActiveCred.AccessKey:
-		for _, policy := range iampolicy.DefaultPolicies {
+		for _, policy := range policy.DefaultPolicies {
 			if policy.Name == "consoleAdmin" {
 				effectivePolicy = policy.Definition
 				break
@@ -1325,7 +1263,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		effectivePolicy = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
 
 	default:
-		policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
+		policies, err := globalIAMSys.PolicyDBGet(accountName, cred.Groups...)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1417,7 +1355,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1463,7 +1401,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1475,7 +1413,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	newPolicies := make(map[string]iampolicy.Policy)
+	newPolicies := make(map[string]policy.Policy)
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
@@ -1494,7 +1432,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1505,7 +1443,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	newPolicies := make(map[string]iampolicy.Policy)
+	newPolicies := make(map[string]policy.Policy)
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
@@ -1524,7 +1462,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DeletePolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1550,7 +1488,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.CreatePolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1582,7 +1520,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	iamPolicy, err := iampolicy.ParseConfig(bytes.NewReader(iamPolicyBytes))
+	iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1614,7 +1552,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1688,7 +1626,7 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
 
 	// Check authorization.
 	objectAPI, cred := validateAdminReq(ctx, w, r,
-		iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
+		policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1726,8 +1664,8 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
 func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction,
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction,
-		iampolicy.AttachPolicyAdminAction)
+		policy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1826,7 +1764,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportIAMAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportIAMAction)
 	if objectAPI == nil {
 		return
 	}
@@ -2096,7 +2034,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			return
 		default:
 			defer f.Close()
-			var allPolicies map[string]iampolicy.Policy
+			var allPolicies map[string]policy.Policy
 			data, err = io.ReadAll(f)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
@@ -2177,10 +2115,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					checkDenyOnly = true
 				}
 
-				if !globalIAMSys.IsAllowed(iampolicy.Args{
+				if !globalIAMSys.IsAllowed(policy.Args{
 					AccountName:     cred.AccessKey,
 					Groups:          cred.Groups,
-					Action:          iampolicy.CreateUserAdminAction,
+					Action:          policy.CreateUserAdminAction,
 					ConditionValues: getConditionValues(r, "", cred),
 					IsOwner:         owner,
 					Claims:          cred.Claims,
@@ -2257,10 +2195,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			for user, svcAcctReq := range serviceAcctReqs {
-				var sp *iampolicy.Policy
+				var sp *policy.Policy
 				var err error
 				if len(svcAcctReq.SessionPolicy) > 0 {
-					sp, err = iampolicy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
+					sp, err = policy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
 					if err != nil {
 						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
 						return
@@ -2271,10 +2209,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
 					return
 				}
-				if !globalIAMSys.IsAllowed(iampolicy.Args{
+				if !globalIAMSys.IsAllowed(policy.Args{
 					AccountName:     cred.AccessKey,
 					Groups:          cred.Groups,
-					Action:          iampolicy.CreateServiceAccountAdminAction,
+					Action:          policy.CreateServiceAccountAdminAction,
 					ConditionValues: getConditionValues(r, "", cred),
 					IsOwner:         owner,
 					Claims:          cred.Claims,
@@ -2475,3 +2413,85 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 }
+
+func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrServerNotInitialized)
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(s3Err)
+	}
+
+	password := cred.SecretKey
+	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err)
+	}
+
+	var createReq madmin.AddServiceAccountReq
+	if err = json.Unmarshal(reqBytes, &createReq); err != nil {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err)
+	}
+
+	// service account access key cannot have space characters beginning and end of the string.
+	if hasSpaceBE(createReq.AccessKey) {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument)
+	}
+
+	if err := createReq.Validate(); err != nil {
+		// Since this validation would happen client side as well, we only send
+		// a generic error message here.
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument)
+	}
+	// If the request did not set a TargetUser, the service account is
+	// created for the request sender.
+	targetUser := createReq.TargetUser
+	if targetUser == "" {
+		targetUser = cred.AccessKey
+	}
+
+	description := createReq.Description
+	if description == "" {
+		description = createReq.Comment
+	}
+	opts := newServiceAccountOpts{
+		accessKey:   createReq.AccessKey,
+		secretKey:   createReq.SecretKey,
+		name:        createReq.Name,
+		description: description,
+		expiration:  createReq.Expiration,
+		claims:      make(map[string]interface{}),
+	}
+
+	// Check if action is allowed if creating access key for another user
+	// Check if action is explicitly denied if for self
+	if !globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.CreateServiceAccountAdminAction,
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		DenyOnly:        (targetUser == cred.AccessKey || targetUser == cred.ParentUser),
+	}) {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
+	}
+
+	var sp *policy.Policy
+	if len(createReq.Policy) > 0 {
+		sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
+		if err != nil {
+			return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
+		}
+	}
+
+	opts.sessionPolicy = sp
+
+	return ctx, cred, opts, createReq, targetUser, APIError{}
+}

cmd/admin-handlers-users_test.go

@@ -39,7 +39,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/signer"
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/v2/env"
 )
 
 const (

cmd/admin-handlers.go

@@ -52,18 +52,22 @@ import (
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/logger/message/log"
-	"github.com/minio/pkg/logger/message/log"
+	xnet "github.com/minio/pkg/v2/net"
-	xnet "github.com/minio/pkg/net"
+	"github.com/minio/pkg/v2/policy"
 	"github.com/secure-io/sio-go"
+	"github.com/zeebo/xxh3"
 )
 
 const (
 	maxEConfigJSONSize        = 262272
 	kubernetesVersionEndpoint = "https://kubernetes.default.svc/version"
+	anonymizeParam            = "anonymize"
+	anonymizeStrict           = "strict"
 )
 
 // Only valid query params for mgmt admin APIs.
@@ -81,7 +85,7 @@ const (
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -252,11 +256,11 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	var objectAPI ObjectLayer
 	switch serviceSig {
 	case serviceRestart:
-		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction)
 	case serviceStop:
-		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction)
 	case serviceFreeze, serviceUnFreeze:
-		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceFreezeAdminAction)
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction)
 	}
 	if objectAPI == nil {
 		return
@@ -331,12 +335,12 @@ type ServerHTTPStats struct {
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.StorageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, true)
 
 	// Collect any disk healing.
 	healing, _ := getAggregatedBackgroundHealState(ctx, nil)
@@ -370,7 +374,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -479,7 +483,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -562,7 +566,7 @@ type PeerLocks struct {
 func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ForceUnlockAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -597,7 +601,7 @@ func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.TopLocksAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -648,7 +652,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -733,7 +737,7 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -754,11 +758,8 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 			return
 		}
 	}
-	// read request body
-	io.CopyN(io.Discard, r.Body, 1)
 
 	globalProfilerMu.Lock()
-
 	if globalProfiler == nil {
 		globalProfiler = make(map[string]minioProfiler, 10)
 	}
@@ -828,7 +829,7 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -926,7 +927,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1001,7 +1002,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 							Message:   hr.errBody,
 							Resource:  r.URL.Path,
 							RequestID: w.Header().Get(xhttp.AmzRequestID),
-							HostID:    globalDeploymentID,
+							HostID:    globalDeploymentID(),
 						})
 					}
 					if !started {
@@ -1056,7 +1057,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	respCh := make(chan healResp)
+	respCh := make(chan healResp, 1)
 	switch {
 	case hip.forceStop:
 		go func() {
@@ -1110,7 +1111,7 @@ func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmi
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1131,7 +1132,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1179,7 +1180,7 @@ func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) ClientDevNullExtraTime(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1196,7 +1197,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 	ctx := r.Context()
 
 	timeStart := time.Now()
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1218,7 +1219,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 	totalRx := int64(0)
 	connectTime := time.Now()
 	for {
-		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			// would mean the network is not stable. Logging here will help in debugging network issues.
 			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
@@ -1238,7 +1239,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1284,7 +1285,7 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1312,7 +1313,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		duration = time.Second * 10
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, true)
 
 	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 	if !sufficientCapacity {
@@ -1400,13 +1401,21 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 }
 
 func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
-	if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{}); err != nil {
+	if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{VersioningEnabled: globalSiteReplicationSys.isEnabled()}); err != nil {
 		if _, ok := err.(BucketExists); !ok {
 			// Only BucketExists error can be ignored.
 			return false, err
 		}
 		bucketExists = true
 	}
+
+	if globalSiteReplicationSys.isEnabled() {
+		configData := []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>speedtest/*</Prefix></ExcludedPrefixes></VersioningConfiguration>`)
+		if _, err = globalBucketMetadataSys.Update(ctx, bucketName, bucketVersioningConfig, configData); err != nil {
+			return false, err
+		}
+	}
+
 	return bucketExists, nil
 }
 
@@ -1445,7 +1454,7 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo,
 func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1566,7 +1575,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.TraceAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -1581,7 +1590,9 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	traceCh := make(chan madmin.TraceInfo, 4000)
+
+	// Keep 100k buffered channel, should be sufficient to ensure we do not lose any events.
+	traceCh := make(chan madmin.TraceInfo, 100000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 
@@ -1605,7 +1616,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		peer.Trace(traceCh, ctx.Done(), traceOpts)
 	}
 
-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()
 
 	enc := json.NewEncoder(w)
@@ -1637,7 +1648,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConsoleLogAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1718,7 +1729,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSCreateKeyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1739,7 +1750,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1776,7 +1787,7 @@ func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1900,7 +1911,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 		}
 	}
 
-	log, audit := fetchLoggerInfo()
+	log, audit := fetchLoggerInfo(ctx)
 
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo()
@@ -1980,7 +1991,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 		Domain:        domain,
 		Region:        globalSite.Region,
 		SQSARN:        globalEventNotifier.GetARNList(false),
-		DeploymentID:  globalDeploymentID,
+		DeploymentID:  globalDeploymentID(),
 		Buckets:       buckets,
 		Objects:       objects,
 		Versions:      versions,
@@ -2025,8 +2036,14 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
 
 func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *url.Values, healthInfoCh chan madmin.HealthInfo, healthInfo madmin.HealthInfo) {
 	hostAnonymizer := createHostAnonymizer()
-	// anonAddr - Anonymizes hosts in given input string.
+
+	anonParam := query.Get(anonymizeParam)
+	// anonAddr - Anonymizes hosts in given input string
+	// (only if the anonymize param is set to srict).
 	anonAddr := func(addr string) string {
+		if anonParam != anonymizeStrict {
+			return addr
+		}
 		newAddr, found := hostAnonymizer[addr]
 		if found {
 			return newAddr
@@ -2087,6 +2104,20 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		}
 	}
 
+	getAndWriteNetInfo := func() {
+		if query.Get(string(madmin.HealthDataTypeSysNet)) == "true" {
+			localNetInfo := madmin.GetNetInfo(globalLocalNodeName, globalInternodeInterface)
+			healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, localNetInfo)
+
+			peerNetInfos := globalNotificationSys.GetNetInfo(healthCtx)
+			for _, n := range peerNetInfos {
+				anonymizeAddr(&n)
+				healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, n)
+			}
+			partialWrite(healthInfo)
+		}
+	}
+
 	getAndWriteOSInfo := func() {
 		if query.Get("sysosinfo") == "true" {
 			localOSInfo := madmin.GetOSInfo(healthCtx, globalLocalNodeName)
@@ -2187,6 +2218,10 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 	}
 
 	anonymizeCmdLine := func(cmdLine string) string {
+		if anonParam != anonymizeStrict {
+			return cmdLine
+		}
+
 		if !globalIsDistErasure {
 			// FS mode - single server - hard code to `server1`
 			anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1")
@@ -2308,6 +2343,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		getAndWritePlatformInfo()
 		getAndWriteCPUs()
 		getAndWritePartitions()
+		getAndWriteNetInfo()
 		getAndWriteOSInfo()
 		getAndWriteMemInfo()
 		getAndWriteProcInfo()
@@ -2376,7 +2412,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -2390,7 +2426,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		Version:   madmin.HealthInfoVersion,
 		Minio: madmin.MinioHealthInfo{
 			Info: madmin.MinioInfo{
-				DeploymentID: globalDeploymentID,
+				DeploymentID: globalDeploymentID(),
 			},
 		},
 	}
@@ -2465,11 +2501,27 @@ func getTLSInfo() madmin.TLSInfo {
 
 	if globalIsTLS {
 		for _, c := range globalPublicCerts {
+			check := xxh3.Hash(c.RawIssuer)
+			check ^= xxh3.Hash(c.RawSubjectPublicKeyInfo)
+			// We XOR, so order doesn't matter.
+			for _, v := range c.DNSNames {
+				check ^= xxh3.HashString(v)
+			}
+			for _, v := range c.EmailAddresses {
+				check ^= xxh3.HashString(v)
+			}
+			for _, v := range c.IPAddresses {
+				check ^= xxh3.HashString(v.String())
+			}
+			for _, v := range c.URIs {
+				check ^= xxh3.HashString(v.String())
+			}
 			tlsInfo.Certs = append(tlsInfo.Certs, madmin.TLSCert{
 				PubKeyAlgo:    c.PublicKeyAlgorithm.String(),
 				SignatureAlgo: c.SignatureAlgorithm.String(),
 				NotBefore:     c.NotBefore,
 				NotAfter:      c.NotAfter,
+				Checksum:      strconv.FormatUint(check, 16),
 			})
 		}
 	}
@@ -2483,7 +2535,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ServerInfoAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -2520,7 +2572,7 @@ func assignPoolNumbers(servers []madmin.ServerProperties) {
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
-	for _, tgt := range globalNotifyTargetList.Targets() {
+	for _, tgt := range globalEventNotifier.Targets() {
 		targetIDStatus := make(map[string]madmin.Status)
 		active, _ := tgt.IsActive()
 		targetID := tgt.ID()
@@ -2608,31 +2660,38 @@ func fetchKMSStatusV2(ctx context.Context) []madmin.KMS {
 	return stats
 }
 
+func targetStatus(ctx context.Context, h logger.Target) madmin.Status {
+	if h.IsOnline(ctx) {
+		return madmin.Status{Status: string(madmin.ItemOnline)}
+	}
+	return madmin.Status{Status: string(madmin.ItemOffline)}
+}
+
 // fetchLoggerDetails return log info
-func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
+func fetchLoggerInfo(ctx context.Context) ([]madmin.Logger, []madmin.Audit) {
 	var loggerInfo []madmin.Logger
 	var auditloggerInfo []madmin.Audit
 	for _, tgt := range logger.SystemTargets() {
 		if tgt.Endpoint() != "" {
-			loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
+			loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): targetStatus(ctx, tgt)})
 		}
 	}
 
 	for _, tgt := range logger.AuditTargets() {
 		if tgt.Endpoint() != "" {
-			auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
+			auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): targetStatus(ctx, tgt)})
 		}
 	}
 
 	return loggerInfo, auditloggerInfo
 }
 
-func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
+func embedFileInZip(zipWriter *zip.Writer, name string, data []byte, fileMode os.FileMode) error {
 	// Send profiling data to zip as file
 	header, zerr := zip.FileInfoHeader(dummyFileInfo{
 		name:    name,
 		size:    int64(len(data)),
-		mode:    0o600,
+		mode:    fileMode,
 		modTime: UTCNow(),
 		isDir:   false,
 		sys:     nil,
@@ -2674,7 +2733,7 @@ func getClusterMetaInfo(ctx context.Context) []byte {
 		ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
 		ci.Info.MinioVersion = Version
 
-		si := objectAPI.StorageInfo(ctx)
+		si := objectAPI.StorageInfo(ctx, true)
 
 		ci.Info.NoOfDrives = len(si.Disks)
 		for _, disk := range si.Disks {
@@ -2688,7 +2747,7 @@ func getClusterMetaInfo(ctx context.Context) []byte {
 		ci.Info.NoOfBuckets = dataUsageInfo.BucketsCount
 		ci.Info.NoOfObjects = dataUsageInfo.ObjectsTotalCount
 
-		ci.DeploymentID = globalDeploymentID
+		ci.DeploymentID = globalDeploymentID()
 		ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion)
 
 		select {
@@ -2735,7 +2794,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.InspectDataAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.InspectDataAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -2863,7 +2922,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		defer inspectZipW.Close()
 
 		if b := getClusterMetaInfo(ctx); len(b) > 0 {
-			logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b))
+			logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b, 0o600))
 		}
 	}
 
@@ -2927,11 +2986,16 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		sb.WriteString(pool.CmdLine)
 	}
 	sb.WriteString("\n")
-	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes()))
+	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes(), 0o600))
+
+	scheme := "https"
+	if !globalIsTLS {
+		scheme = "http"
+	}
 
 	// save MinIO start script to inspect command
 	var scrb bytes.Buffer
-	scrb.WriteString(`#!/usr/bin/env bash
+	fmt.Fprintf(&scrb, `#!/usr/bin/env bash
 
 function main() {
 	for file in $(ls -1); do
@@ -2940,10 +3004,10 @@ function main() {
 	done
 
 	# Read content of inspect-input.txt
-	MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#https:\/\/#\.\/#g")
+	MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#%s:\/\/#\.\/#g")
 
 	# Start MinIO instance using the options
-	START_CMD="CI=on MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server ${MINIO_OPTS} &"
+	START_CMD="CI=on _MINIO_AUTO_DRIVE_HEALING=off minio server ${MINIO_OPTS} &"
 	echo
 	echo "Starting MinIO instance: ${START_CMD}"
 	echo
@@ -2955,9 +3019,8 @@ function main() {
 	sleep 10
 }
 
-main "$@"`,
+main "$@"`, scheme)
-	)
+	logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes(), 0o755))
-	logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes()))
 }
 
 func getSubnetAdminPublicKey() []byte {

cmd/admin-heal-ops.go

@@ -683,13 +683,6 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	}
 }
 
-func (h *healSequence) logHeal(healType madmin.HealItemType) {
-	h.mutex.Lock()
-	h.scannedItemsMap[healType]++
-	h.lastHealActivity = UTCNow()
-	h.mutex.Unlock()
-}
-
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
 	// Send heal request
 	task := healTask{
@@ -817,7 +810,7 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
+		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
@@ -874,7 +867,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 
 	if !h.settings.Recursive {
 		if h.object != "" {
-			if err := h.healObject(bucket, h.object, ""); err != nil {
+			if err := h.healObject(bucket, h.object, "", h.settings.ScanMode); err != nil {
 				return err
 			}
 		}
@@ -889,7 +882,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 }
 
 // healObject - heal the given object and record result
-func (h *healSequence) healObject(bucket, object, versionID string) error {
+func (h *healSequence) healObject(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}

cmd/admin-router.go

@@ -300,6 +300,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.GetIdentityProviderCfg))
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.DeleteIdentityProviderCfg))
 
+		// LDAP specific service accounts ops
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp/ldap/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccountLDAP))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys").
+			HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAP)).
+			Queries("userDN", "{userDN:.*}", "listType", "{listType:.*}")
+
 		// LDAP IAM operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListLDAPPolicyMappingEntities))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyLDAP))
@@ -376,6 +382,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(adminMiddleware(adminAPI.SRPeerEdit))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(adminMiddleware(adminAPI.SRPeerRemove))
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationResyncOp)).Queries("operation", "{operation:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/state/edit").HandlerFunc(adminMiddleware(adminAPI.SRStateEdit))
 
 		if globalIsDistErasure {
 			// Top locks

cmd/admin-server-info.go

@@ -141,7 +141,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 
 	objLayer := newObjectLayerFn()
 	if objLayer != nil {
-		storageInfo := objLayer.LocalStorageInfo(GlobalContext)
+		storageInfo := objLayer.LocalStorageInfo(GlobalContext, true)
 		props.State = string(madmin.ItemOnline)
 		props.Disks = storageInfo.Disks
 	} else {

cmd/api-errors.go

@@ -47,7 +47,7 @@ import (
 	levent "github.com/minio/minio/internal/config/lambda/event"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // APIError structure
@@ -310,6 +310,7 @@ const (
 	ErrSiteReplicationBucketMetaError
 	ErrSiteReplicationIAMError
 	ErrSiteReplicationConfigMissing
+	ErrSiteReplicationIAMConfigMismatch
 
 	// Pool rebalance errors
 	ErrAdminRebalanceAlreadyStarted
@@ -1512,6 +1513,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Site not found in site replication configuration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrSiteReplicationIAMConfigMismatch: {
+		Code:           "XMinioSiteReplicationIAMConfigMismatch",
+		Description:    "IAM configuration mismatch between sites",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminRebalanceAlreadyStarted: {
 		Code:           "XMinioAdminRebalanceAlreadyStarted",
 		Description:    "Pool rebalance is already started",
@@ -2236,8 +2242,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrSlowDownWrite
 	case InsufficientReadQuorum:
 		apiErr = ErrSlowDownRead
-	case InvalidMarkerPrefixCombination:
-		apiErr = ErrNotImplemented
 	case InvalidUploadIDKeyCombination:
 		apiErr = ErrNotImplemented
 	case MalformedUploadID:
@@ -2339,6 +2343,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case dns.ErrBucketConflict:
 		apiErr = ErrBucketAlreadyExists
 	default:
+		if _, ok := err.(tags.Error); ok {
+			// tag errors are not exported, so we check their custom interface to avoid logging.
+			// The correct type is inserted by toAPIError.
+			apiErr = ErrInternalError
+			break
+		}
 		var ie, iw int
 		// This work-around is to handle the issue golang/go#30648
 		//nolint:gocritic

cmd/api-errors_test.go

@@ -42,7 +42,6 @@ var toAPIErrorTests = []struct {
 	{err: InvalidPart{}, errCode: ErrInvalidPart},
 	{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
 	{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
-	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
 	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
 	{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
 	{err: PartTooSmall{}, errCode: ErrEntityTooSmall},

cmd/api-headers.go

@@ -133,11 +133,6 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}
 
-	if globalCacheConfig.Enabled {
-		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
-		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
-	}
-
 	// Set tag count if object has tags
 	if len(objInfo.UserTags) > 0 {
 		tags, _ := tags.ParseObjectTags(objInfo.UserTags)

cmd/api-response.go

@@ -35,7 +35,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 	xxml "github.com/minio/xxml"
 )
 
@@ -502,6 +502,47 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	return data
 }
 
+func cleanReservedKeys(metadata map[string]string) map[string]string {
+	m := cloneMSS(metadata)
+
+	switch kind, _ := crypto.IsEncrypted(metadata); kind {
+	case crypto.S3:
+		m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+	case crypto.S3KMS:
+		m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
+		m[xhttp.AmzServerSideEncryptionKmsID] = kmsKeyIDFromMetadata(metadata)
+		if kmsCtx, ok := metadata[crypto.MetaContext]; ok {
+			m[xhttp.AmzServerSideEncryptionKmsContext] = kmsCtx
+		}
+	case crypto.SSEC:
+		m[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
+
+	}
+
+	var toRemove []string
+	for k := range cleanMinioInternalMetadataKeys(m) {
+		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
+			// Do not need to send any internal metadata
+			// values to client.
+			toRemove = append(toRemove, k)
+			continue
+		}
+
+		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+			toRemove = append(toRemove, k)
+			continue
+		}
+	}
+
+	for _, k := range toRemove {
+		delete(m, k)
+		delete(m, strings.ToLower(k))
+	}
+
+	return m
+}
+
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
@@ -549,18 +590,11 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 			case crypto.SSEC:
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 			}
-			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+			for k, v := range cleanReservedKeys(object.UserDefined) {
-				if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
-					// Do not need to send any internal metadata
-					// values to client.
-					continue
-				}
-				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
-					continue
-				}
 				content.UserMetadata.Set(k, v)
 			}
+
+			content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
 			content.Internal = &ObjectInternalInfo{
 				K: object.DataBlocks,
 				M: object.ParityBlocks,
@@ -692,18 +726,10 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 				case crypto.SSEC:
 					content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 				}
-				for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+				for k, v := range cleanReservedKeys(object.UserDefined) {
-					if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
-						// Do not need to send any internal metadata
-						// values to client.
-						continue
-					}
-					// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-					if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
-						continue
-					}
 					content.UserMetadata.Set(k, v)
 				}
+				content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
 				content.Internal = &ObjectInternalInfo{
 					K: object.DataBlocks,
 					M: object.ParityBlocks,
@@ -978,7 +1004,7 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
 		RequestID:  w.Header().Get(xhttp.AmzRequestID),
-		HostID:     globalDeploymentID,
+		HostID:     globalDeploymentID(),
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)

cmd/api-router.go

@@ -27,7 +27,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/wildcard"
+	"github.com/minio/pkg/v2/wildcard"
 	"github.com/rs/cors"
 )
 
@@ -61,18 +61,6 @@ func newObjectLayerFn() ObjectLayer {
 	return globalObjectAPI
 }
 
-func newCachedObjectLayerFn() CacheObjectLayer {
-	globalObjLayerMutex.RLock()
-	defer globalObjLayerMutex.RUnlock()
-	return globalCacheObjectAPI
-}
-
-func setCacheObjectLayer(c CacheObjectLayer) {
-	globalObjLayerMutex.Lock()
-	globalCacheObjectAPI = c
-	globalObjLayerMutex.Unlock()
-}
-
 func setObjectLayer(o ObjectLayer) {
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = o
@@ -82,7 +70,6 @@ func setObjectLayer(o ObjectLayer) {
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
-	CacheAPI  func() CacheObjectLayer
 }
 
 // getHost tries its best to return the request host.
@@ -189,7 +176,6 @@ func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
-		CacheAPI:  newCachedObjectLayerFn,
 	}
 
 	// API Router

cmd/auth-handler.go

@@ -41,8 +41,7 @@ import (
 	xjwt "github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
-	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // Verify if request has JWT.
@@ -186,15 +185,15 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 // checkAdminRequestAuth checks for authentication and authorization for the incoming
 // request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
 // are automatically rejected.
-func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
+func checkAdminRequestAuth(ctx context.Context, r *http.Request, action policy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
 	cred, owner, s3Err := validateAdminSignature(ctx, r, region)
 	if s3Err != ErrNone {
 		return cred, s3Err
 	}
-	if globalIAMSys.IsAllowed(iampolicy.Args{
+	if globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.Action(action),
+		Action:          policy.Action(action),
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -248,7 +247,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 	}
 
 	// Check if a session policy is set. If so, decode it here.
-	sp, spok := claims.Lookup(iampolicy.SessionPolicyName)
+	sp, spok := claims.Lookup(policy.SessionPolicyName)
 	if spok {
 		// Looks like subpolicy is set and is a string, if set then its
 		// base64 encoded, decode it. Decoding fails reject such
@@ -413,7 +412,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 
 	if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
 		// Anonymous checks are not meant for ListAllBuckets action
-		if globalPolicySys.IsAllowed(policy.Args{
+		if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
 			Action:          action,
@@ -429,7 +428,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 		if action == policy.ListBucketVersionsAction {
 			// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
 			// verify as a fallback.
-			if globalPolicySys.IsAllowed(policy.Args{
+			if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
 				AccountName:     cred.AccessKey,
 				Groups:          cred.Groups,
 				Action:          policy.ListBucketAction,
@@ -446,10 +445,10 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 		return ErrAccessDenied
 	}
 	if action == policy.DeleteObjectAction && versionID != "" {
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.Action(policy.DeleteObjectVersionAction),
+			Action:          policy.Action(policy.DeleteObjectVersionAction),
 			BucketName:      bucket,
 			ConditionValues: getConditionValues(r, "", cred),
 			ObjectName:      object,
@@ -460,10 +459,10 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 			return ErrAccessDenied
 		}
 	}
-	if globalIAMSys.IsAllowed(iampolicy.Args{
+	if globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.Action(action),
+		Action:          action,
 		BucketName:      bucket,
 		ConditionValues: getConditionValues(r, "", cred),
 		ObjectName:      object,
@@ -477,10 +476,10 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 	if action == policy.ListBucketVersionsAction {
 		// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
 		// verify as a fallback.
-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.ListBucketAction,
+			Action:          policy.ListBucketAction,
 			BucketName:      bucket,
 			ConditionValues: getConditionValues(r, "", cred),
 			ObjectName:      object,
@@ -568,7 +567,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 
 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
+	reader, err := hash.NewReader(ctx, r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}
@@ -696,10 +695,10 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
 	}
 	if retMode == objectlock.RetGovernance && byPassSet {
-		byPassSet = globalIAMSys.IsAllowed(iampolicy.Args{
+		byPassSet = globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.BypassGovernanceRetentionAction,
+			Action:          policy.BypassGovernanceRetentionAction,
 			BucketName:      bucketName,
 			ObjectName:      objectName,
 			ConditionValues: conditions,
@@ -707,10 +706,10 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 			Claims:          cred.Claims,
 		})
 	}
-	if globalIAMSys.IsAllowed(iampolicy.Args{
+	if globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.PutObjectRetentionAction,
+		Action:          policy.PutObjectRetentionAction,
 		BucketName:      bucketName,
 		ConditionValues: conditions,
 		ObjectName:      objectName,
@@ -728,7 +727,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 // isPutActionAllowed - check if PUT operation is allowed on the resource, this
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.
-func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
+func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	region := globalSite.Region
@@ -751,17 +750,17 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 	// Do not check for PutObjectRetentionAction permission,
 	// if mode and retain until date are not set.
 	// Can happen when bucket has default lock config set
-	if action == iampolicy.PutObjectRetentionAction &&
+	if action == policy.PutObjectRetentionAction &&
 		r.Header.Get(xhttp.AmzObjectLockMode) == "" &&
 		r.Header.Get(xhttp.AmzObjectLockRetainUntilDate) == "" {
 		return ErrNone
 	}
 
 	if cred.AccessKey == "" {
-		if globalPolicySys.IsAllowed(policy.Args{
+		if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          policy.Action(action),
+			Action:          action,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 			IsOwner:         false,
@@ -772,7 +771,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 		return ErrAccessDenied
 	}
 
-	if globalIAMSys.IsAllowed(iampolicy.Args{
+	if globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          action,

cmd/auth-handler_test.go

@@ -28,7 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/auth"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 type nullReader struct{}
@@ -443,7 +443,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 	}
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, policy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}

cmd/background-heal-ops.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/v2/env"
 )
 
 // healTask represents what to heal along with options

cmd/background-newdisks-heal-ops.go

@@ -34,7 +34,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/v2/env"
 )
 
 const (
@@ -232,7 +232,7 @@ func (h *healingTracker) delete(ctx context.Context) error {
 		pathJoin(bucketMetaPrefix, healingTrackerFilename),
 		DeleteOptions{
 			Recursive: false,
-			Force:     false,
+			Immediate: false,
 		},
 	)
 }
@@ -345,18 +345,18 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
 
+	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn || env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
 		globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
-	if env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
 		go monitorLocalDisksAndHeal(ctx, z)
 	}
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
 	globalLocalDrivesMu.RLock()
-	globalLocalDrives := globalLocalDrives
+	localDrives := globalLocalDrives
 	globalLocalDrivesMu.RUnlock()
-	for _, disk := range globalLocalDrives {
+	for _, disk := range localDrives {
 		_, err := disk.GetDiskID()
 		if errors.Is(err, errUnformattedDisk) {
 			disksToHeal = append(disksToHeal, disk.Endpoint())

cmd/batch-expire.go

@@ -0,0 +1,677 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"runtime"
+	"strconv"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v2/env"
+	"github.com/minio/pkg/v2/wildcard"
+	"github.com/minio/pkg/v2/workers"
+)
+
+// expire: # Expire objects that match a condition
+//   apiVersion: v1
+//   bucket: mybucket # Bucket where this batch job will expire matching objects from
+//   prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
+//   rules:
+//     - type: object  # regular objects with zero ore more older versions
+//       name: NAME # match object names that satisfy the wildcard expression.
+//       olderThan: 70h # match objects older than this value
+//       createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+//       tags:
+//         - key: name
+//           value: pick* # match objects with tag 'name', all values starting with 'pick'
+//       metadata:
+//         - key: content-type
+//           value: image/* # match objects with 'content-type', all values starting with 'image/'
+//       size:
+//         lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
+//         greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
+//       purge:
+//           # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+//           # retainVersions: 5 # keep the latest 5 versions of the object.
+//
+//     - type: deleted # objects with delete marker as their latest version
+//       name: NAME # match object names that satisfy the wildcard expression.
+//       olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
+//       createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+//       purge:
+//           # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+//           # retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
+//
+//   notify:
+//     endpoint: https://notify.endpoint # notification endpoint to receive job completion status
+//     token: Bearer xxxxx # optional authentication token for the notification endpoint
+//
+//   retry:
+//     attempts: 10 # number of retries for the job before giving up
+//     delay: 500ms # least amount of delay between each retry
+
+//go:generate msgp -file $GOFILE
+
+// BatchJobExpirePurge type accepts non-negative versions to be retained
+type BatchJobExpirePurge struct {
+	RetainVersions int `yaml:"retainVersions" json:"retainVersions"`
+}
+
+// Validate returns nil if value is valid, ie > 0.
+func (p BatchJobExpirePurge) Validate() error {
+	if p.RetainVersions < 0 {
+		return errors.New("retainVersions must be >= 0")
+	}
+	return nil
+}
+
+// BatchJobExpireFilter holds all the filters currently supported for batch replication
+type BatchJobExpireFilter struct {
+	OlderThan     time.Duration       `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedBefore *time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchJobKV        `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchJobKV        `yaml:"metadata,omitempty" json:"metadata"`
+	Size          BatchJobSizeFilter  `yaml:"size" json:"size"`
+	Type          string              `yaml:"type" json:"type"`
+	Name          string              `yaml:"name" json:"name"`
+	Purge         BatchJobExpirePurge `yaml:"purge" json:"purge"`
+}
+
+// Matches returns true if obj matches the filter conditions specified in ef.
+func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
+	switch ef.Type {
+	case BatchJobExpireObject:
+		if obj.DeleteMarker {
+			return false
+		}
+	case BatchJobExpireDeleted:
+		if !obj.DeleteMarker {
+			return false
+		}
+	default:
+		// we should never come here, Validate should have caught this.
+		logger.LogOnceIf(context.Background(), fmt.Errorf("invalid filter type: %s", ef.Type), ef.Type)
+		return false
+	}
+
+	if len(ef.Name) > 0 && !wildcard.Match(ef.Name, obj.Name) {
+		return false
+	}
+	if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan {
+		return false
+	}
+
+	if ef.CreatedBefore != nil && !obj.ModTime.Before(*ef.CreatedBefore) {
+		return false
+	}
+
+	if len(ef.Tags) > 0 && !obj.DeleteMarker {
+		// Only parse object tags if tags filter is specified.
+		var tagMap map[string]string
+		if len(obj.UserTags) != 0 {
+			t, err := tags.ParseObjectTags(obj.UserTags)
+			if err != nil {
+				return false
+			}
+			tagMap = t.ToMap()
+		}
+
+		for _, kv := range ef.Tags {
+			// Object (version) must match all tags specified in
+			// the filter
+			var match bool
+			for t, v := range tagMap {
+				if kv.Match(BatchJobKV{Key: t, Value: v}) {
+					match = true
+				}
+			}
+			if !match {
+				return false
+			}
+		}
+
+	}
+	if len(ef.Metadata) > 0 && !obj.DeleteMarker {
+		for _, kv := range ef.Metadata {
+			// Object (version) must match all x-amz-meta and
+			// standard metadata headers
+			// specified in the filter
+			var match bool
+			for k, v := range obj.UserDefined {
+				if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
+					continue
+				}
+				// We only need to match x-amz-meta or standardHeaders
+				if kv.Match(BatchJobKV{Key: k, Value: v}) {
+					match = true
+				}
+			}
+			if !match {
+				return false
+			}
+		}
+	}
+
+	return ef.Size.InRange(obj.Size)
+}
+
+const (
+	// BatchJobExpireObject - object type
+	BatchJobExpireObject string = "object"
+	// BatchJobExpireDeleted - delete marker type
+	BatchJobExpireDeleted string = "deleted"
+)
+
+// Validate returns nil if ef has valid fields, validation error otherwise.
+func (ef BatchJobExpireFilter) Validate() error {
+	switch ef.Type {
+	case BatchJobExpireObject:
+	case BatchJobExpireDeleted:
+		if len(ef.Tags) > 0 || len(ef.Metadata) > 0 {
+			return errors.New("invalid batch-expire rule filter")
+		}
+	default:
+		return errors.New("invalid batch-expire type")
+	}
+
+	for _, tag := range ef.Tags {
+		if err := tag.Validate(); err != nil {
+			return err
+		}
+	}
+
+	for _, meta := range ef.Metadata {
+		if err := meta.Validate(); err != nil {
+			return err
+		}
+	}
+	if err := ef.Purge.Validate(); err != nil {
+		return err
+	}
+	if err := ef.Size.Validate(); err != nil {
+		return err
+	}
+	if ef.CreatedBefore != nil && !ef.CreatedBefore.Before(time.Now()) {
+		return errors.New("CreatedBefore is in the future")
+	}
+	return nil
+}
+
+// BatchJobExpire represents configuration parameters for a batch expiration
+// job typically supplied in yaml form
+type BatchJobExpire struct {
+	APIVersion      string                 `yaml:"apiVersion" json:"apiVersion"`
+	Bucket          string                 `yaml:"bucket" json:"bucket"`
+	Prefix          string                 `yaml:"prefix" json:"prefix"`
+	NotificationCfg BatchJobNotification   `yaml:"notify" json:"notify"`
+	Retry           BatchJobRetry          `yaml:"retry" json:"retry"`
+	Rules           []BatchJobExpireFilter `yaml:"rules" json:"rules"`
+}
+
+// Notify notifies notification endpoint if configured regarding job failure or success.
+func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
+	if r.NotificationCfg.Endpoint == "" {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.NotificationCfg.Endpoint, body)
+	if err != nil {
+		return err
+	}
+
+	if r.NotificationCfg.Token != "" {
+		req.Header.Set("Authorization", r.NotificationCfg.Token)
+	}
+
+	clnt := http.Client{Transport: getRemoteInstanceTransport}
+	resp, err := clnt.Do(req)
+	if err != nil {
+		return err
+	}
+
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(resp.Status)
+	}
+
+	return nil
+}
+
+// Expire expires object versions which have already matched supplied filter conditions
+func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error {
+	opts := ObjectOptions{
+		PrefixEnabledFn:  vc.PrefixEnabled,
+		VersionSuspended: vc.Suspended(),
+	}
+	_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel, opts)
+	return errs
+}
+
+const (
+	batchExpireName                 = "batch-expire.bin"
+	batchExpireFormat               = 1
+	batchExpireVersionV1            = 1
+	batchExpireVersion              = batchExpireVersionV1
+	batchExpireAPIVersion           = "v1"
+	batchExpireJobDefaultRetries    = 3
+	batchExpireJobDefaultRetryDelay = 250 * time.Millisecond
+)
+
+type objInfoCache map[string]*ObjectInfo
+
+func newObjInfoCache() objInfoCache {
+	return objInfoCache(make(map[string]*ObjectInfo))
+}
+
+func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) {
+	oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi
+}
+
+func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
+	oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)]
+	return oi, ok
+}
+
+func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
+	vc, _ := globalBucketVersioningSys.Get(r.Bucket)
+	retryAttempts := r.Retry.Attempts
+	delay := job.Expire.Retry.Delay
+	if delay == 0 {
+		delay = batchExpireJobDefaultRetryDelay
+	}
+
+	var i int
+	for toExpire := range expireCh {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+		}
+		if i > 0 {
+			if wait := globalBatchConfig.ExpirationWait(); wait > 0 {
+				time.Sleep(wait)
+			}
+		}
+		i++
+		wk.Take()
+		go func(toExpire []expireObjInfo) {
+			defer wk.Give()
+
+			toExpireAll := make([]ObjectInfo, 0, len(toExpire))
+			toDel := make([]ObjectToDelete, 0, len(toExpire))
+			oiCache := newObjInfoCache()
+			for _, exp := range toExpire {
+				if exp.ExpireAll {
+					toExpireAll = append(toExpireAll, exp.ObjectInfo)
+					continue
+				}
+				// Cache ObjectInfo value via pointers for
+				// subsequent use to track objects which
+				// couldn't be deleted.
+				od := ObjectToDelete{
+					ObjectV: ObjectV{
+						ObjectName: exp.Name,
+						VersionID:  exp.VersionID,
+					},
+				}
+				toDel = append(toDel, od)
+				oiCache.Add(od, &exp.ObjectInfo)
+			}
+
+			var done bool
+			// DeleteObject(deletePrefix: true) to expire all versions of an object
+			for _, exp := range toExpireAll {
+				var success bool
+				for attempts := 1; attempts <= retryAttempts; attempts++ {
+					select {
+					case <-ctx.Done():
+						done = true
+					default:
+					}
+					stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+					_, err := api.DeleteObject(ctx, exp.Bucket, exp.Name, ObjectOptions{
+						DeletePrefix: true,
+					})
+					if err != nil {
+						stopFn(exp, err)
+						logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", toExpire[i].Bucket, toExpire[i].Name, toExpire[i].VersionID, err, attempts))
+					} else {
+						stopFn(exp, err)
+						success = true
+						break
+					}
+				}
+				ri.trackMultipleObjectVersions(r.Bucket, exp, success)
+				if done {
+					break
+				}
+			}
+
+			if done {
+				return
+			}
+
+			// DeleteMultiple objects
+			toDelCopy := make([]ObjectToDelete, len(toDel))
+			for attempts := 1; attempts <= retryAttempts; attempts++ {
+				select {
+				case <-ctx.Done():
+					return
+				default:
+				}
+
+				stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
+				// Copying toDel to select from objects whose
+				// deletion failed
+				copy(toDelCopy, toDel)
+				var failed int
+				errs := r.Expire(ctx, api, vc, toDel)
+				// reslice toDel in preparation for next retry
+				// attempt
+				toDel = toDel[:0]
+				for i, err := range errs {
+					if err != nil {
+						stopFn(toDelCopy[i], err)
+						logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, err, attempts))
+						failed++
+						if attempts == retryAttempts { // all retry attempts failed, record failure
+							if oi, ok := oiCache.Get(toDelCopy[i]); ok {
+								ri.trackCurrentBucketObject(r.Bucket, *oi, false)
+							}
+						} else {
+							toDel = append(toDel, toDelCopy[i])
+						}
+					} else {
+						stopFn(toDelCopy[i], nil)
+						if oi, ok := oiCache.Get(toDelCopy[i]); ok {
+							ri.trackCurrentBucketObject(r.Bucket, *oi, true)
+						}
+					}
+				}
+
+				globalBatchJobsMetrics.save(ri.JobID, ri)
+
+				if failed == 0 {
+					break
+				}
+
+				// Add a delay between retry attempts
+				if attempts < retryAttempts {
+					time.Sleep(delay)
+				}
+			}
+		}(toExpire)
+	}
+}
+
+type expireObjInfo struct {
+	ObjectInfo
+	ExpireAll bool
+}
+
+// Start the batch expiration job, resumes if there was a pending job via "job.ID"
+func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	ri := &batchJobInfo{
+		JobID:     job.ID,
+		JobType:   string(job.Type()),
+		StartTime: job.Started,
+	}
+	if err := ri.load(ctx, api, job); err != nil {
+		return err
+	}
+
+	globalBatchJobsMetrics.save(job.ID, ri)
+	lastObject := ri.Object
+
+	now := time.Now().UTC()
+
+	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_EXPIRATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+	if err != nil {
+		return err
+	}
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		// invalid worker size.
+		return err
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	results := make(chan ObjectInfo, workerSize)
+	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
+		Marker:       lastObject,
+		LatestOnly:   false, // we need to visit all versions of the object to implement purge: retainVersions
+		VersionsSort: WalkVersionsSortDesc,
+	}); err != nil {
+		// Do not need to retry if we can't list objects on source.
+		return err
+	}
+
+	// Goroutine to periodically save batch-expire job's in-memory state
+	saverQuitCh := make(chan struct{})
+	go func() {
+		saveTicker := time.NewTicker(10 * time.Second)
+		defer saveTicker.Stop()
+		for {
+			select {
+			case <-saveTicker.C:
+				// persist in-memory state to disk after every 10secs.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+
+			case <-ctx.Done():
+				// persist in-memory state immediately before exiting due to context cancelation.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+				return
+
+			case <-saverQuitCh:
+				// persist in-memory state immediately to disk.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+				return
+			}
+		}
+	}()
+
+	expireCh := make(chan []expireObjInfo, workerSize)
+	go batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
+
+	var (
+		prevObj       ObjectInfo
+		matchedFilter BatchJobExpireFilter
+		versionsCount int
+		toDel         []expireObjInfo
+	)
+	for result := range results {
+		// Apply filter to find the matching rule to apply expiry
+		// actions accordingly.
+		// nolint:gocritic
+		if result.IsLatest {
+			// send down filtered entries to be deleted using
+			// DeleteObjects method
+			if len(toDel) > 10 { // batch up to 10 objects/versions to be expired simultaneously.
+				xfer := make([]expireObjInfo, len(toDel))
+				copy(xfer, toDel)
+
+				var done bool
+				select {
+				case <-ctx.Done():
+					done = true
+				case expireCh <- xfer:
+					toDel = toDel[:0] // resetting toDel
+				}
+				if done {
+					break
+				}
+			}
+			var match BatchJobExpireFilter
+			var found bool
+			for _, rule := range r.Rules {
+				if rule.Matches(result, now) {
+					match = rule
+					found = true
+					break
+				}
+			}
+			if !found {
+				continue
+			}
+
+			prevObj = result
+			matchedFilter = match
+			versionsCount = 1
+			// Include the latest version
+			if matchedFilter.Purge.RetainVersions == 0 {
+				toDel = append(toDel, expireObjInfo{
+					ObjectInfo: result,
+					ExpireAll:  true,
+				})
+				continue
+			}
+		} else if prevObj.Name == result.Name {
+			if matchedFilter.Purge.RetainVersions == 0 {
+				continue // including latest version in toDel suffices, skipping other versions
+			}
+			versionsCount++
+		} else {
+			continue
+		}
+
+		if versionsCount <= matchedFilter.Purge.RetainVersions {
+			continue // retain versions
+		}
+		toDel = append(toDel, expireObjInfo{
+			ObjectInfo: result,
+		})
+	}
+	// Send any remaining objects downstream
+	if len(toDel) > 0 {
+		select {
+		case <-ctx.Done():
+		case expireCh <- toDel:
+		}
+	}
+	close(expireCh)
+
+	wk.Wait() // waits for all expire goroutines to complete
+
+	ri.Complete = ri.ObjectsFailed == 0
+	ri.Failed = ri.ObjectsFailed > 0
+	globalBatchJobsMetrics.save(job.ID, ri)
+
+	// Close the saverQuitCh - this also triggers saving in-memory state
+	// immediately one last time before we exit this method.
+	close(saverQuitCh)
+
+	// Notify expire jobs final status to the configured endpoint
+	buf, _ := json.Marshal(ri)
+	if err := r.Notify(context.Background(), bytes.NewReader(buf)); err != nil {
+		logger.LogIf(context.Background(), fmt.Errorf("unable to notify %v", err))
+	}
+
+	return nil
+}
+
+//msgp:ignore batchExpireJobError
+type batchExpireJobError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+func (e batchExpireJobError) Error() string {
+	return e.Description
+}
+
+// maxBatchRules maximum number of rules a batch-expiry job supports
+const maxBatchRules = 50
+
+// Validate validates the job definition input
+func (r *BatchJobExpire) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
+	if r == nil {
+		return nil
+	}
+
+	if r.APIVersion != batchExpireAPIVersion {
+		return batchExpireJobError{
+			Code:           "InvalidArgument",
+			Description:    "Unsupported batch expire API version",
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	}
+
+	if r.Bucket == "" {
+		return batchExpireJobError{
+			Code:           "InvalidArgument",
+			Description:    "Bucket argument missing",
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	}
+
+	if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil {
+		if isErrBucketNotFound(err) {
+			return batchExpireJobError{
+				Code:           "NoSuchSourceBucket",
+				Description:    "The specified source bucket does not exist",
+				HTTPStatusCode: http.StatusNotFound,
+			}
+		}
+		return err
+	}
+
+	if len(r.Rules) > maxBatchRules {
+		return batchExpireJobError{
+			Code:           "InvalidArgument",
+			Description:    "Too many rules. Batch expire job can't have more than 100 rules",
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	}
+
+	for _, rule := range r.Rules {
+		if err := rule.Validate(); err != nil {
+			return batchExpireJobError{
+				Code:           "InvalidArgument",
+				Description:    fmt.Sprintf("Invalid batch expire rule: %s", err),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
+		}
+	}
+
+	if err := r.Retry.Validate(); err != nil {
+		return batchExpireJobError{
+			Code:           "InvalidArgument",
+			Description:    fmt.Sprintf("Invalid batch expire retry configuration: %s", err),
+			HTTPStatusCode: http.StatusBadRequest,
+		}
+	}
+	return nil
+}

cmd/batch-expire_gen.go

@@ -0,0 +1,856 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"time"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobExpire) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "APIVersion":
+			z.APIVersion, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "APIVersion")
+				return
+			}
+		case "Bucket":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			z.Prefix, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "NotificationCfg":
+			err = z.NotificationCfg.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationCfg")
+				return
+			}
+		case "Retry":
+			err = z.Retry.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Retry")
+				return
+			}
+		case "Rules":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Rules")
+				return
+			}
+			if cap(z.Rules) >= int(zb0002) {
+				z.Rules = (z.Rules)[:zb0002]
+			} else {
+				z.Rules = make([]BatchJobExpireFilter, zb0002)
+			}
+			for za0001 := range z.Rules {
+				err = z.Rules[za0001].DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Rules", za0001)
+					return
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BatchJobExpire) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 6
+	// write "APIVersion"
+	err = en.Append(0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.APIVersion)
+	if err != nil {
+		err = msgp.WrapError(err, "APIVersion")
+		return
+	}
+	// write "Bucket"
+	err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "Prefix"
+	err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Prefix)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
+	// write "NotificationCfg"
+	err = en.Append(0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67)
+	if err != nil {
+		return
+	}
+	err = z.NotificationCfg.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "NotificationCfg")
+		return
+	}
+	// write "Retry"
+	err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
+	if err != nil {
+		return
+	}
+	err = z.Retry.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Retry")
+		return
+	}
+	// write "Rules"
+	err = en.Append(0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Rules)))
+	if err != nil {
+		err = msgp.WrapError(err, "Rules")
+		return
+	}
+	for za0001 := range z.Rules {
+		err = z.Rules[za0001].EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Rules", za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BatchJobExpire) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 6
+	// string "APIVersion"
+	o = append(o, 0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	o = msgp.AppendString(o, z.APIVersion)
+	// string "Bucket"
+	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "Prefix"
+	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	o = msgp.AppendString(o, z.Prefix)
+	// string "NotificationCfg"
+	o = append(o, 0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67)
+	o, err = z.NotificationCfg.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "NotificationCfg")
+		return
+	}
+	// string "Retry"
+	o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
+	o, err = z.Retry.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Retry")
+		return
+	}
+	// string "Rules"
+	o = append(o, 0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Rules)))
+	for za0001 := range z.Rules {
+		o, err = z.Rules[za0001].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Rules", za0001)
+			return
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobExpire) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "APIVersion":
+			z.APIVersion, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "APIVersion")
+				return
+			}
+		case "Bucket":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			z.Prefix, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "NotificationCfg":
+			bts, err = z.NotificationCfg.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationCfg")
+				return
+			}
+		case "Retry":
+			bts, err = z.Retry.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Retry")
+				return
+			}
+		case "Rules":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Rules")
+				return
+			}
+			if cap(z.Rules) >= int(zb0002) {
+				z.Rules = (z.Rules)[:zb0002]
+			} else {
+				z.Rules = make([]BatchJobExpireFilter, zb0002)
+			}
+			for za0001 := range z.Rules {
+				bts, err = z.Rules[za0001].UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Rules", za0001)
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BatchJobExpire) Msgsize() (s int) {
+	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 16 + z.NotificationCfg.Msgsize() + 6 + z.Retry.Msgsize() + 6 + msgp.ArrayHeaderSize
+	for za0001 := range z.Rules {
+		s += z.Rules[za0001].Msgsize()
+	}
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobExpireFilter) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "OlderThan":
+			z.OlderThan, err = dc.ReadDuration()
+			if err != nil {
+				err = msgp.WrapError(err, "OlderThan")
+				return
+			}
+		case "CreatedBefore":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "CreatedBefore")
+					return
+				}
+				z.CreatedBefore = nil
+			} else {
+				if z.CreatedBefore == nil {
+					z.CreatedBefore = new(time.Time)
+				}
+				*z.CreatedBefore, err = dc.ReadTime()
+				if err != nil {
+					err = msgp.WrapError(err, "CreatedBefore")
+					return
+				}
+			}
+		case "Tags":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Tags")
+				return
+			}
+			if cap(z.Tags) >= int(zb0002) {
+				z.Tags = (z.Tags)[:zb0002]
+			} else {
+				z.Tags = make([]BatchJobKV, zb0002)
+			}
+			for za0001 := range z.Tags {
+				err = z.Tags[za0001].DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Tags", za0001)
+					return
+				}
+			}
+		case "Metadata":
+			var zb0003 uint32
+			zb0003, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Metadata")
+				return
+			}
+			if cap(z.Metadata) >= int(zb0003) {
+				z.Metadata = (z.Metadata)[:zb0003]
+			} else {
+				z.Metadata = make([]BatchJobKV, zb0003)
+			}
+			for za0002 := range z.Metadata {
+				err = z.Metadata[za0002].DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Metadata", za0002)
+					return
+				}
+			}
+		case "Size":
+			err = z.Size.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
+		case "Type":
+			z.Type, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Type")
+				return
+			}
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Purge":
+			var zb0004 uint32
+			zb0004, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Purge")
+				return
+			}
+			for zb0004 > 0 {
+				zb0004--
+				field, err = dc.ReadMapKeyPtr()
+				if err != nil {
+					err = msgp.WrapError(err, "Purge")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "RetainVersions":
+					z.Purge.RetainVersions, err = dc.ReadInt()
+					if err != nil {
+						err = msgp.WrapError(err, "Purge", "RetainVersions")
+						return
+					}
+				default:
+					err = dc.Skip()
+					if err != nil {
+						err = msgp.WrapError(err, "Purge")
+						return
+					}
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BatchJobExpireFilter) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 8
+	// write "OlderThan"
+	err = en.Append(0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteDuration(z.OlderThan)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
+	// write "CreatedBefore"
+	err = en.Append(0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
+	if err != nil {
+		return
+	}
+	if z.CreatedBefore == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteTime(*z.CreatedBefore)
+		if err != nil {
+			err = msgp.WrapError(err, "CreatedBefore")
+			return
+		}
+	}
+	// write "Tags"
+	err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Tags)))
+	if err != nil {
+		err = msgp.WrapError(err, "Tags")
+		return
+	}
+	for za0001 := range z.Tags {
+		err = z.Tags[za0001].EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Tags", za0001)
+			return
+		}
+	}
+	// write "Metadata"
+	err = en.Append(0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.Metadata)))
+	if err != nil {
+		err = msgp.WrapError(err, "Metadata")
+		return
+	}
+	for za0002 := range z.Metadata {
+		err = z.Metadata[za0002].EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Metadata", za0002)
+			return
+		}
+	}
+	// write "Size"
+	err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
+	if err != nil {
+		return
+	}
+	err = z.Size.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	// write "Type"
+	err = en.Append(0xa4, 0x54, 0x79, 0x70, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Type)
+	if err != nil {
+		err = msgp.WrapError(err, "Type")
+		return
+	}
+	// write "Name"
+	err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "Purge"
+	err = en.Append(0xa5, 0x50, 0x75, 0x72, 0x67, 0x65)
+	if err != nil {
+		return
+	}
+	// map header, size 1
+	// write "RetainVersions"
+	err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Purge.RetainVersions)
+	if err != nil {
+		err = msgp.WrapError(err, "Purge", "RetainVersions")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BatchJobExpireFilter) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 8
+	// string "OlderThan"
+	o = append(o, 0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
+	o = msgp.AppendDuration(o, z.OlderThan)
+	// string "CreatedBefore"
+	o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
+	if z.CreatedBefore == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendTime(o, *z.CreatedBefore)
+	}
+	// string "Tags"
+	o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Tags)))
+	for za0001 := range z.Tags {
+		o, err = z.Tags[za0001].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Tags", za0001)
+			return
+		}
+	}
+	// string "Metadata"
+	o = append(o, 0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.Metadata)))
+	for za0002 := range z.Metadata {
+		o, err = z.Metadata[za0002].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Metadata", za0002)
+			return
+		}
+	}
+	// string "Size"
+	o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+	o, err = z.Size.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Size")
+		return
+	}
+	// string "Type"
+	o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = msgp.AppendString(o, z.Type)
+	// string "Name"
+	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "Purge"
+	o = append(o, 0xa5, 0x50, 0x75, 0x72, 0x67, 0x65)
+	// map header, size 1
+	// string "RetainVersions"
+	o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	o = msgp.AppendInt(o, z.Purge.RetainVersions)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "OlderThan":
+			z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "OlderThan")
+				return
+			}
+		case "CreatedBefore":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.CreatedBefore = nil
+			} else {
+				if z.CreatedBefore == nil {
+					z.CreatedBefore = new(time.Time)
+				}
+				*z.CreatedBefore, bts, err = msgp.ReadTimeBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "CreatedBefore")
+					return
+				}
+			}
+		case "Tags":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Tags")
+				return
+			}
+			if cap(z.Tags) >= int(zb0002) {
+				z.Tags = (z.Tags)[:zb0002]
+			} else {
+				z.Tags = make([]BatchJobKV, zb0002)
+			}
+			for za0001 := range z.Tags {
+				bts, err = z.Tags[za0001].UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Tags", za0001)
+					return
+				}
+			}
+		case "Metadata":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Metadata")
+				return
+			}
+			if cap(z.Metadata) >= int(zb0003) {
+				z.Metadata = (z.Metadata)[:zb0003]
+			} else {
+				z.Metadata = make([]BatchJobKV, zb0003)
+			}
+			for za0002 := range z.Metadata {
+				bts, err = z.Metadata[za0002].UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Metadata", za0002)
+					return
+				}
+			}
+		case "Size":
+			bts, err = z.Size.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
+		case "Type":
+			z.Type, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Type")
+				return
+			}
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Purge":
+			var zb0004 uint32
+			zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Purge")
+				return
+			}
+			for zb0004 > 0 {
+				zb0004--
+				field, bts, err = msgp.ReadMapKeyZC(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Purge")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "RetainVersions":
+					z.Purge.RetainVersions, bts, err = msgp.ReadIntBytes(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Purge", "RetainVersions")
+						return
+					}
+				default:
+					bts, err = msgp.Skip(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Purge")
+						return
+					}
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BatchJobExpireFilter) Msgsize() (s int) {
+	s = 1 + 10 + msgp.DurationSize + 14
+	if z.CreatedBefore == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.TimeSize
+	}
+	s += 5 + msgp.ArrayHeaderSize
+	for za0001 := range z.Tags {
+		s += z.Tags[za0001].Msgsize()
+	}
+	s += 9 + msgp.ArrayHeaderSize
+	for za0002 := range z.Metadata {
+		s += z.Metadata[za0002].Msgsize()
+	}
+	s += 5 + z.Size.Msgsize() + 5 + msgp.StringPrefixSize + len(z.Type) + 5 + msgp.StringPrefixSize + len(z.Name) + 6 + 1 + 15 + msgp.IntSize
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobExpirePurge) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "RetainVersions":
+			z.RetainVersions, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "RetainVersions")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobExpirePurge) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 1
+	// write "RetainVersions"
+	err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.RetainVersions)
+	if err != nil {
+		err = msgp.WrapError(err, "RetainVersions")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobExpirePurge) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 1
+	// string "RetainVersions"
+	o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	o = msgp.AppendInt(o, z.RetainVersions)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobExpirePurge) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "RetainVersions":
+			z.RetainVersions, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RetainVersions")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobExpirePurge) Msgsize() (s int) {
+	s = 1 + 15 + msgp.IntSize
+	return
+}

cmd/batch-expire_gen_test.go

@@ -0,0 +1,349 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBatchJobExpire(t *testing.T) {
+	v := BatchJobExpire{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobExpire(t *testing.T) {
+	v := BatchJobExpire{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobExpire Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobExpire{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobExpire(b *testing.B) {
+	v := BatchJobExpire{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobExpireFilter(t *testing.T) {
+	v := BatchJobExpireFilter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobExpireFilter(t *testing.T) {
+	v := BatchJobExpireFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobExpireFilter Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobExpireFilter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobExpireFilter(b *testing.B) {
+	v := BatchJobExpireFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobExpirePurge(t *testing.T) {
+	v := BatchJobExpirePurge{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobExpirePurge(t *testing.T) {
+	v := BatchJobExpirePurge{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobExpirePurge Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobExpirePurge{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobExpirePurge(b *testing.B) {
+	v := BatchJobExpirePurge{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-expire_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestParseBatchJobExpire(t *testing.T) {
+	expireYaml := `
+expire: # Expire objects that match a condition
+  apiVersion: v1
+  bucket: mybucket # Bucket where this batch job will expire matching objects from
+  prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
+  rules:
+    - type: object  # regular objects with zero ore more older versions
+      name: NAME # match object names that satisfy the wildcard expression.
+      olderThan: 70h # match objects older than this value
+      createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+      tags:
+        - key: name
+          value: pick* # match objects with tag 'name', all values starting with 'pick'
+      metadata:
+        - key: content-type
+          value: image/* # match objects with 'content-type', all values starting with 'image/'
+      size:
+        lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
+        greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
+      purge:
+          # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+          # retainVersions: 5 # keep the latest 5 versions of the object.
+  
+    - type: deleted # objects with delete marker as their latest version
+      name: NAME # match object names that satisfy the wildcard expression.
+      olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
+      createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+      purge:
+          # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+          # retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
+
+  notify:
+    endpoint: https://notify.endpoint # notification endpoint to receive job completion status
+    token: Bearer xxxxx # optional authentication token for the notification endpoint
+  
+  retry:
+    attempts: 10 # number of retries for the job before giving up
+    delay: 500ms # least amount of delay between each retry
+`
+	var job BatchJobRequest
+	err := yaml.UnmarshalStrict([]byte(expireYaml), &job)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-expire yaml", err)
+	}
+}

cmd/batch-handlers.go

@@ -28,7 +28,6 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
-	"path"
 	"runtime"
 	"strconv"
 	"strings"
@@ -38,22 +37,26 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/config/batch"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/console"
+	"github.com/minio/pkg/v2/console"
-	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/v2/env"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/policy"
-	"github.com/minio/pkg/workers"
+	"github.com/minio/pkg/v2/workers"
 	"gopkg.in/yaml.v2"
 )
 
+var globalBatchConfig batch.Config
+
 // BatchJobRequest this is an internal data structure not for external consumption.
 type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
@@ -62,6 +65,7 @@ type BatchJobRequest struct {
 	Location  string               `yaml:"-" json:"location"`
 	Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"`
 	KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"`
+	Expire    *BatchJobExpire      `yaml:"expire" json:"expire"`
 	ctx       context.Context      `msg:"-"`
 }
 
@@ -114,7 +118,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 	srcObject := srcObjInfo.Name
 	tgtObject := srcObjInfo.Name
 	if r.Target.Prefix != "" {
-		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+		tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name)
 	}
 
 	versionID := srcObjInfo.VersionID
@@ -163,11 +167,11 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 	}
 	rd, objInfo, _, err := core.GetObject(ctx, srcBucket, srcObject, gopts)
 	if err != nil {
-		return err
+		return ErrorRespToObjectError(err, srcBucket, srcObject, srcObjInfo.VersionID)
 	}
 	defer rd.Close()
 
-	hr, err := hash.NewReader(rd, objInfo.Size, "", "", objInfo.Size)
+	hr, err := hash.NewReader(ctx, rd, objInfo.Size, "", "", objInfo.Size)
 	if err != nil {
 		return err
 	}
@@ -182,7 +186,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 	srcObject := srcObjInfo.Name
 	tgtObject := srcObjInfo.Name
 	if r.Target.Prefix != "" {
-		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+		tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name)
 	}
 	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
 		opts.VersionID = ""
@@ -226,11 +230,11 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 		}
 		rd, objInfo, _, err := c.GetObject(ctx, srcBucket, srcObject, gopts)
 		if err != nil {
-			return err
+			return ErrorRespToObjectError(err, srcBucket, srcObject, srcObjInfo.VersionID)
 		}
 		defer rd.Close()
 
-		hr, err = hash.NewReader(io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size)
+		hr, err = hash.NewReader(ctx, io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size)
 		if err != nil {
 			return err
 		}
@@ -262,6 +266,9 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	if err := ri.load(ctx, api, job); err != nil {
 		return err
 	}
+	if ri.Complete {
+		return nil
+	}
 	globalBatchJobsMetrics.save(job.ID, ri)
 
 	delay := job.Replicate.Flags.Retry.Delay
@@ -270,6 +277,10 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
+	isTags := len(r.Flags.Filter.Tags) != 0
+	isMetadata := len(r.Flags.Filter.Metadata) != 0
+	isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass)
+
 	skip := func(oi ObjectInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
@@ -290,7 +301,8 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			// skip all objects that are created after the specified time.
 			return true
 		}
-		if len(r.Flags.Filter.Tags) > 0 {
+
+		if isTags {
 			// Only parse object tags if tags filter is specified.
 			tagMap := map[string]string{}
 			tagStr := oi.UserTags
@@ -313,7 +325,6 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			return false
 		}
 
-		if len(r.Flags.Filter.Metadata) > 0 {
 		for _, kv := range r.Flags.Filter.Metadata {
 			for k, v := range oi.UserDefined {
 				if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
@@ -326,10 +337,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			}
 		}
 
-			// None of the provided metadata filters match skip the object.
+		// None of the provided filters match
-			return false
-		}
-
 		return false
 	}
 
@@ -384,17 +392,32 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		for obj := range objInfoCh {
 			oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
 			if !minioSrc {
+				// Check if metadata filter was requested and it is expected to have
+				// all user metadata or just storageClass. If its only storageClass
+				// List() already returns relevant information for filter to be applied.
+				if isMetadata && !isStorageClassOnly {
 					oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, miniogo.StatObjectOptions{})
 					if err == nil {
 						oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2)
 					} else {
-					if isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) ||
+						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
-						isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+							logger.LogIf(ctx, err)
+						}
 						continue
 					}
+				}
+				if isTags {
+					tags, err := c.GetObjectTagging(ctx, r.Source.Bucket, obj.Key, minio.GetObjectTaggingOptions{})
+					if err == nil {
+						oi.UserTags = tags.String()
+					} else {
+						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
+							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
 							logger.LogIf(ctx, err)
-					cancel()
+						}
-					return err
+						continue
+					}
 				}
 			}
 			if skip(oi) {
@@ -412,23 +435,27 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			wk.Take()
 			go func() {
 				defer wk.Give()
-				stopFn := globalBatchJobsMetrics.trace(batchReplicationMetricObject, job.ID, attempts, oi)
+				stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts)
 				success := true
 				if err := r.ReplicateFromSource(ctx, api, core, oi, retry); err != nil {
 					// object must be deleted concurrently, allow these failures but do not count them
 					if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
 						return
 					}
-					stopFn(err)
+					stopFn(oi, err)
 					logger.LogIf(ctx, err)
 					success = false
 				} else {
-					stopFn(nil)
+					stopFn(oi, nil)
 				}
 				ri.trackCurrentBucketObject(r.Target.Bucket, oi, success)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
 				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+
+				if wait := globalBatchConfig.ReplicationWait(); wait > 0 {
+					time.Sleep(wait)
+				}
 			}()
 		}
 		wk.Wait()
@@ -482,10 +509,12 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
 		ReplicationStatusInternal: objInfo.ReplicationStatus,
 		UserTags:                  tags.String(),
 	}
+
 	oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
 	for k, v := range objInfo.Metadata {
 		oi.UserDefined[k] = v[0]
 	}
+
 	ce, ok := oi.UserDefined[xhttp.ContentEncoding]
 	if !ok {
 		ce, ok = oi.UserDefined[strings.ToLower(xhttp.ContentEncoding)]
@@ -493,9 +522,74 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
 	if ok {
 		oi.ContentEncoding = ce
 	}
+
+	_, ok = oi.UserDefined[xhttp.AmzStorageClass]
+	if !ok {
+		oi.UserDefined[xhttp.AmzStorageClass] = objInfo.StorageClass
+	}
+
+	for k, v := range objInfo.UserMetadata {
+		oi.UserDefined[k] = v
+	}
+
 	return oi
 }
 
+func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo) error {
+	input := make(chan minio.SnowballObject, 1)
+	opts := minio.SnowballOptions{
+		Opts:     minio.PutObjectOptions{},
+		InMemory: *r.Source.Snowball.InMemory,
+		Compress: *r.Source.Snowball.Compress,
+		SkipErrs: *r.Source.Snowball.SkipErrs,
+	}
+
+	go func() {
+		defer close(input)
+
+		for _, entry := range entries {
+			gr, err := objAPI.GetObjectNInfo(ctx, r.Source.Bucket,
+				entry.Name, nil, nil, ObjectOptions{
+					VersionID: entry.VersionID,
+				})
+			if err != nil {
+				logger.LogIf(ctx, err)
+				continue
+			}
+
+			snowballObj := minio.SnowballObject{
+				// Create path to store objects within the bucket.
+				Key:       entry.Name,
+				Size:      entry.Size,
+				ModTime:   entry.ModTime,
+				VersionID: entry.VersionID,
+				Content:   gr,
+				Headers:   make(http.Header),
+				Close: func() {
+					gr.Close()
+				},
+			}
+
+			opts, err := batchReplicationOpts(ctx, "", gr.ObjInfo)
+			if err != nil {
+				logger.LogIf(ctx, err)
+				continue
+			}
+
+			for k, vals := range opts.Header() {
+				for _, v := range vals {
+					snowballObj.Headers.Add(k, v)
+				}
+			}
+
+			input <- snowballObj
+		}
+	}()
+
+	// Collect and upload all entries.
+	return remoteClnt.PutObjectsSnowball(ctx, r.Target.Bucket, opts, input)
+}
+
 // ReplicateToTarget read from source and replicate to configured target
 func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *miniogo.Core, srcObjInfo ObjectInfo, retry bool) error {
 	srcBucket := r.Source.Bucket
@@ -639,7 +733,12 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 		fileName = batchKeyRotationName
 		version = batchKeyRotateVersionV1
 		format = batchKeyRotationFormat
-
+	case job.Expire != nil:
+		fileName = batchExpireName
+		version = batchExpireVersionV1
+		format = batchExpireFormat
+	default:
+		return errors.New("no supported batch job request specified")
 	}
 	data, err := readConfig(ctx, api, pathJoin(job.Location, fileName))
 	if err != nil {
@@ -656,6 +755,11 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 				if job.KeyRotate.Flags.Retry.Attempts > 0 {
 					ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
 				}
+			case job.Expire != nil:
+				ri.RetryAttempts = batchExpireJobDefaultRetries
+				if job.Expire.Retry.Attempts > 0 {
+					ri.RetryAttempts = job.Expire.Retry.Attempts
+				}
 			}
 			return nil
 		}
@@ -765,6 +869,12 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 			jobTyp = string(job.Type())
 			fileName = batchKeyRotationName
 			ri.Version = batchKeyRotateVersionV1
+		case madmin.BatchJobExpire:
+			format = batchExpireFormat
+			version = batchExpireVersion
+			jobTyp = string(job.Type())
+			fileName = batchExpireName
+			ri.Version = batchExpireVersionV1
 		default:
 			return errInvalidArgument
 		}
@@ -790,7 +900,18 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 	return nil
 }
 
-func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, failed bool) {
+// Note: to be used only with batch jobs that affect multiple versions through
+// a single action. e.g batch-expire has an option to expire all versions of an
+// object which matches the given filters.
+func (ri *batchJobInfo) trackMultipleObjectVersions(bucket string, info ObjectInfo, success bool) {
+	if success {
+		ri.Objects += int64(info.NumVersions)
+	} else {
+		ri.ObjectsFailed += int64(info.NumVersions)
+	}
+}
+
+func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool) {
 	if ri == nil {
 		return
 	}
@@ -800,7 +921,7 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo,
 
 	ri.Bucket = bucket
 	ri.Object = info.Name
-	ri.countItem(info.Size, info.DeleteMarker, failed)
+	ri.countItem(info.Size, info.DeleteMarker, success)
 }
 
 // Start start the batch replication job, resumes if there was a pending job via "job.ID"
@@ -813,6 +934,9 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	if err := ri.load(ctx, api, job); err != nil {
 		return err
 	}
+	if ri.Complete {
+		return nil
+	}
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
 
@@ -822,7 +946,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	skip := func(info FileInfo) (ok bool) {
+	selectObj := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
 			return false
@@ -883,12 +1007,9 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			// None of the provided metadata filters match skip the object.
 			return false
 		}
-		// if one of source or target is non MinIO, just replicate the top most version like `mc mirror`
-		if (r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3) && !info.IsLatest {
-			return false
-		}
 
-		return true
+		// if one of source or target is non MinIO, just replicate the top most version like `mc mirror`
+		return !((r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3) && !info.IsLatest)
 	}
 
 	u, err := url.Parse(r.Target.Endpoint)
@@ -907,8 +1028,73 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	if err != nil {
 		return err
 	}
+
 	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
+	var (
+		walkCh = make(chan ObjectInfo, 100)
+		slowCh = make(chan ObjectInfo, 100)
+	)
+
+	if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
+		go func() {
+			defer close(slowCh)
+
+			// Snowball currently needs the high level minio-go Client, not the Core one
+			cl, err := miniogo.New(u.Host, &miniogo.Options{
+				Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+				Secure:       u.Scheme == "https",
+				Transport:    getRemoteInstanceTransport,
+				BucketLookup: lookupStyle(r.Target.Path),
+			})
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return
+			}
+
+			// Already validated before arriving here
+			smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
+
+			var (
+				obj   = ObjectInfo{}
+				batch = make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
+				valid = true
+			)
+
+			for valid {
+				obj, valid = <-walkCh
+
+				if !valid {
+					goto write
+				}
+
+				if obj.DeleteMarker || !obj.VersionPurgeStatus.Empty() || obj.Size >= int64(smallerThan) {
+					slowCh <- obj
+					continue
+				}
+
+				batch = append(batch, obj)
+
+				if len(batch) < *r.Source.Snowball.Batch {
+					continue
+				}
+
+			write:
+				if len(batch) > 0 {
+					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
+						logger.LogIf(ctx, err)
+						for _, b := range batch {
+							slowCh <- b
+						}
+					}
+					batch = batch[:0]
+				}
+			}
+		}()
+	} else {
+		slowCh = walkCh
+	}
+
 	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
 	if err != nil {
 		return err
@@ -920,6 +1106,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		return err
 	}
 
+	walkQuorum := env.Get("_MINIO_BATCH_REPLICATION_WALK_QUORUM", "strict")
+	if walkQuorum == "" {
+		walkQuorum = "strict"
+	}
+
 	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
@@ -929,10 +1120,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		// one of source/target is s3, skip delete marker and all versions under the same object name.
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
 
-		results := make(chan ObjectInfo, 100)
+		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, walkCh, WalkOptions{
-		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, results, ObjectOptions{
+			Marker:   lastObject,
-			WalkMarker: lastObject,
+			Filter:   selectObj,
-			WalkFilter: skip,
+			AskDisks: walkQuorum,
 		}); err != nil {
 			cancel()
 			// Do not need to retry if we can't list objects on source.
@@ -942,7 +1133,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		prevObj := ""
 
 		skipReplicate := false
-		for result := range results {
+		for result := range slowCh {
 			result := result
 			if result.Name != prevObj {
 				prevObj = result.Name
@@ -955,7 +1146,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			go func() {
 				defer wk.Give()
 
-				stopFn := globalBatchJobsMetrics.trace(batchReplicationMetricObject, job.ID, attempts, result)
+				stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts)
 				success := true
 				if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil {
 					if miniogo.ToErrorResponse(err).Code == "PreconditionFailed" {
@@ -966,16 +1157,20 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 					if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
 						return
 					}
-					stopFn(err)
+					stopFn(result, err)
 					logger.LogIf(ctx, err)
 					success = false
 				} else {
-					stopFn(nil)
+					stopFn(result, nil)
 				}
 				ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
 				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+
+				if wait := globalBatchConfig.ReplicationWait(); wait > 0 {
+					time.Sleep(wait)
+				}
 			}()
 		}
 		wk.Wait()
@@ -1056,6 +1251,9 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	if err := r.Source.Type.Validate(); err != nil {
 		return err
 	}
+	if err := r.Source.Snowball.Validate(); err != nil {
+		return err
+	}
 	if r.Source.Creds.Empty() && r.Target.Creds.Empty() {
 		return errInvalidArgument
 	}
@@ -1177,6 +1375,8 @@ func (j BatchJobRequest) Type() madmin.BatchJobType {
 		return madmin.BatchJobReplicate
 	case j.KeyRotate != nil:
 		return madmin.BatchJobKeyRotate
+	case j.Expire != nil:
+		return madmin.BatchJobExpire
 	}
 	return madmin.BatchJobType("unknown")
 }
@@ -1189,6 +1389,8 @@ func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
 		return j.Replicate.Validate(ctx, j, o)
 	case j.KeyRotate != nil:
 		return j.KeyRotate.Validate(ctx, j, o)
+	case j.Expire != nil:
+		return j.Expire.Validate(ctx, j, o)
 	}
 	return errInvalidArgument
 }
@@ -1199,12 +1401,14 @@ func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
 		deleteConfig(ctx, api, pathJoin(j.Location, batchReplName))
 	case j.KeyRotate != nil:
 		deleteConfig(ctx, api, pathJoin(j.Location, batchKeyRotationName))
+	case j.Expire != nil:
+		deleteConfig(ctx, api, pathJoin(j.Location, batchExpireName))
 	}
 	deleteConfig(ctx, api, j.Location)
 }
 
 func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
-	if j.Replicate == nil && j.KeyRotate == nil {
+	if j.Replicate == nil && j.KeyRotate == nil && j.Expire == nil {
 		return errInvalidArgument
 	}
 
@@ -1258,7 +1462,7 @@ func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (p
 func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListBatchJobsAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListBatchJobsAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1273,7 +1477,7 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	if err := objectAPI.Walk(ctx, minioMetaBucket, batchJobPrefix, resultCh, ObjectOptions{}); err != nil {
+	if err := objectAPI.Walk(ctx, minioMetaBucket, batchJobPrefix, resultCh, WalkOptions{}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1308,7 +1512,7 @@ var errNoSuchJob = errors.New("no such job")
 func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DescribeBatchJobAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DescribeBatchJobAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1343,7 +1547,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.StartBatchJobAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, policy.StartBatchJobAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1365,6 +1569,34 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	// Fill with default values
+	if job.Replicate != nil {
+		if job.Replicate.Source.Snowball.Disable == nil {
+			job.Replicate.Source.Snowball.Disable = ptr(false)
+		}
+		if job.Replicate.Source.Snowball.Batch == nil {
+			job.Replicate.Source.Snowball.Batch = ptr(100)
+		}
+		if job.Replicate.Source.Snowball.InMemory == nil {
+			job.Replicate.Source.Snowball.InMemory = ptr(true)
+		}
+		if job.Replicate.Source.Snowball.Compress == nil {
+			job.Replicate.Source.Snowball.Compress = ptr(false)
+		}
+		if job.Replicate.Source.Snowball.SmallerThan == nil {
+			job.Replicate.Source.Snowball.SmallerThan = ptr("5MiB")
+		}
+		if job.Replicate.Source.Snowball.SkipErrs == nil {
+			job.Replicate.Source.Snowball.SkipErrs = ptr(true)
+		}
+	}
+
+	//  Validate the incoming job request
+	if err := job.Validate(ctx, objectAPI); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	job.ID = fmt.Sprintf("%s:%d", shortuuid.New(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	job.User = user
 	job.Started = time.Now()
@@ -1397,7 +1629,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CancelBatchJobAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.CancelBatchJobAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1462,7 +1694,7 @@ func (j *BatchJobPool) resume() {
 	results := make(chan ObjectInfo, 100)
 	ctx, cancel := context.WithCancel(j.ctx)
 	defer cancel()
-	if err := j.objLayer.Walk(ctx, minioMetaBucket, batchJobPrefix, results, ObjectOptions{}); err != nil {
+	if err := j.objLayer.Walk(ctx, minioMetaBucket, batchJobPrefix, results, WalkOptions{}); err != nil {
 		logger.LogIf(j.ctx, err)
 		return
 	}
@@ -1501,7 +1733,8 @@ func (j *BatchJobPool) AddWorker() {
 			if !ok {
 				return
 			}
-			if job.Replicate != nil {
+			switch {
+			case job.Replicate != nil:
 				if job.Replicate.RemoteToLocal() {
 					if err := job.Replicate.StartFromSource(job.ctx, j.objLayer, *job); err != nil {
 						if !isErrBucketNotFound(err) {
@@ -1521,14 +1754,20 @@ func (j *BatchJobPool) AddWorker() {
 						// Bucket not found proceed to delete such a job.
 					}
 				}
-			}
+			case job.KeyRotate != nil:
-			if job.KeyRotate != nil {
 				if err := job.KeyRotate.Start(job.ctx, j.objLayer, *job); err != nil {
 					if !isErrBucketNotFound(err) {
 						logger.LogIf(j.ctx, err)
 						continue
 					}
 				}
+			case job.Expire != nil:
+				if err := job.Expire.Start(job.ctx, j.objLayer, *job); err != nil {
+					if !isErrBucketNotFound(err) {
+						logger.LogIf(j.ctx, err)
+						continue
+					}
+				}
 			}
 			job.delete(j.ctx, j.objLayer)
 			j.canceler(job.ID, false)
@@ -1606,24 +1845,26 @@ type batchJobMetrics struct {
 type batchJobMetric uint8
 
 const (
-	batchReplicationMetricObject batchJobMetric = iota
+	batchJobMetricReplication batchJobMetric = iota
-	batchKeyRotationMetricObject
+	batchJobMetricKeyRotation
+	batchJobMetricExpire
 )
 
-func batchJobTrace(d batchJobMetric, job string, startTime time.Time, duration time.Duration, info ObjectInfo, attempts int, err error) madmin.TraceInfo {
+func batchJobTrace(d batchJobMetric, job string, startTime time.Time, duration time.Duration, info objTraceInfoer, attempts int, err error) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
 	}
-	jobKind := "batchReplication"
 	traceType := madmin.TraceBatchReplication
-	if d == batchKeyRotationMetricObject {
+	switch d {
-		jobKind = "batchKeyRotation"
+	case batchJobMetricKeyRotation:
 		traceType = madmin.TraceBatchKeyRotation
+	case batchJobMetricExpire:
+		traceType = madmin.TraceBatchExpire
 	}
-	funcName := fmt.Sprintf("%s.%s (job-name=%s)", jobKind, d.String(), job)
+	funcName := fmt.Sprintf("%s() (job-name=%s)", d.String(), job)
 	if attempts > 0 {
-		funcName = fmt.Sprintf("%s.%s (job-name=%s,attempts=%s)", jobKind, d.String(), job, humanize.Ordinal(attempts))
+		funcName = fmt.Sprintf("%s() (job-name=%s,attempts=%s)", d.String(), job, humanize.Ordinal(attempts))
 	}
 	return madmin.TraceInfo{
 		TraceType: traceType,
@@ -1631,55 +1872,65 @@ func batchJobTrace(d batchJobMetric, job string, startTime time.Time, duration t
 		NodeName:  globalLocalNodeName,
 		FuncName:  funcName,
 		Duration:  duration,
-		Path:      info.Name,
+		Path:      fmt.Sprintf("%s (versionID=%s)", info.TraceObjName(), info.TraceVersionID()),
 		Error:     errStr,
 	}
 }
 
+func (ri *batchJobInfo) metric() madmin.JobMetric {
+	m := madmin.JobMetric{
+		JobID:         ri.JobID,
+		JobType:       ri.JobType,
+		StartTime:     ri.StartTime,
+		LastUpdate:    ri.LastUpdate,
+		RetryAttempts: ri.RetryAttempts,
+		Complete:      ri.Complete,
+		Failed:        ri.Failed,
+	}
+
+	switch ri.JobType {
+	case string(madmin.BatchJobReplicate):
+		m.Replicate = &madmin.ReplicateInfo{
+			Bucket:           ri.Bucket,
+			Object:           ri.Object,
+			Objects:          ri.Objects,
+			ObjectsFailed:    ri.ObjectsFailed,
+			BytesTransferred: ri.BytesTransferred,
+			BytesFailed:      ri.BytesFailed,
+		}
+	case string(madmin.BatchJobKeyRotate):
+		m.KeyRotate = &madmin.KeyRotationInfo{
+			Bucket:        ri.Bucket,
+			Object:        ri.Object,
+			Objects:       ri.Objects,
+			ObjectsFailed: ri.ObjectsFailed,
+		}
+	case string(madmin.BatchJobExpire):
+		m.Expired = &madmin.ExpirationInfo{
+			Bucket:        ri.Bucket,
+			Object:        ri.Object,
+			Objects:       ri.Objects,
+			ObjectsFailed: ri.ObjectsFailed,
+		}
+	}
+
+	return m
+}
+
 func (m *batchJobMetrics) report(jobID string) (metrics *madmin.BatchJobMetrics) {
 	metrics = &madmin.BatchJobMetrics{CollectedAt: time.Now(), Jobs: make(map[string]madmin.JobMetric)}
 	m.RLock()
 	defer m.RUnlock()
 
-	match := true
-	for id, job := range m.metrics {
 	if jobID != "" {
-			match = id == jobID
+		if job, ok := m.metrics[jobID]; ok {
+			metrics.Jobs[jobID] = job.metric()
 		}
-		if !match {
+		return metrics
-			continue
 	}
 
-		m := madmin.JobMetric{
+	for id, job := range m.metrics {
-			JobID:         job.JobID,
+		metrics.Jobs[id] = job.metric()
-			JobType:       job.JobType,
-			StartTime:     job.StartTime,
-			LastUpdate:    job.LastUpdate,
-			RetryAttempts: job.RetryAttempts,
-			Complete:      job.Complete,
-			Failed:        job.Failed,
-		}
-
-		switch job.JobType {
-		case string(madmin.BatchJobReplicate):
-			m.Replicate = &madmin.ReplicateInfo{
-				Bucket:           job.Bucket,
-				Object:           job.Object,
-				Objects:          job.Objects,
-				ObjectsFailed:    job.ObjectsFailed,
-				BytesTransferred: job.BytesTransferred,
-				BytesFailed:      job.BytesFailed,
-			}
-		case string(madmin.BatchJobKeyRotate):
-			m.KeyRotate = &madmin.KeyRotationInfo{
-				Bucket:        job.Bucket,
-				Object:        job.Object,
-				Objects:       job.Objects,
-				ObjectsFailed: job.ObjectsFailed,
-			}
-		}
-
-		metrics.Jobs[id] = m
 	}
 	return metrics
 }
@@ -1724,19 +1975,52 @@ func (m *batchJobMetrics) save(jobID string, ri *batchJobInfo) {
 	m.metrics[jobID] = ri.clone()
 }
 
-func (m *batchJobMetrics) trace(d batchJobMetric, job string, attempts int, info ObjectInfo) func(err error) {
+type objTraceInfoer interface {
+	TraceObjName() string
+	TraceVersionID() string
+}
+
+// TraceObjName returns name of object being traced
+func (td ObjectToDelete) TraceObjName() string {
+	return td.ObjectName
+}
+
+// TraceVersionID returns version-id of object being traced
+func (td ObjectToDelete) TraceVersionID() string {
+	return td.VersionID
+}
+
+// TraceObjName returns name of object being traced
+func (oi ObjectInfo) TraceObjName() string {
+	return oi.Name
+}
+
+// TraceVersionID returns version-id of object being traced
+func (oi ObjectInfo) TraceVersionID() string {
+	return oi.VersionID
+}
+
+func (m *batchJobMetrics) trace(d batchJobMetric, job string, attempts int) func(info objTraceInfoer, err error) {
 	startTime := time.Now()
-	return func(err error) {
+	return func(info objTraceInfoer, err error) {
 		duration := time.Since(startTime)
+		if globalTrace.NumSubscribers(madmin.TraceBatch) > 0 {
+			globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err))
+			return
+		}
 		switch d {
-		case batchReplicationMetricObject:
+		case batchJobMetricReplication:
 			if globalTrace.NumSubscribers(madmin.TraceBatchReplication) > 0 {
 				globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err))
 			}
-		case batchKeyRotationMetricObject:
+		case batchJobMetricKeyRotation:
 			if globalTrace.NumSubscribers(madmin.TraceBatchKeyRotation) > 0 {
 				globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err))
 			}
+		case batchJobMetricExpire:
+			if globalTrace.NumSubscribers(madmin.TraceBatchExpire) > 0 {
+				globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err))
+			}
 		}
 	}
 }

cmd/batch-handlers_gen.go

@@ -84,6 +84,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "Expire":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Expire")
+					return
+				}
+				z.Expire = nil
+			} else {
+				if z.Expire == nil {
+					z.Expire = new(BatchJobExpire)
+				}
+				err = z.Expire.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Expire")
+					return
+				}
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -97,9 +115,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 6
+	// map header, size 7
 	// write "ID"
-	err = en.Append(0x86, 0xa2, 0x49, 0x44)
+	err = en.Append(0x87, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -172,15 +190,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "Expire"
+	err = en.Append(0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65)
+	if err != nil {
+		return
+	}
+	if z.Expire == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.Expire.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Expire")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 6
+	// map header, size 7
 	// string "ID"
-	o = append(o, 0x86, 0xa2, 0x49, 0x44)
+	o = append(o, 0x87, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -213,6 +248,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 			return
 		}
 	}
+	// string "Expire"
+	o = append(o, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65)
+	if z.Expire == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.Expire.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Expire")
+			return
+		}
+	}
 	return
 }
 
@@ -292,6 +338,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "Expire":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Expire = nil
+			} else {
+				if z.Expire == nil {
+					z.Expire = new(BatchJobExpire)
+				}
+				bts, err = z.Expire.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Expire")
+					return
+				}
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -318,6 +381,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
 	} else {
 		s += z.KeyRotate.Msgsize()
 	}
+	s += 7
+	if z.Expire == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.Expire.Msgsize()
+	}
 	return
 }
 

cmd/batch-job-common-types.go

@@ -18,10 +18,12 @@
 package cmd
 
 import (
+	"errors"
 	"strings"
 	"time"
 
-	"github.com/minio/pkg/wildcard"
+	"github.com/dustin/go-humanize"
+	"github.com/minio/pkg/v2/wildcard"
 )
 
 //go:generate msgp -file $GOFILE
@@ -81,3 +83,81 @@ func (r BatchJobRetry) Validate() error {
 
 	return nil
 }
+
+//   # snowball based archive transfer is by default enabled when source
+//   # is local and target is remote which is also minio.
+//   snowball:
+//     disable: false # optionally turn-off snowball archive transfer
+//     batch: 100 # upto this many objects per archive
+//     inmemory: true # indicates if the archive must be staged locally or in-memory
+//     compress: true # S2/Snappy compressed archive
+//     smallerThan: 5MiB # create archive for all objects smaller than 5MiB
+//     skipErrs: false # skips any source side read() errors
+
+// BatchJobSnowball describes the snowball feature when replicating objects from a local source to a remote target
+type BatchJobSnowball struct {
+	Disable     *bool   `yaml:"disable" json:"disable"`
+	Batch       *int    `yaml:"batch" json:"batch"`
+	InMemory    *bool   `yaml:"inmemory" json:"inmemory"`
+	Compress    *bool   `yaml:"compress" json:"compress"`
+	SmallerThan *string `yaml:"smallerThan" json:"smallerThan"`
+	SkipErrs    *bool   `yaml:"skipErrs" json:"skipErrs"`
+}
+
+// Validate the snowball parameters in the job description
+func (b BatchJobSnowball) Validate() error {
+	if *b.Batch <= 0 {
+		return errors.New("batch number should be non positive zero")
+	}
+	_, err := humanize.ParseBytes(*b.SmallerThan)
+	return err
+}
+
+// BatchJobSizeFilter supports size based filters - LesserThan and GreaterThan
+type BatchJobSizeFilter struct {
+	UpperBound BatchJobSize `yaml:"lessThan" json:"lessThan"`
+	LowerBound BatchJobSize `yaml:"greaterThan" json:"greaterThan"`
+}
+
+// InRange returns true in the following cases and false otherwise,
+// - sf.LowerBound < sz, when sf.LowerBound alone is specified
+// - sz < sf.UpperBound, when sf.UpperBound alone is specified
+// - sf.LowerBound < sz < sf.UpperBound when both are specified,
+func (sf BatchJobSizeFilter) InRange(sz int64) bool {
+	if sf.UpperBound > 0 && sz > int64(sf.UpperBound) {
+		return false
+	}
+
+	if sf.LowerBound > 0 && sz < int64(sf.LowerBound) {
+		return false
+	}
+	return true
+}
+
+var errInvalidBatchJobSizeFilter = errors.New("invalid batch-job size filter")
+
+// Validate checks if sf is a valid batch-job size filter
+func (sf BatchJobSizeFilter) Validate() error {
+	if sf.LowerBound > 0 && sf.UpperBound > 0 && sf.LowerBound >= sf.UpperBound {
+		return errInvalidBatchJobSizeFilter
+	}
+	return nil
+}
+
+// BatchJobSize supports humanized byte values in yaml files type BatchJobSize uint64
+type BatchJobSize int64
+
+// UnmarshalYAML to parse humanized byte values
+func (s *BatchJobSize) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var batchExpireSz string
+	err := unmarshal(&batchExpireSz)
+	if err != nil {
+		return err
+	}
+	sz, err := humanize.ParseBytes(batchExpireSz)
+	if err != nil {
+		return err
+	}
+	*s = BatchJobSize(sz)
+	return nil
+}

cmd/batch-job-common-types_gen.go

@@ -389,3 +389,666 @@ func (z BatchJobRetry) Msgsize() (s int) {
 	s = 1 + 9 + msgp.IntSize + 6 + msgp.DurationSize
 	return
 }
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobSize) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zb0001 int64
+		zb0001, err = dc.ReadInt64()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = BatchJobSize(zb0001)
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobSize) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteInt64(int64(z))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobSize) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendInt64(o, int64(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobSize) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 int64
+		zb0001, bts, err = msgp.ReadInt64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = BatchJobSize(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobSize) Msgsize() (s int) {
+	s = msgp.Int64Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobSizeFilter) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "UpperBound":
+			{
+				var zb0002 int64
+				zb0002, err = dc.ReadInt64()
+				if err != nil {
+					err = msgp.WrapError(err, "UpperBound")
+					return
+				}
+				z.UpperBound = BatchJobSize(zb0002)
+			}
+		case "LowerBound":
+			{
+				var zb0003 int64
+				zb0003, err = dc.ReadInt64()
+				if err != nil {
+					err = msgp.WrapError(err, "LowerBound")
+					return
+				}
+				z.LowerBound = BatchJobSize(zb0003)
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobSizeFilter) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "UpperBound"
+	err = en.Append(0x82, 0xaa, 0x55, 0x70, 0x70, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(int64(z.UpperBound))
+	if err != nil {
+		err = msgp.WrapError(err, "UpperBound")
+		return
+	}
+	// write "LowerBound"
+	err = en.Append(0xaa, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(int64(z.LowerBound))
+	if err != nil {
+		err = msgp.WrapError(err, "LowerBound")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobSizeFilter) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "UpperBound"
+	o = append(o, 0x82, 0xaa, 0x55, 0x70, 0x70, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64)
+	o = msgp.AppendInt64(o, int64(z.UpperBound))
+	// string "LowerBound"
+	o = append(o, 0xaa, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64)
+	o = msgp.AppendInt64(o, int64(z.LowerBound))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobSizeFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "UpperBound":
+			{
+				var zb0002 int64
+				zb0002, bts, err = msgp.ReadInt64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "UpperBound")
+					return
+				}
+				z.UpperBound = BatchJobSize(zb0002)
+			}
+		case "LowerBound":
+			{
+				var zb0003 int64
+				zb0003, bts, err = msgp.ReadInt64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "LowerBound")
+					return
+				}
+				z.LowerBound = BatchJobSize(zb0003)
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobSizeFilter) Msgsize() (s int) {
+	s = 1 + 11 + msgp.Int64Size + 11 + msgp.Int64Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobSnowball) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Disable":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Disable")
+					return
+				}
+				z.Disable = nil
+			} else {
+				if z.Disable == nil {
+					z.Disable = new(bool)
+				}
+				*z.Disable, err = dc.ReadBool()
+				if err != nil {
+					err = msgp.WrapError(err, "Disable")
+					return
+				}
+			}
+		case "Batch":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Batch")
+					return
+				}
+				z.Batch = nil
+			} else {
+				if z.Batch == nil {
+					z.Batch = new(int)
+				}
+				*z.Batch, err = dc.ReadInt()
+				if err != nil {
+					err = msgp.WrapError(err, "Batch")
+					return
+				}
+			}
+		case "InMemory":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "InMemory")
+					return
+				}
+				z.InMemory = nil
+			} else {
+				if z.InMemory == nil {
+					z.InMemory = new(bool)
+				}
+				*z.InMemory, err = dc.ReadBool()
+				if err != nil {
+					err = msgp.WrapError(err, "InMemory")
+					return
+				}
+			}
+		case "Compress":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Compress")
+					return
+				}
+				z.Compress = nil
+			} else {
+				if z.Compress == nil {
+					z.Compress = new(bool)
+				}
+				*z.Compress, err = dc.ReadBool()
+				if err != nil {
+					err = msgp.WrapError(err, "Compress")
+					return
+				}
+			}
+		case "SmallerThan":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "SmallerThan")
+					return
+				}
+				z.SmallerThan = nil
+			} else {
+				if z.SmallerThan == nil {
+					z.SmallerThan = new(string)
+				}
+				*z.SmallerThan, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "SmallerThan")
+					return
+				}
+			}
+		case "SkipErrs":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "SkipErrs")
+					return
+				}
+				z.SkipErrs = nil
+			} else {
+				if z.SkipErrs == nil {
+					z.SkipErrs = new(bool)
+				}
+				*z.SkipErrs, err = dc.ReadBool()
+				if err != nil {
+					err = msgp.WrapError(err, "SkipErrs")
+					return
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BatchJobSnowball) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 6
+	// write "Disable"
+	err = en.Append(0x86, 0xa7, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65)
+	if err != nil {
+		return
+	}
+	if z.Disable == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteBool(*z.Disable)
+		if err != nil {
+			err = msgp.WrapError(err, "Disable")
+			return
+		}
+	}
+	// write "Batch"
+	err = en.Append(0xa5, 0x42, 0x61, 0x74, 0x63, 0x68)
+	if err != nil {
+		return
+	}
+	if z.Batch == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteInt(*z.Batch)
+		if err != nil {
+			err = msgp.WrapError(err, "Batch")
+			return
+		}
+	}
+	// write "InMemory"
+	err = en.Append(0xa8, 0x49, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79)
+	if err != nil {
+		return
+	}
+	if z.InMemory == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteBool(*z.InMemory)
+		if err != nil {
+			err = msgp.WrapError(err, "InMemory")
+			return
+		}
+	}
+	// write "Compress"
+	err = en.Append(0xa8, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73)
+	if err != nil {
+		return
+	}
+	if z.Compress == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteBool(*z.Compress)
+		if err != nil {
+			err = msgp.WrapError(err, "Compress")
+			return
+		}
+	}
+	// write "SmallerThan"
+	err = en.Append(0xab, 0x53, 0x6d, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
+	if err != nil {
+		return
+	}
+	if z.SmallerThan == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteString(*z.SmallerThan)
+		if err != nil {
+			err = msgp.WrapError(err, "SmallerThan")
+			return
+		}
+	}
+	// write "SkipErrs"
+	err = en.Append(0xa8, 0x53, 0x6b, 0x69, 0x70, 0x45, 0x72, 0x72, 0x73)
+	if err != nil {
+		return
+	}
+	if z.SkipErrs == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = en.WriteBool(*z.SkipErrs)
+		if err != nil {
+			err = msgp.WrapError(err, "SkipErrs")
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BatchJobSnowball) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 6
+	// string "Disable"
+	o = append(o, 0x86, 0xa7, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65)
+	if z.Disable == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendBool(o, *z.Disable)
+	}
+	// string "Batch"
+	o = append(o, 0xa5, 0x42, 0x61, 0x74, 0x63, 0x68)
+	if z.Batch == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendInt(o, *z.Batch)
+	}
+	// string "InMemory"
+	o = append(o, 0xa8, 0x49, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79)
+	if z.InMemory == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendBool(o, *z.InMemory)
+	}
+	// string "Compress"
+	o = append(o, 0xa8, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73)
+	if z.Compress == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendBool(o, *z.Compress)
+	}
+	// string "SmallerThan"
+	o = append(o, 0xab, 0x53, 0x6d, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
+	if z.SmallerThan == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendString(o, *z.SmallerThan)
+	}
+	// string "SkipErrs"
+	o = append(o, 0xa8, 0x53, 0x6b, 0x69, 0x70, 0x45, 0x72, 0x72, 0x73)
+	if z.SkipErrs == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o = msgp.AppendBool(o, *z.SkipErrs)
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobSnowball) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Disable":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Disable = nil
+			} else {
+				if z.Disable == nil {
+					z.Disable = new(bool)
+				}
+				*z.Disable, bts, err = msgp.ReadBoolBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Disable")
+					return
+				}
+			}
+		case "Batch":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Batch = nil
+			} else {
+				if z.Batch == nil {
+					z.Batch = new(int)
+				}
+				*z.Batch, bts, err = msgp.ReadIntBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Batch")
+					return
+				}
+			}
+		case "InMemory":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.InMemory = nil
+			} else {
+				if z.InMemory == nil {
+					z.InMemory = new(bool)
+				}
+				*z.InMemory, bts, err = msgp.ReadBoolBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "InMemory")
+					return
+				}
+			}
+		case "Compress":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Compress = nil
+			} else {
+				if z.Compress == nil {
+					z.Compress = new(bool)
+				}
+				*z.Compress, bts, err = msgp.ReadBoolBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Compress")
+					return
+				}
+			}
+		case "SmallerThan":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.SmallerThan = nil
+			} else {
+				if z.SmallerThan == nil {
+					z.SmallerThan = new(string)
+				}
+				*z.SmallerThan, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "SmallerThan")
+					return
+				}
+			}
+		case "SkipErrs":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.SkipErrs = nil
+			} else {
+				if z.SkipErrs == nil {
+					z.SkipErrs = new(bool)
+				}
+				*z.SkipErrs, bts, err = msgp.ReadBoolBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "SkipErrs")
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BatchJobSnowball) Msgsize() (s int) {
+	s = 1 + 8
+	if z.Disable == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.BoolSize
+	}
+	s += 6
+	if z.Batch == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.IntSize
+	}
+	s += 9
+	if z.InMemory == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.BoolSize
+	}
+	s += 9
+	if z.Compress == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.BoolSize
+	}
+	s += 12
+	if z.SmallerThan == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.StringPrefixSize + len(*z.SmallerThan)
+	}
+	s += 9
+	if z.SkipErrs == nil {
+		s += msgp.NilSize
+	} else {
+		s += msgp.BoolSize
+	}
+	return
+}

cmd/batch-job-common-types_gen_test.go

@@ -347,3 +347,229 @@ func BenchmarkDecodeBatchJobRetry(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalBatchJobSizeFilter(t *testing.T) {
+	v := BatchJobSizeFilter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobSizeFilter(t *testing.T) {
+	v := BatchJobSizeFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobSizeFilter Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobSizeFilter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobSizeFilter(b *testing.B) {
+	v := BatchJobSizeFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobSnowball(t *testing.T) {
+	v := BatchJobSnowball{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobSnowball(t *testing.T) {
+	v := BatchJobSnowball{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobSnowball Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobSnowball{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobSnowball(b *testing.B) {
+	v := BatchJobSnowball{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-job-common-types_test.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestBatchJobSizeInRange(t *testing.T) {
+	tests := []struct {
+		objSize    int64
+		sizeFilter BatchJobSizeFilter
+		want       bool
+	}{
+		{
+			// 1Mib < 2Mib < 10MiB -> in range
+			objSize: 2 << 20,
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+				LowerBound: 1 << 20,
+			},
+			want: true,
+		},
+		{
+			// 2KiB < 1 MiB -> out of range from left
+			objSize: 2 << 10,
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+				LowerBound: 1 << 20,
+			},
+			want: false,
+		},
+		{
+			// 11MiB > 10 MiB -> out of range from right
+			objSize: 11 << 20,
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+				LowerBound: 1 << 20,
+			},
+			want: false,
+		},
+		{
+			//  2MiB < 10MiB -> in range
+			objSize: 2 << 20,
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+			},
+			want: true,
+		},
+		{
+			//  2MiB > 1MiB -> in range
+			objSize: 2 << 20,
+			sizeFilter: BatchJobSizeFilter{
+				LowerBound: 1 << 20,
+			},
+			want: true,
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) {
+			if got := test.sizeFilter.InRange(test.objSize); got != test.want {
+				t.Fatalf("Expected %v but got %v", test.want, got)
+			}
+		})
+	}
+}
+
+func TestBatchJobSizeValidate(t *testing.T) {
+	tests := []struct {
+		sizeFilter BatchJobSizeFilter
+		err        error
+	}{
+		{
+			// Unspecified size filter is a valid filter
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 0,
+				LowerBound: 0,
+			},
+			err: nil,
+		},
+		{
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 0,
+				LowerBound: 1 << 20,
+			},
+			err: nil,
+		},
+		{
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 10 << 20,
+				LowerBound: 0,
+			},
+			err: nil,
+		},
+		{
+			// LowerBound > UpperBound -> empty range
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 1 << 20,
+				LowerBound: 10 << 20,
+			},
+			err: errInvalidBatchJobSizeFilter,
+		},
+		{
+			// LowerBound == UpperBound -> empty range
+			sizeFilter: BatchJobSizeFilter{
+				UpperBound: 1 << 20,
+				LowerBound: 1 << 20,
+			},
+			err: errInvalidBatchJobSizeFilter,
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) {
+			if err := test.sizeFilter.Validate(); err != test.err {
+				t.Fatalf("Expected %v but got %v", test.err, err)
+			}
+		})
+	}
+}

cmd/batch-replicate.go

@@ -155,6 +155,7 @@ type BatchJobReplicateSource struct {
 	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
 	Path     string                        `yaml:"path" json:"path"`
 	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
+	Snowball BatchJobSnowball              `yaml:"snowball" json:"snowball"`
 }
 
 // ValidPath returns true if path is valid

cmd/batch-replicate_gen.go

@@ -469,6 +469,12 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {
 					}
 				}
 			}
+		case "Snowball":
+			err = z.Snowball.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Snowball")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -482,9 +488,9 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 6
+	// map header, size 7
 	// write "Type"
-	err = en.Append(0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	err = en.Append(0x87, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	if err != nil {
 		return
 	}
@@ -569,15 +575,25 @@ func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Creds", "SessionToken")
 		return
 	}
+	// write "Snowball"
+	err = en.Append(0xa8, 0x53, 0x6e, 0x6f, 0x77, 0x62, 0x61, 0x6c, 0x6c)
+	if err != nil {
+		return
+	}
+	err = z.Snowball.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Snowball")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 6
+	// map header, size 7
 	// string "Type"
-	o = append(o, 0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = append(o, 0x87, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	o = msgp.AppendString(o, string(z.Type))
 	// string "Bucket"
 	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
@@ -603,6 +619,13 @@ func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "SessionToken"
 	o = append(o, 0xac, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
 	o = msgp.AppendString(o, z.Creds.SessionToken)
+	// string "Snowball"
+	o = append(o, 0xa8, 0x53, 0x6e, 0x6f, 0x77, 0x62, 0x61, 0x6c, 0x6c)
+	o, err = z.Snowball.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Snowball")
+		return
+	}
 	return
 }
 
@@ -699,6 +722,12 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 					}
 				}
 			}
+		case "Snowball":
+			bts, err = z.Snowball.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Snowball")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -713,7 +742,7 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobReplicateSource) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
+	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken) + 9 + z.Snowball.Msgsize()
 	return
 }
 

cmd/batch-rotate.go

@@ -34,8 +34,8 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/v2/env"
-	"github.com/minio/pkg/workers"
+	"github.com/minio/pkg/v2/workers"
 )
 
 // keyrotate:
@@ -154,7 +154,6 @@ type BatchJobKeyRotateV1 struct {
 	Flags      BatchJobKeyRotateFlags      `yaml:"flags" json:"flags"`
 	Bucket     string                      `yaml:"bucket" json:"bucket"`
 	Prefix     string                      `yaml:"prefix" json:"prefix"`
-	Endpoint   string                      `yaml:"endpoint" json:"endpoint"`
 	Encryption BatchJobKeyRotateEncryption `yaml:"encryption" json:"encryption"`
 }
 
@@ -261,6 +260,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	if err := ri.load(ctx, api, job); err != nil {
 		return err
 	}
+	if ri.Complete {
+		return nil
+	}
 
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
@@ -356,9 +358,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	ctx, cancel := context.WithCancel(ctx)
 
 	results := make(chan ObjectInfo, 100)
-	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, ObjectOptions{
+	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
-		WalkMarker: lastObject,
+		Marker: lastObject,
-		WalkFilter: skip,
+		Filter: skip,
 	}); err != nil {
 		cancel()
 		// Do not need to retry if we can't list objects on source.
@@ -377,14 +379,14 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			defer wk.Give()
 			for attempts := 1; attempts <= retryAttempts; attempts++ {
 				attempts := attempts
-				stopFn := globalBatchJobsMetrics.trace(batchKeyRotationMetricObject, job.ID, attempts, result)
+				stopFn := globalBatchJobsMetrics.trace(batchJobMetricKeyRotation, job.ID, attempts)
 				success := true
 				if err := r.KeyRotate(ctx, api, result); err != nil {
-					stopFn(err)
+					stopFn(result, err)
 					logger.LogIf(ctx, err)
 					success = false
 				} else {
-					stopFn(nil)
+					stopFn(result, nil)
 				}
 				ri.trackCurrentBucketObject(r.Bucket, result, success)
 				ri.RetryAttempts = attempts
@@ -398,6 +400,10 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 					time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
 				}
 			}
+
+			if wait := globalBatchConfig.KeyRotationWait(); wait > 0 {
+				time.Sleep(wait)
+			}
 		}()
 	}
 	wk.Wait()

cmd/batch-rotate_gen.go

@@ -409,12 +409,6 @@ func (z *BatchJobKeyRotateV1) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Prefix")
 				return
 			}
-		case "Endpoint":
-			z.Endpoint, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Endpoint")
-				return
-			}
 		case "Encryption":
 			err = z.Encryption.DecodeMsg(dc)
 			if err != nil {
@@ -434,9 +428,9 @@ func (z *BatchJobKeyRotateV1) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobKeyRotateV1) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 6
+	// map header, size 5
 	// write "APIVersion"
-	err = en.Append(0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	err = en.Append(0x85, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
 	if err != nil {
 		return
 	}
@@ -501,16 +495,6 @@ func (z *BatchJobKeyRotateV1) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Prefix")
 		return
 	}
-	// write "Endpoint"
-	err = en.Append(0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Endpoint)
-	if err != nil {
-		err = msgp.WrapError(err, "Endpoint")
-		return
-	}
 	// write "Encryption"
 	err = en.Append(0xaa, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e)
 	if err != nil {
@@ -527,9 +511,9 @@ func (z *BatchJobKeyRotateV1) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobKeyRotateV1) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 6
+	// map header, size 5
 	// string "APIVersion"
-	o = append(o, 0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	o = append(o, 0x85, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
 	o = msgp.AppendString(o, z.APIVersion)
 	// string "Flags"
 	o = append(o, 0xa5, 0x46, 0x6c, 0x61, 0x67, 0x73)
@@ -561,9 +545,6 @@ func (z *BatchJobKeyRotateV1) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Prefix"
 	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
 	o = msgp.AppendString(o, z.Prefix)
-	// string "Endpoint"
-	o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
-	o = msgp.AppendString(o, z.Endpoint)
 	// string "Encryption"
 	o = append(o, 0xaa, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e)
 	o, err = z.Encryption.MarshalMsg(o)
@@ -651,12 +632,6 @@ func (z *BatchJobKeyRotateV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Prefix")
 				return
 			}
-		case "Endpoint":
-			z.Endpoint, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Endpoint")
-				return
-			}
 		case "Encryption":
 			bts, err = z.Encryption.UnmarshalMsg(bts)
 			if err != nil {
@@ -677,7 +652,7 @@ func (z *BatchJobKeyRotateV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobKeyRotateV1) Msgsize() (s int) {
-	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + 1 + 7 + z.Flags.Filter.Msgsize() + 7 + z.Flags.Notify.Msgsize() + 6 + z.Flags.Retry.Msgsize() + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 11 + z.Encryption.Msgsize()
+	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + 1 + 7 + z.Flags.Filter.Msgsize() + 7 + z.Flags.Notify.Msgsize() + 6 + z.Flags.Retry.Msgsize() + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 11 + z.Encryption.Msgsize()
 	return
 }
 

cmd/batchjobmetric_string.go

@@ -8,13 +8,14 @@ func _() {
 	// An "invalid array index" compiler error signifies that the constant values have changed.
 	// Re-run the stringer command to generate them again.
 	var x [1]struct{}
-	_ = x[batchReplicationMetricObject-0]
+	_ = x[batchJobMetricReplication-0]
-	_ = x[batchKeyRotationMetricObject-1]
+	_ = x[batchJobMetricKeyRotation-1]
+	_ = x[batchJobMetricExpire-2]
 }
 
-const _batchJobMetric_name = "batchReplicationMetricObjectbatchKeyRotationMetricObject"
+const _batchJobMetric_name = "ReplicationKeyRotationExpire"
 
-var _batchJobMetric_index = [...]uint8{0, 28, 56}
+var _batchJobMetric_index = [...]uint8{0, 11, 22, 28}
 
 func (i batchJobMetric) String() string {
 	if i >= batchJobMetric(len(_batchJobMetric_index)-1) {

cmd/bitrot-streaming.go

@@ -95,7 +95,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 	h := algo.New()
 
 	bw := &streamingBitrotWriter{
-		iow:          ioutil.NewDeadlineWriter(w, diskMaxTimeout),
+		iow:          ioutil.NewDeadlineWriter(w, globalDriveConfig.GetMaxTimeout()),
 		closeWithErr: w.CloseWithError,
 		h:            h,
 		shardSize:    shardSize,

cmd/bootstrap-peer-server.go

@@ -20,12 +20,14 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
 	"reflect"
+	"strings"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -33,7 +35,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/rest"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/v2/env"
 )
 
 const (
@@ -82,7 +84,11 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 				ep.Platform, s2.MinioEndpoints[i].Platform)
 		}
 	}
-	if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
+	if reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
+		return nil
+	}
+
+	// Report differences in environment variables.
 	var missing []string
 	var mismatching []string
 	for k, v := range s1.MinioEnv {
@@ -93,12 +99,25 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 			mismatching = append(mismatching, k)
 		}
 	}
+	var extra []string
+	for k := range s2.MinioEnv {
+		_, ok := s1.MinioEnv[k]
+		if !ok {
+			extra = append(extra, k)
+		}
+	}
+	msg := "Expected same MINIO_ environment variables and values across all servers: "
+	if len(missing) > 0 {
+		msg += fmt.Sprintf(`Missing environment values: %v. `, missing)
+	}
 	if len(mismatching) > 0 {
-			return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s / Mismatch environment values: %s`, missing, mismatching)
+		msg += fmt.Sprintf(`Mismatching environment values: %v. `, mismatching)
 	}
-		return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s`, missing)
+	if len(extra) > 0 {
+		msg += fmt.Sprintf(`Extra environment values: %v. `, extra)
 	}
-	return nil
+
+	return errors.New(strings.TrimSpace(msg))
 }
 
 var skipEnvs = map[string]struct{}{

cmd/bucket-encryption-handlers.go

@@ -30,7 +30,7 @@ import (
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 const (

cmd/bucket-handlers.go

@@ -51,6 +51,7 @@ import (
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
@@ -60,9 +61,8 @@ import (
 	"github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
-	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/v2/sync/errgroup"
-	"github.com/minio/pkg/sync/errgroup"
 )
 
 const (
@@ -367,10 +367,10 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		// Use the following trick to filter in place
 		// https://github.com/golang/go/wiki/SliceTricks#filter-in-place
 		for _, bucketInfo := range bucketsInfo {
-			if globalIAMSys.IsAllowed(iampolicy.Args{
+			if globalIAMSys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
 				Groups:          cred.Groups,
-				Action:          iampolicy.ListBucketAction,
+				Action:          policy.ListBucketAction,
 				BucketName:      bucketInfo.Name,
 				ConditionValues: getConditionValues(r, "", cred),
 				IsOwner:         owner,
@@ -379,10 +379,10 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 			}) {
 				bucketsInfo[n] = bucketInfo
 				n++
-			} else if globalIAMSys.IsAllowed(iampolicy.Args{
+			} else if globalIAMSys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
 				Groups:          cred.Groups,
-				Action:          iampolicy.GetBucketLocationAction,
+				Action:          policy.GetBucketLocationAction,
 				BucketName:      bucketInfo.Name,
 				ConditionValues: getConditionValues(r, "", cred),
 				IsOwner:         owner,
@@ -475,9 +475,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	deleteObjectsFn := objectAPI.DeleteObjects
-	if api.CacheAPI() != nil {
-		deleteObjectsFn = api.CacheAPI().DeleteObjects
-	}
 
 	// Return Malformed XML as S3 spec if the number of objects is empty
 	if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList {
@@ -487,9 +484,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	objectsToDelete := map[ObjectToDelete]int{}
 	getObjectInfoFn := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfoFn = api.CacheAPI().GetObjectInfo
-	}
 
 	var (
 		hasLockEnabled bool
@@ -675,6 +669,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			continue
 		}
 
+		defer globalCacheConfig.Delete(bucket, dobj.ObjectName)
+
 		if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) {
 			// copy so we can re-add null ID.
 			dobj := dobj
@@ -717,7 +713,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		if os == nil { // skip objects that weren't deleted due to invalid versionID etc.
 			continue
 		}
-		logger.LogIf(ctx, os.Sweep())
+		os.Sweep()
 	}
 }
 
@@ -770,8 +766,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	if objectLockEnabled {
 		// Creating a bucket with locking requires the user having more permissions
-		for _, action := range []iampolicy.Action{iampolicy.PutBucketObjectLockConfigurationAction, iampolicy.PutBucketVersioningAction} {
+		for _, action := range []policy.Action{policy.PutBucketObjectLockConfigurationAction, policy.PutBucketVersioningAction} {
-			if !globalIAMSys.IsAllowed(iampolicy.Args{
+			if !globalIAMSys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
 				Groups:          cred.Groups,
 				Action:          action,
@@ -795,7 +791,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	// check if client is attempting to create more buckets, complain about it.
 	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
-		logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
+		logger.LogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets))
 	}
 
 	opts := MakeBucketOptions{
@@ -990,7 +986,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		}
 
 		var b bytes.Buffer
-		if fileName == "" {
+		if name != "file" {
 			if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") {
 				dec := json.NewDecoder(part)
 
@@ -1046,11 +1042,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		break
 	}
 
-	if _, ok := formValues["Key"]; !ok {
+	if keyName, ok := formValues["Key"]; !ok {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The name of the uploaded key is missing"))
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
+	} else if fileName == "" && len(keyName) >= 1 {
+		// if we can't get fileName. We use keyName[0] to fileName
+		fileName = keyName[0]
 	}
 
 	if fileName == "" {
@@ -1103,10 +1102,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if len(fanOutEntries) > 0 {
 		// Once signature is validated, check if the user has
 		// explicit permissions for the user.
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.PutObjectFanOutAction,
+			Action:          policy.PutObjectFanOutAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			BucketName:      bucket,
 			ObjectName:      object,
@@ -1119,10 +1118,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	} else {
 		// Once signature is validated, check if the user has
 		// explicit permissions for the user.
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.PutObjectAction,
+			Action:          policy.PutObjectAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			BucketName:      bucket,
 			ObjectName:      object,
@@ -1140,9 +1139,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	hashReader, err := hash.NewReader(reader, fileSize, "", "", fileSize)
+	hashReader, err := hash.NewReader(ctx, reader, fileSize, "", "", fileSize)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1197,7 +1195,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	})
 
 	var opts ObjectOptions
-	opts, err = putOpts(ctx, r, bucket, object, metadata)
+	opts, err = putOptsFromReq(ctx, r, bucket, object, metadata)
 	if err != nil {
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
@@ -1255,7 +1253,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				return
 			}
 			// do not try to verify encrypted content/
-			hashReader, err = hash.NewReader(reader, -1, "", "", -1)
+			hashReader, err = hash.NewReader(ctx, reader, -1, "", "", -1)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -1290,7 +1288,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		// instead of "copying" from source, we need the stream to be seekable
 		// to ensure that we can make fan-out calls concurrently.
 		buf := bytebufferpool.Get()
-		defer bytebufferpool.Put(buf)
+		defer func() {
+			buf.Reset()
+			bytebufferpool.Put(buf)
+		}()
 
 		md5w := md5.New()
 
@@ -1345,6 +1346,22 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 					continue
 				}
 
+				asize, err := objInfo.GetActualSize()
+				if err != nil {
+					asize = objInfo.Size
+				}
+
+				globalCacheConfig.Set(&cache.ObjectInfo{
+					Key:          objInfo.Name,
+					Bucket:       objInfo.Bucket,
+					ETag:         getDecryptedETag(formValues, objInfo, false),
+					ModTime:      objInfo.ModTime,
+					Expires:      objInfo.Expires.UTC().Format(http.TimeFormat),
+					CacheControl: objInfo.CacheControl,
+					Metadata:     cleanReservedKeys(objInfo.UserDefined),
+					Size:         asize,
+				})
+
 				fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
 					Key:          objInfo.Name,
 					ETag:         getDecryptedETag(formValues, objInfo, false),
@@ -1401,10 +1418,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	etag := getDecryptedETag(formValues, objInfo, false)
+
 	// We must not use the http.Header().Set method here because some (broken)
 	// clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive).
 	// Therefore, we have to set the ETag directly as map entry.
-	w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
+	w.Header()[xhttp.ETag] = []string{`"` + etag + `"`}
 
 	// Set the relevant version ID as part of the response header.
 	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
@@ -1415,6 +1434,22 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		w.Header().Set(xhttp.Location, obj)
 	}
 
+	asize, err := objInfo.GetActualSize()
+	if err != nil {
+		asize = objInfo.Size
+	}
+
+	defer globalCacheConfig.Set(&cache.ObjectInfo{
+		Key:          objInfo.Name,
+		Bucket:       objInfo.Bucket,
+		ETag:         etag,
+		ModTime:      objInfo.ModTime,
+		Expires:      objInfo.ExpiresStr(),
+		CacheControl: objInfo.CacheControl,
+		Metadata:     cleanReservedKeys(objInfo.UserDefined),
+		Size:         asize,
+	})
+
 	// Notify object created event.
 	defer sendEvent(eventArgs{
 		EventName:    event.ObjectCreatedPost,
@@ -1498,7 +1533,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	}
 
 	// Check if anonymous (non-owner) has access to list objects.
-	readable := globalPolicySys.IsAllowed(policy.Args{
+	readable := globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
 		Action:          policy.ListBucketAction,
 		BucketName:      bucket,
 		ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
@@ -1506,7 +1541,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	})
 
 	// Check if anonymous (non-owner) has access to upload objects.
-	writable := globalPolicySys.IsAllowed(policy.Args{
+	writable := globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
 		Action:          policy.PutObjectAction,
 		BucketName:      bucket,
 		ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
@@ -1622,10 +1657,12 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Return an error if the bucket does not exist
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil && !forceDelete {
+	if !forceDelete {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
+	}
 
 	// Attempt to delete bucket.
 	if err := objectAPI.DeleteBucket(ctx, bucket, DeleteBucketOptions{

cmd/bucket-lifecycle-audit.go

@@ -25,6 +25,7 @@ type lcEventSrc uint8
 //revive:disable:var-naming Underscores is used here to indicate where common prefix ends and the enumeration name begins
 const (
 	lcEventSrc_None lcEventSrc = iota
+	lcEventSrc_Heal
 	lcEventSrc_Scanner
 	lcEventSrc_Decom
 	lcEventSrc_Rebal

cmd/bucket-lifecycle-handlers.go

@@ -22,12 +22,13 @@ import (
 	"io"
 	"net/http"
 	"strconv"
+	"time"
 
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 const (
@@ -86,6 +87,41 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Create a map of updated set of rules in request
+	updatedRules := make(map[string]lifecycle.Rule, len(bucketLifecycle.Rules))
+	for _, rule := range bucketLifecycle.Rules {
+		updatedRules[rule.ID] = rule
+	}
+
+	// Get list of rules for the bucket from disk
+	meta, err := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	expiryRuleRemoved := false
+	if len(meta.LifecycleConfigXML) > 0 {
+		var lcCfg lifecycle.Lifecycle
+		if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+		for _, rl := range lcCfg.Rules {
+			updRule, ok := updatedRules[rl.ID]
+			// original rule had expiry that is no longer in the new config,
+			// or rule is present but missing expiration flags
+			if (!rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull()) &&
+				(!ok || (updRule.Expiration.IsNull() && updRule.NoncurrentVersionExpiration.IsNull())) {
+				expiryRuleRemoved = true
+			}
+		}
+	}
+
+	if bucketLifecycle.HasExpiry() || expiryRuleRemoved {
+		currtime := time.Now()
+		bucketLifecycle.ExpiryUpdatedAt = &currtime
+	}
+
 	configData, err := xml.Marshal(bucketLifecycle)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -142,6 +178,8 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	// explicitly set ExpiryUpdatedAt nil as its meant for internal consumption only
+	config.ExpiryUpdatedAt = nil
 
 	configData, err := xml.Marshal(config)
 	if err != nil {

cmd/bucket-lifecycle.go

@@ -41,8 +41,10 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
-	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/v2/env"
-	"github.com/minio/pkg/workers"
+	xnet "github.com/minio/pkg/v2/net"
+	"github.com/minio/pkg/v2/workers"
+	"github.com/zeebo/xxh3"
 )
 
 const (
@@ -217,17 +219,24 @@ type transitionState struct {
 	numWorkers int
 	killCh     chan struct{}
 
-	activeTasks int32
+	activeTasks          atomic.Int64
+	missedImmediateTasks atomic.Int64
 
 	lastDayMu    sync.RWMutex
 	lastDayStats map[string]*lastDayTierStats
 }
 
 func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
+	task := transitionTask{objInfo: oi, event: event, src: src}
 	select {
 	case <-t.ctx.Done():
-	case t.transitionCh <- transitionTask{objInfo: oi, event: event, src: src}:
+	case t.transitionCh <- task:
 	default:
+		switch src {
+		case lcEventSrc_s3PutObject, lcEventSrc_s3CopyObject, lcEventSrc_s3CompleteMultipartUpload:
+			// Update missed immediate tasks only for incoming requests.
+			t.missedImmediateTasks.Add(1)
+		}
 	}
 }
 
@@ -237,7 +246,7 @@ var globalTransitionState *transitionState
 // via its Init method.
 func newTransitionState(ctx context.Context) *transitionState {
 	return &transitionState{
-		transitionCh: make(chan transitionTask, 10000),
+		transitionCh: make(chan transitionTask, 100000),
 		ctx:          ctx,
 		killCh:       make(chan struct{}),
 		lastDayStats: make(map[string]*lastDayTierStats),
@@ -262,8 +271,14 @@ func (t *transitionState) PendingTasks() int {
 }
 
 // ActiveTasks returns the number of active (ongoing) ILM transition tasks.
-func (t *transitionState) ActiveTasks() int {
+func (t *transitionState) ActiveTasks() int64 {
-	return int(atomic.LoadInt32(&t.activeTasks))
+	return t.activeTasks.Load()
+}
+
+// MissedImmediateTasks returns the number of tasks - deferred to scanner due
+// to tasks channel being backlogged.
+func (t *transitionState) MissedImmediateTasks() int64 {
+	return t.missedImmediateTasks.Load()
 }
 
 // worker waits for transition tasks
@@ -278,12 +293,14 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 			if !ok {
 				return
 			}
-			atomic.AddInt32(&t.activeTasks, 1)
+			t.activeTasks.Add(1)
 			if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
-				if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+				if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !xnet.IsNetworkOrHostDown(err, false) {
+					if !strings.Contains(err.Error(), "use of closed network connection") {
 						logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
 							task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 					}
+				}
 			} else {
 				ts := tierStats{
 					TotalSize:   uint64(task.objInfo.Size),
@@ -294,7 +311,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 				}
 				t.addLastDayStats(task.event.StorageClass, ts)
 			}
-			atomic.AddInt32(&t.activeTasks, -1)
+			t.activeTasks.Add(-1)
 		}
 	}
 }
@@ -444,7 +461,8 @@ func genTransitionObjName(bucket string) (string, error) {
 		return "", err
 	}
 	us := u.String()
-	obj := fmt.Sprintf("%s/%s/%s/%s/%s", globalDeploymentID, bucket, us[0:2], us[2:4], us)
+	hash := xxh3.HashString(pathJoin(globalDeploymentID(), bucket))
+	obj := fmt.Sprintf("%s/%s/%s/%s", strconv.FormatUint(hash, 16), us[0:2], us[2:4], us)
 	return obj, nil
 }
 
@@ -489,9 +507,13 @@ func auditTierActions(ctx context.Context, tier string, bytes int64) func(err er
 		}
 
 		if err == nil {
-			op.TimeToResponseNS = time.Since(startTime).Nanoseconds()
+			since := time.Since(startTime)
+			op.TimeToResponseNS = since.Nanoseconds()
+			globalTierMetrics.Observe(tier, since)
+			globalTierMetrics.logSuccess(tier)
 		} else {
 			op.Error = err.Error()
+			globalTierMetrics.logFailure(tier)
 		}
 
 		logger.GetReqInfo(ctx).AppendTags("tierStats", op)
@@ -862,6 +884,7 @@ func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
 		UserTags:         oi.UserTags,
 		VersionID:        oi.VersionID,
 		ModTime:          oi.ModTime,
+		Size:             oi.Size,
 		IsLatest:         oi.IsLatest,
 		NumVersions:      oi.NumVersions,
 		DeleteMarker:     oi.DeleteMarker,

cmd/bucket-listobjects-handlers.go

@@ -26,7 +26,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode

cmd/bucket-metadata-sys.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"sync"
@@ -35,8 +36,8 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
-	"github.com/minio/pkg/sync/errgroup"
+	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
 // BucketMetadataSys captures all bucket metadata for a given cluster.
@@ -177,6 +178,40 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
 // Delete delete the bucket metadata for the specified bucket.
 // must be used by all callers instead of using Update() with nil configData.
 func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
+	if configFile == bucketLifecycleConfig {
+		// Get bucket config from current site
+		meta, e := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket)
+		if e != nil && !errors.Is(e, errConfigNotFound) {
+			return updatedAt, e
+		}
+		var expiryRuleRemoved bool
+		if len(meta.LifecycleConfigXML) > 0 {
+			var lcCfg lifecycle.Lifecycle
+			if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil {
+				return updatedAt, err
+			}
+			// find a single expiry rule set the flag
+			for _, rl := range lcCfg.Rules {
+				if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
+					expiryRuleRemoved = true
+					break
+				}
+			}
+		}
+
+		// Form empty ILM details with `ExpiryUpdatedAt` field and save
+		var cfgData []byte
+		if expiryRuleRemoved {
+			var lcCfg lifecycle.Lifecycle
+			currtime := time.Now()
+			lcCfg.ExpiryUpdatedAt = &currtime
+			cfgData, err = xml.Marshal(lcCfg)
+			if err != nil {
+				return updatedAt, err
+			}
+		}
+		return sys.updateAndParse(ctx, bucket, configFile, cfgData, false)
+	}
 	return sys.updateAndParse(ctx, bucket, configFile, nil, false)
 }
 
@@ -267,7 +302,10 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 		}
 		return nil, time.Time{}, err
 	}
-	if meta.lifecycleConfig == nil {
+	// there could be just `ExpiryUpdatedAt` field populated as part
+	// of last delete all. Treat this situation as not lifecycle configuration
+	// available
+	if meta.lifecycleConfig == nil || len(meta.lifecycleConfig.Rules) == 0 {
 		return nil, time.Time{}, BucketLifecycleNotFound{Bucket: bucket}
 	}
 	return meta.lifecycleConfig, meta.LifecycleConfigUpdatedAt, nil
@@ -310,7 +348,7 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
 
 // GetPolicyConfig returns configured bucket policy
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
+func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.BucketPolicy, time.Time, error) {
 	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
@@ -437,21 +475,8 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 	return nil
 }
 
-func (sys *BucketMetadataSys) loadBucketMetadata(ctx context.Context, bucket BucketInfo) error {
-	meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket.Name)
-	if err != nil {
-		return err
-	}
-
-	sys.Lock()
-	sys.metadataMap[bucket.Name] = meta
-	sys.Unlock()
-
-	return nil
-}
-
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo) {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, failedBuckets map[string]struct{}) {
 	g := errgroup.WithNErrs(len(buckets))
 	bucketMetas := make([]BucketMetadata, len(buckets))
 	for index := range buckets {
@@ -491,6 +516,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 
 	for i, meta := range bucketMetas {
 		if errs[i] != nil {
+			if failedBuckets == nil {
+				failedBuckets = make(map[string]struct{})
+			}
+			failedBuckets[buckets[i].Name] = struct{}{}
 			continue
 		}
 		globalEventNotifier.set(buckets[i], meta)   // set notification targets
@@ -498,7 +527,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	}
 }
 
-func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
+func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, failedBuckets map[string]struct{}) {
 	const bucketMetadataRefresh = 15 * time.Minute
 
 	sleeper := newDynamicSleeper(2, 150*time.Millisecond, false)
@@ -513,7 +542,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
 			buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{})
 			if err != nil {
 				logger.LogIf(ctx, err)
-				continue
+				break
 			}
 
 			// Handle if we have some buckets in-memory those are stale.
@@ -525,38 +554,49 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
 			}
 			sys.RemoveStaleBuckets(diskBuckets)
 
-			for _, bucket := range buckets {
+			for i := range buckets {
 				wait := sleeper.Timer(ctx)
 
-				err := sys.loadBucketMetadata(ctx, bucket)
+				meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[i].Name)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					wait() // wait to proceed to next entry.
 					continue
 				}
 
-				wait() // wait to proceed to next entry.
+				sys.Lock()
+				sys.metadataMap[buckets[i].Name] = meta
+				sys.Unlock()
+
+				// Initialize the failed buckets
+				if _, ok := failedBuckets[buckets[i].Name]; ok {
+					globalEventNotifier.set(buckets[i], meta)
+					globalBucketTargetSys.set(buckets[i], meta)
+					delete(failedBuckets, buckets[i].Name)
 				}
 
-			t.Reset(bucketMetadataRefresh)
+				wait() // wait to proceed to next entry.
 			}
 		}
+		t.Reset(bucketMetadataRefresh)
+	}
 }
 
 // Loads bucket metadata for all buckets into BucketMetadataSys.
 func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) {
 	count := 100 // load 100 bucket metadata at a time.
+	failedBuckets := make(map[string]struct{})
 	for {
 		if len(buckets) < count {
-			sys.concurrentLoad(ctx, buckets)
+			sys.concurrentLoad(ctx, buckets, failedBuckets)
 			break
 		}
-		sys.concurrentLoad(ctx, buckets[:count])
+		sys.concurrentLoad(ctx, buckets[:count], failedBuckets)
 		buckets = buckets[count:]
 	}
 
 	if globalIsDistErasure {
-		go sys.refreshBucketsMetadataLoop(ctx)
+		go sys.refreshBucketsMetadataLoop(ctx, failedBuckets)
 	}
 }
 

cmd/bucket-metadata.go

@@ -41,7 +41,7 @@ import (
 	"github.com/minio/minio/internal/fips"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 	"github.com/minio/sio"
 )
 
@@ -91,7 +91,7 @@ type BucketMetadata struct {
 	LifecycleConfigUpdatedAt    time.Time
 
 	// Unexported fields. Must be updated atomically.
-	policyConfig           *policy.Policy
+	policyConfig           *policy.BucketPolicy
 	notificationConfig     *event.Config
 	lifecycleConfig        *lifecycle.Lifecycle
 	objectLockConfig       *objectlock.Config
@@ -217,7 +217,7 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 // The first error encountered is returned.
 func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
 	if len(b.PolicyConfigJSON) != 0 {
-		b.policyConfig, err = policy.ParseConfig(bytes.NewReader(b.PolicyConfigJSON), b.Name)
+		b.policyConfig, err = policy.ParseBucketPolicyConfig(bytes.NewReader(b.PolicyConfigJSON), b.Name)
 		if err != nil {
 			return err
 		}

cmd/bucket-notification-handlers.go

@@ -26,7 +26,7 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 const (

cmd/bucket-object-lock.go

@@ -28,7 +28,7 @@ import (
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // BucketObjectLockSys - map of bucket and retention configuration.

cmd/bucket-policy-handlers.go

@@ -27,7 +27,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 const (
@@ -84,7 +84,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
+	bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "MalformedPolicy",

cmd/bucket-policy-handlers_test.go

@@ -29,31 +29,31 @@ import (
 	"testing"
 
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
-	"github.com/minio/pkg/bucket/policy/condition"
+	"github.com/minio/pkg/v2/policy/condition"
 )
 
-func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {
+func getAnonReadOnlyBucketPolicy(bucketName string) *policy.BucketPolicy {
-	return &policy.Policy{
+	return &policy.BucketPolicy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{
+		Statements: []policy.BPStatement{
-			policy.NewStatement(
+			policy.NewBPStatement(
 				"",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
-				policy.NewResourceSet(policy.NewResource(bucketName, "")),
+				policy.NewResourceSet(policy.NewResource(bucketName)),
 				condition.NewFunctions(),
 			),
 		},
 	}
 }
 
-func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.Policy {
+func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.BucketPolicy {
-	return &policy.Policy{
+	return &policy.BucketPolicy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{
+		Statements: []policy.BPStatement{
-			policy.NewStatement(
+			policy.NewBPStatement(
 				"",
 				policy.Allow,
 				policy.NewPrincipal("*"),
@@ -61,34 +61,34 @@ func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.Policy {
 					policy.GetBucketLocationAction,
 					policy.ListBucketMultipartUploadsAction,
 				),
-				policy.NewResourceSet(policy.NewResource(bucketName, "")),
+				policy.NewResourceSet(policy.NewResource(bucketName)),
 				condition.NewFunctions(),
 			),
 		},
 	}
 }
 
-func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
+func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.BucketPolicy {
-	return &policy.Policy{
+	return &policy.BucketPolicy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{
+		Statements: []policy.BPStatement{
-			policy.NewStatement(
+			policy.NewBPStatement(
 				"",
 				policy.Allow,
 				policy.NewPrincipal("*"),
 				policy.NewActionSet(policy.GetObjectAction),
-				policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+				policy.NewResourceSet(policy.NewResource(bucketName+"/"+prefix)),
 				condition.NewFunctions(),
 			),
 		},
 	}
 }
 
-func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
+func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.BucketPolicy {
-	return &policy.Policy{
+	return &policy.BucketPolicy{
 		Version: policy.DefaultVersion,
-		Statements: []policy.Statement{
+		Statements: []policy.BPStatement{
-			policy.NewStatement(
+			policy.NewBPStatement(
 				"",
 				policy.Allow,
 				policy.NewPrincipal("*"),
@@ -98,7 +98,7 @@ func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
 					policy.ListMultipartUploadPartsAction,
 					policy.PutObjectAction,
 				),
-				policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+				policy.NewResourceSet(policy.NewResource(bucketName+"/"+prefix)),
 				condition.NewFunctions(),
 			),
 		},
@@ -493,13 +493,13 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 		if recV4.Code != testCase.expectedRespStatus {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			var expectedPolicy *policy.Policy
+			var expectedPolicy *policy.BucketPolicy
-			expectedPolicy, err = policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			expectedPolicy, err = policy.ParseBucketPolicyConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}
-			var gotPolicy *policy.Policy
+			var gotPolicy *policy.BucketPolicy
-			gotPolicy, err = policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			gotPolicy, err = policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}
@@ -530,11 +530,11 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		}
 		if recV2.Code == http.StatusOK {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			expectedPolicy, err := policy.ParseBucketPolicyConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}
-			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			gotPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
 			if err != nil {
 				t.Fatalf("unexpected error. %v", err)
 			}

cmd/bucket-policy.go

@@ -32,20 +32,20 @@ import (
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // PolicySys - policy subsystem.
 type PolicySys struct{}
 
 // Get returns stored bucket policy
-func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
+func (sys *PolicySys) Get(bucket string) (*policy.BucketPolicy, error) {
 	policy, _, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
 	return policy, err
 }
 
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
-func (sys *PolicySys) IsAllowed(args policy.Args) bool {
+func (sys *PolicySys) IsAllowed(args policy.BucketPolicyArgs) bool {
 	p, err := sys.Get(args.BucketName)
 	if err == nil {
 		return p.IsAllowed(args)
@@ -121,7 +121,7 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
 		"CurrentTime":      {currTime.Format(time.RFC3339)},
 		"EpochTime":        {strconv.FormatInt(currTime.Unix(), 10)},
 		"SecureTransport":  {strconv.FormatBool(r.TLS != nil)},
-		"SourceIp":         {handlers.GetSourceIP(r)},
+		"SourceIp":         {handlers.GetSourceIPRaw(r)},
 		"UserAgent":        {r.UserAgent()},
 		"Referer":          {r.Referer()},
 		"principaltype":    {principalType},
@@ -137,6 +137,10 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
 	}
 
 	cloneHeader := r.Header.Clone()
+	if v := cloneHeader.Get("x-amz-signature-age"); v != "" {
+		args["signatureAge"] = []string{v}
+		cloneHeader.Del("x-amz-signature-age")
+	}
 
 	if userTags := cloneHeader.Get(xhttp.AmzObjectTagging); userTags != "" {
 		tag, _ := tags.ParseObjectTags(userTags)
@@ -236,7 +240,7 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
 }
 
 // PolicyToBucketAccessPolicy converts a MinIO policy into a minio-go policy data structure.
-func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.BucketAccessPolicy, error) {
+func PolicyToBucketAccessPolicy(bucketPolicy *policy.BucketPolicy) (*miniogopolicy.BucketAccessPolicy, error) {
 	// Return empty BucketAccessPolicy for empty bucket policy.
 	if bucketPolicy == nil {
 		return &miniogopolicy.BucketAccessPolicy{Version: policy.DefaultVersion}, nil
@@ -258,15 +262,15 @@ func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.Buc
 	return &policyInfo, nil
 }
 
-// BucketAccessPolicyToPolicy - converts minio-go/policy.BucketAccessPolicy to policy.Policy.
+// BucketAccessPolicyToPolicy - converts minio-go/policy.BucketAccessPolicy to policy.BucketPolicy.
-func BucketAccessPolicyToPolicy(policyInfo *miniogopolicy.BucketAccessPolicy) (*policy.Policy, error) {
+func BucketAccessPolicyToPolicy(policyInfo *miniogopolicy.BucketAccessPolicy) (*policy.BucketPolicy, error) {
 	data, err := json.Marshal(policyInfo)
 	if err != nil {
 		// This should not happen because policyInfo is valid to convert to JSON data.
 		return nil, err
 	}
 
-	var bucketPolicy policy.Policy
+	var bucketPolicy policy.BucketPolicy
 	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	if err = json.Unmarshal(data, &bucketPolicy); err != nil {
 		// This should not happen because data is valid to JSON data.

cmd/bucket-quota.go

@@ -29,14 +29,12 @@ import (
 )
 
 // BucketQuotaSys - map of bucket and quota configuration.
-type BucketQuotaSys struct {
+type BucketQuotaSys struct{}
-	bucketStorageCache timedValue
-}
 
 // Get - Get quota configuration.
 func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.BucketQuota, error) {
-	qCfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
+	cfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
-	return qCfg, err
+	return cfg, err
 }
 
 // NewBucketQuotaSys returns initialized BucketQuotaSys
@@ -44,16 +42,18 @@ func NewBucketQuotaSys() *BucketQuotaSys {
 	return &BucketQuotaSys{}
 }
 
+var bucketStorageCache timedValue
+
 // Init initialize bucket quota.
 func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
-	sys.bucketStorageCache.Once.Do(func() {
+	bucketStorageCache.Once.Do(func() {
 		// Set this to 10 secs since its enough, as scanner
 		// does not update the bucket usage values frequently.
-		sys.bucketStorageCache.TTL = 10 * time.Second
+		bucketStorageCache.TTL = 10 * time.Second
 		// Rely on older value if usage loading fails from disk.
-		sys.bucketStorageCache.Relax = true
+		bucketStorageCache.Relax = true
-		sys.bucketStorageCache.Update = func() (interface{}, error) {
+		bucketStorageCache.Update = func() (interface{}, error) {
-			ctx, done := context.WithTimeout(context.Background(), 1*time.Second)
+			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
 			defer done()
 
 			return loadDataUsageFromBackend(ctx, objAPI)
@@ -63,16 +63,17 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 
 // GetBucketUsageInfo return bucket usage info for a given bucket
 func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
-	v, err := sys.bucketStorageCache.Get()
+	v, err := bucketStorageCache.Get()
-	if err != nil && v != nil {
+	timedout := OperationTimedOut{}
+	if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &timedout) {
+		if v != nil {
 			logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
-	}
+		} else {
-	if v == nil {
 			logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
 		}
+	}
 
 	var bui BucketUsageInfo
-
 	dui, ok := v.(DataUsageInfo)
 	if ok {
 		bui = dui.BucketsUsage[bucket]
@@ -107,8 +108,16 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
 		return err
 	}
 
-	if q != nil && q.Type == madmin.HardQuota && q.Quota > 0 {
+	var quotaSize uint64
-		if uint64(size) >= q.Quota { // check if file size already exceeds the quota
+	if q != nil && q.Type == madmin.HardQuota {
+		if q.Size > 0 {
+			quotaSize = q.Size
+		} else if q.Quota > 0 {
+			quotaSize = q.Quota
+		}
+	}
+	if quotaSize > 0 {
+		if uint64(size) >= quotaSize { // check if file size already exceeds the quota
 			return BucketQuotaExceeded{Bucket: bucket}
 		}
 
@@ -117,7 +126,7 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
 			return err
 		}
 
-		if bui.Size > 0 && ((bui.Size + uint64(size)) >= q.Quota) {
+		if bui.Size > 0 && ((bui.Size + uint64(size)) >= quotaSize) {
 			return BucketQuotaExceeded{Bucket: bucket}
 		}
 	}

cmd/bucket-replication-handlers.go

@@ -34,7 +34,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
@@ -232,10 +232,10 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	enc := json.NewEncoder(w)
 	stats := globalReplicationStats.getLatestReplicationStats(bucket)
 	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
-	bwMap := bwRpt.BucketStats[bucket]
+	bwMap := bwRpt.BucketStats
 	for arn, st := range stats.ReplicationStats.Stats {
-		if bwMap != nil {
+		for opts, bw := range bwMap {
-			if bw, ok := bwMap[arn]; ok {
+			if opts.ReplicationARN != "" && opts.ReplicationARN == arn {
 				st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond
 				st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond
 				stats.ReplicationStats.Stats[arn] = st
@@ -288,10 +288,10 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsV2Handler(w http.Respons
 	enc := json.NewEncoder(w)
 	stats := globalReplicationStats.getLatestReplicationStats(bucket)
 	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
-	bwMap := bwRpt.BucketStats[bucket]
+	bwMap := bwRpt.BucketStats
 	for arn, st := range stats.ReplicationStats.Stats {
-		if bwMap != nil {
+		for opts, bw := range bwMap {
-			if bw, ok := bwMap[arn]; ok {
+			if opts.ReplicationARN != "" && opts.ReplicationARN == arn {
 				st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond
 				st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond
 				stats.ReplicationStats.Stats[arn] = st
@@ -476,27 +476,17 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	var tgtStats map[string]TargetReplicationResyncStatus
+	brs, err := loadBucketResyncMetadata(ctx, bucket, objectAPI)
-	globalReplicationPool.resyncer.RLock()
-	brs, ok := globalReplicationPool.resyncer.statusMap[bucket]
-	if ok {
-		tgtStats = brs.cloneTgtStats()
-	}
-	globalReplicationPool.resyncer.RUnlock()
-	if !ok {
-		brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
 	if err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
 			Bucket: bucket,
-				Err:    fmt.Errorf("No replication resync status available for %s", arn),
+			Err:    fmt.Errorf("replication resync status not available for %s (%s)", arn, err.Error()),
 		}), r.URL)
 		return
 	}
-		tgtStats = brs.cloneTgtStats()
-	}
 
 	var rinfo ResyncTargetsInfo
-	for tarn, st := range tgtStats {
+	for tarn, st := range brs.TargetsMap {
 		if arn != "" && tarn != arn {
 			continue
 		}
@@ -585,7 +575,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
 		if rule.Status == replication.Disabled {
 			continue
 		}
-		clnt := globalBucketTargetSys.GetRemoteTargetClient(rule.Destination.Bucket)
+		clnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, rule.Destination.Bucket)
 		if clnt == nil {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotFoundError, fmt.Errorf("replication config with rule ID %s has a stale target", rule.ID)), r.URL)
 			return

cmd/bucket-replication-metrics.go

@@ -337,13 +337,13 @@ type SMA struct {
 	filledBuf bool
 }
 
-func newSMA(len int) *SMA {
+func newSMA(ln int) *SMA {
-	if len <= 0 {
+	if ln <= 0 {
-		len = defaultWindowSize
+		ln = defaultWindowSize
 	}
 	return &SMA{
-		buf:    make([]float64, len),
+		buf:    make([]float64, ln),
-		window: len,
+		window: ln,
 		idx:    0,
 	}
 }

cmd/bucket-replication-utils.go

@@ -31,8 +31,8 @@ import (
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 )
 
 //go:generate msgp -file=$GOFILE
@@ -167,7 +167,21 @@ func (ri replicatedInfos) Action() replicationAction {
 var replStatusRegex = regexp.MustCompile(`([^=].*?)=([^,].*?);`)
 
 // TargetReplicationStatus - returns replication status of a target
-func (o *ObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
+func (ri ReplicateObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
+	repStatMatches := replStatusRegex.FindAllStringSubmatch(ri.ReplicationStatusInternal, -1)
+	for _, repStatMatch := range repStatMatches {
+		if len(repStatMatch) != 3 {
+			return
+		}
+		if repStatMatch[1] == arn {
+			return replication.StatusType(repStatMatch[2])
+		}
+	}
+	return
+}
+
+// TargetReplicationStatus - returns replication status of a target
+func (o ObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
 	repStatMatches := replStatusRegex.FindAllStringSubmatch(o.ReplicationStatusInternal, -1)
 	for _, repStatMatch := range repStatMatches {
 		if len(repStatMatch) != 3 {
@@ -185,7 +199,6 @@ type replicateTargetDecision struct {
 	Synchronous bool   // Synchronous replication configured.
 	Arn         string // ARN of replication target
 	ID          string
-	Tgt         *TargetClient
 }
 
 func (t *replicateTargetDecision) String() string {
@@ -207,7 +220,7 @@ type ReplicateDecision struct {
 }
 
 // ReplicateAny returns true if atleast one target qualifies for replication
-func (d *ReplicateDecision) ReplicateAny() bool {
+func (d ReplicateDecision) ReplicateAny() bool {
 	for _, t := range d.targetsMap {
 		if t.Replicate {
 			return true
@@ -217,7 +230,7 @@ func (d *ReplicateDecision) ReplicateAny() bool {
 }
 
 // Synchronous returns true if atleast one target qualifies for synchronous replication
-func (d *ReplicateDecision) Synchronous() bool {
+func (d ReplicateDecision) Synchronous() bool {
 	for _, t := range d.targetsMap {
 		if t.Synchronous {
 			return true
@@ -226,7 +239,7 @@ func (d *ReplicateDecision) Synchronous() bool {
 	return false
 }
 
-func (d *ReplicateDecision) String() string {
+func (d ReplicateDecision) String() string {
 	b := new(bytes.Buffer)
 	for key, value := range d.targetsMap {
 		fmt.Fprintf(b, "%s=%s,", key, value.String())
@@ -243,7 +256,7 @@ func (d *ReplicateDecision) Set(t replicateTargetDecision) {
 }
 
 // PendingStatus returns a stringified representation of internal replication status with all targets marked as `PENDING`
-func (d *ReplicateDecision) PendingStatus() string {
+func (d ReplicateDecision) PendingStatus() string {
 	b := new(bytes.Buffer)
 	for _, k := range d.targetsMap {
 		if k.Replicate {
@@ -259,11 +272,11 @@ type ResyncDecision struct {
 }
 
 // Empty returns true if no targets with resync decision present
-func (r *ResyncDecision) Empty() bool {
+func (r ResyncDecision) Empty() bool {
 	return r.targets == nil
 }
 
-func (r *ResyncDecision) mustResync() bool {
+func (r ResyncDecision) mustResync() bool {
 	for _, v := range r.targets {
 		if v.Replicate {
 			return true
@@ -272,15 +285,12 @@ func (r *ResyncDecision) mustResync() bool {
 	return false
 }
 
-func (r *ResyncDecision) mustResyncTarget(tgtArn string) bool {
+func (r ResyncDecision) mustResyncTarget(tgtArn string) bool {
 	if r.targets == nil {
 		return false
 	}
 	v, ok := r.targets[tgtArn]
-	if ok && v.Replicate {
+	return ok && v.Replicate
-		return true
-	}
-	return false
 }
 
 // ResyncTargetDecision is struct that represents resync decision for this target
@@ -301,35 +311,20 @@ func parseReplicateDecision(ctx context.Context, bucket, s string) (r ReplicateD
 	if len(s) == 0 {
 		return
 	}
-	pairs := strings.Split(s, ",")
+	for _, p := range strings.Split(s, ",") {
-	for _, p := range pairs {
+		if p == "" {
+			continue
+		}
 		slc := strings.Split(p, "=")
 		if len(slc) != 2 {
 			return r, errInvalidReplicateDecisionFormat
 		}
-		tgtStr := strings.TrimPrefix(slc[1], "\"")
+		tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`)
-		tgtStr = strings.TrimSuffix(tgtStr, "\"")
 		tgt := strings.Split(tgtStr, ";")
 		if len(tgt) != 4 {
 			return r, errInvalidReplicateDecisionFormat
 		}
-		var replicate, sync bool
+		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]}
-		var err error
-		replicate, err = strconv.ParseBool(tgt[0])
-		if err != nil {
-			return r, err
-		}
-		sync, err = strconv.ParseBool(tgt[1])
-		if err != nil {
-			return r, err
-		}
-		tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(slc[0])
-		if tgtClnt == nil {
-			// Skip stale targets if any and log them to be missing atleast once.
-			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, slc[0]), slc[0])
-			// We save the targetDecision even when its not configured or stale.
-		}
-		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: replicate, Synchronous: sync, Arn: tgt[2], ID: tgt[3], Tgt: tgtClnt}
 	}
 	return
 }
@@ -496,8 +491,8 @@ func getCompositeVersionPurgeStatus(m map[string]VersionPurgeStatusType) Version
 }
 
 // getHealReplicateObjectInfo returns info needed by heal replication in ReplicateObjectInfo
-func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) ReplicateObjectInfo {
+func getHealReplicateObjectInfo(oi ObjectInfo, rcfg replicationConfig) ReplicateObjectInfo {
-	oi := objInfo.Clone()
+	userDefined := cloneMSS(oi.UserDefined)
 	if rcfg.Config != nil && rcfg.Config.RoleArn != "" {
 		// For backward compatibility of objects pending/failed replication.
 		// Save replication related statuses in the new internal representation for
@@ -508,17 +503,15 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 		if !oi.VersionPurgeStatus.Empty() {
 			oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus)
 		}
-		for k, v := range oi.UserDefined {
+		for k, v := range userDefined {
 			if strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset) {
-				delete(oi.UserDefined, k)
+				delete(userDefined, k)
-				oi.UserDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
+				userDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
 			}
 		}
 	}
-	var dsc ReplicateDecision
-	var tgtStatuses map[string]replication.StatusType
-	var purgeStatuses map[string]VersionPurgeStatusType
 
+	var dsc ReplicateDecision
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
 			ObjectV: ObjectV{
@@ -530,33 +523,45 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
 		}, nil)
 	} else {
-		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{
+		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(userDefined, oi.UserTags, "", replication.HealReplicationType, ObjectOptions{}))
-			UserDefined: oi.UserDefined,
-		}, replication.HealReplicationType, ObjectOptions{}))
 	}
-	tgtStatuses = replicationStatusesMap(oi.ReplicationStatusInternal)
+
-	purgeStatuses = versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
+	tgtStatuses := replicationStatusesMap(oi.ReplicationStatusInternal)
-	existingObjResync := rcfg.Resync(GlobalContext, oi, &dsc, tgtStatuses)
+	purgeStatuses := versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
-	tm, _ := time.Parse(time.RFC3339Nano, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
+	existingObjResync := rcfg.Resync(GlobalContext, oi, dsc, tgtStatuses)
+	tm, _ := time.Parse(time.RFC3339Nano, userDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
+	rstate := oi.ReplicationState()
+	rstate.ReplicateDecisionStr = dsc.String()
+	asz, _ := oi.GetActualSize()
+
 	return ReplicateObjectInfo{
-		ObjectInfo:           oi,
+		Name:                       oi.Name,
+		Size:                       oi.Size,
+		ActualSize:                 asz,
+		Bucket:                     oi.Bucket,
+		VersionID:                  oi.VersionID,
+		ETag:                       oi.ETag,
+		ModTime:                    oi.ModTime,
+		ReplicationStatus:          oi.ReplicationStatus,
+		ReplicationStatusInternal:  oi.ReplicationStatusInternal,
+		DeleteMarker:               oi.DeleteMarker,
+		VersionPurgeStatusInternal: oi.VersionPurgeStatusInternal,
+		VersionPurgeStatus:         oi.VersionPurgeStatus,
+
+		ReplicationState:     rstate,
 		OpType:               replication.HealReplicationType,
 		Dsc:                  dsc,
 		ExistingObjResync:    existingObjResync,
 		TargetStatuses:       tgtStatuses,
 		TargetPurgeStatuses:  purgeStatuses,
 		ReplicationTimestamp: tm,
+		SSEC:                 crypto.SSEC.IsEncrypted(oi.UserDefined),
+		UserTags:             oi.UserTags,
 	}
 }
 
-func (ri *ReplicateObjectInfo) getReplicationState() ReplicationState {
+// ReplicationState - returns replication state using other internal replication metadata in ObjectInfo
-	rs := ri.ObjectInfo.getReplicationState()
+func (o ObjectInfo) ReplicationState() ReplicationState {
-	rs.ReplicateDecisionStr = ri.Dsc.String()
-	return rs
-}
-
-// vID here represents the versionID client specified in request - need to distinguish between delete marker and delete marker deletion
-func (o *ObjectInfo) getReplicationState() ReplicationState {
 	rs := ReplicationState{
 		ReplicationStatusInternal:  o.ReplicationStatusInternal,
 		VersionPurgeStatusInternal: o.VersionPurgeStatusInternal,
@@ -577,7 +582,7 @@ func (o *ObjectInfo) getReplicationState() ReplicationState {
 }
 
 // ReplicationState returns replication state using other internal replication metadata in ObjectToDelete
-func (o *ObjectToDelete) ReplicationState() ReplicationState {
+func (o ObjectToDelete) ReplicationState() ReplicationState {
 	r := ReplicationState{
 		ReplicationStatusInternal:  o.DeleteMarkerReplicationStatus,
 		VersionPurgeStatusInternal: o.VersionPurgeStatuses,

cmd/bucket-replication_test.go

@@ -88,7 +88,7 @@ var replicationConfigTests = []struct {
 func TestReplicationResync(t *testing.T) {
 	ctx := context.Background()
 	for i, test := range replicationConfigTests {
-		if sync := test.rcfg.Resync(ctx, test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
+		if sync := test.rcfg.Resync(ctx, test.info, test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
 			t.Errorf("Test%d (%s): Resync  got %t , want %t", i+1, test.name, sync.mustResync(), test.expectedSync)
 		}
 	}
@@ -283,7 +283,7 @@ var (
 
 func TestReplicationResyncwrapper(t *testing.T) {
 	for i, test := range replicationConfigTests2 {
-		if sync := test.rcfg.resync(test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
+		if sync := test.rcfg.resync(test.info, test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
 			t.Errorf("%s (%s): Replicationresync  got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync.mustResync(), test.expectedSync)
 		}
 	}

cmd/bucket-targets.go

@@ -40,14 +40,29 @@ const (
 	defaultHealthCheckReloadDuration = 30 * time.Minute
 )
 
+type arnTarget struct {
+	Client      *TargetClient
+	lastRefresh time.Time
+}
+
+// arnErrs represents number of errors seen for a ARN and if update is in progress
+// to refresh remote targets from bucket metadata.
+type arnErrs struct {
+	count            int64
+	updateInProgress bool
+	bucket           string
+}
+
 // BucketTargetSys represents bucket targets subsystem
 type BucketTargetSys struct {
 	sync.RWMutex
-	arnRemotesMap map[string]*TargetClient
+	arnRemotesMap map[string]arnTarget
 	targetsMap    map[string][]madmin.BucketTarget
 	hMutex        sync.RWMutex
 	hc            map[string]epHealth
 	hcClient      *madmin.AnonymousClient
+	aMutex        sync.RWMutex
+	arnErrsMap    map[string]arnErrs // map of ARN to error count of failures to get target
 }
 
 type latencyStat struct {
@@ -364,7 +379,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	}
 
 	sys.targetsMap[bucket] = newtgts
-	sys.arnRemotesMap[tgt.Arn] = clnt
+	sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: clnt}
 	sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
 	return nil
 }
@@ -432,11 +447,71 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	return nil
 }
 
+func (sys *BucketTargetSys) markRefreshInProgress(bucket, arn string) {
+	sys.aMutex.Lock()
+	defer sys.aMutex.Unlock()
+	if v, ok := sys.arnErrsMap[arn]; !ok {
+		sys.arnErrsMap[arn] = arnErrs{
+			updateInProgress: true,
+			count:            v.count + 1,
+			bucket:           bucket,
+		}
+	}
+}
+
+func (sys *BucketTargetSys) markRefreshDone(bucket, arn string) {
+	sys.aMutex.Lock()
+	defer sys.aMutex.Unlock()
+	if v, ok := sys.arnErrsMap[arn]; ok {
+		sys.arnErrsMap[arn] = arnErrs{
+			updateInProgress: false,
+			count:            v.count,
+			bucket:           bucket,
+		}
+	}
+}
+
+func (sys *BucketTargetSys) isReloadingTarget(bucket, arn string) bool {
+	sys.aMutex.RLock()
+	defer sys.aMutex.RUnlock()
+	if v, ok := sys.arnErrsMap[arn]; ok {
+		return v.updateInProgress
+	}
+	return false
+}
+
+func (sys *BucketTargetSys) incTargetErr(bucket, arn string) {
+	sys.aMutex.Lock()
+	defer sys.aMutex.Unlock()
+	if v, ok := sys.arnErrsMap[arn]; ok {
+		sys.arnErrsMap[arn] = arnErrs{
+			updateInProgress: v.updateInProgress,
+			count:            v.count + 1,
+		}
+	}
+}
+
 // GetRemoteTargetClient returns minio-go client for replication target instance
-func (sys *BucketTargetSys) GetRemoteTargetClient(arn string) *TargetClient {
+func (sys *BucketTargetSys) GetRemoteTargetClient(bucket, arn string) *TargetClient {
 	sys.RLock()
-	defer sys.RUnlock()
+	tgt := sys.arnRemotesMap[arn]
-	return sys.arnRemotesMap[arn]
+	sys.RUnlock()
+
+	if tgt.Client != nil {
+		return tgt.Client
+	}
+	defer func() { // lazy refresh remote targets
+		if tgt.Client == nil && !sys.isReloadingTarget(bucket, arn) && (tgt.lastRefresh.Equal(timeSentinel) || tgt.lastRefresh.Before(UTCNow().Add(-5*time.Minute))) {
+			tgts, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
+			if err == nil {
+				sys.markRefreshInProgress(bucket, arn)
+				sys.UpdateAllTargets(bucket, tgts)
+				sys.markRefreshDone(bucket, arn)
+			}
+		}
+		sys.incTargetErr(bucket, arn)
+	}()
+	return nil
 }
 
 // GetRemoteBucketTargetByArn returns BucketTarget for a ARN
@@ -457,8 +532,9 @@ func (sys *BucketTargetSys) GetRemoteBucketTargetByArn(ctx context.Context, buck
 // NewBucketTargetSys - creates new replication system.
 func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
 	sys := &BucketTargetSys{
-		arnRemotesMap: make(map[string]*TargetClient),
+		arnRemotesMap: make(map[string]arnTarget),
 		targetsMap:    make(map[string][]madmin.BucketTarget),
+		arnErrsMap:    make(map[string]arnErrs),
 		hc:            make(map[string]epHealth),
 		hcClient:      newHCClient(),
 	}
@@ -489,31 +565,30 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 	defer sys.Unlock()
 
 	// Remove existingtarget and arn association
-	if tgts, ok := sys.targetsMap[bucket]; ok {
+	if stgts, ok := sys.targetsMap[bucket]; ok {
-		for _, t := range tgts {
+		for _, t := range stgts {
 			delete(sys.arnRemotesMap, t.Arn)
 		}
 		delete(sys.targetsMap, bucket)
 	}
 
-	// No need for more if not adding anything
+	if tgts != nil {
-	if tgts == nil || tgts.Empty() {
-		globalBucketMonitor.DeleteBucket(bucket)
-		return
-	}
-
-	if len(tgts.Targets) > 0 {
-		sys.targetsMap[bucket] = tgts.Targets
-	}
 		for _, tgt := range tgts.Targets {
 			tgtClient, err := sys.getRemoteTargetClient(&tgt)
 			if err != nil {
 				continue
 			}
-		sys.arnRemotesMap[tgt.Arn] = tgtClient
+			sys.arnRemotesMap[tgt.Arn] = arnTarget{
+				Client:      tgtClient,
+				lastRefresh: UTCNow(),
+			}
 			sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
 		}
+
+		if !tgts.Empty() {
 			sys.targetsMap[bucket] = tgts.Targets
+		}
+	}
 }
 
 // create minio-go clients for buckets having remote targets
@@ -524,16 +599,13 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 	}
 	sys.Lock()
 	defer sys.Unlock()
-	if len(cfg.Targets) > 0 {
-		sys.targetsMap[bucket.Name] = cfg.Targets
-	}
 	for _, tgt := range cfg.Targets {
 		tgtClient, err := sys.getRemoteTargetClient(&tgt)
 		if err != nil {
 			logger.LogIf(GlobalContext, err)
 			continue
 		}
-		sys.arnRemotesMap[tgt.Arn] = tgtClient
+		sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: tgtClient}
 		sys.updateBandwidthLimit(bucket.Name, tgt.Arn, tgt.BandwidthLimit)
 	}
 	sys.targetsMap[bucket.Name] = cfg.Targets