Add minio_bucket_usage_version_total metric to Prometheus (#17023)

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>

.github/workflows/go-lint.yml

@@ -20,13 +20,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x, 1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest, windows-latest]
-        exclude:
-          - os: ubuntu-latest
-            go-version: 1.19.x
-          - os: windows-latest
-            go-version: 1.20.x
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3

.github/workflows/mint.yml

@@ -0,0 +1,57 @@
+name: Mint Tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  mint-test:
+    runs-on: mint
+    timeout-minutes: 120
+    steps:
+      - name: cleanup #https://github.com/actions/checkout/issues/273
+        run: |
+          sudo -S rm -rf ${GITHUB_WORKSPACE}
+          mkdir ${GITHUB_WORKSPACE}
+      - name: checkout-step
+        uses: actions/checkout@v3
+
+      - name: setup-go-step
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.20.x
+
+      - name: github sha short
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: build-minio
+        run: |
+          make install
+          docker build . -t "minio/minio:${{ steps.vars.outputs.sha_short }}"
+
+      - name: compress and encrypt
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "compress-encrypt" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+
+      - name: multiple pools
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "pools" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+
+      - name: standalone erasure
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+          docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
+
+
+

.github/workflows/mint/minio-compress-encrypt.yaml

@@ -0,0 +1,79 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_COMPRESS: "true"
+    MINIO_COMPRESS_MIMETYPES: "*"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - cdata1-1:/cdata1
+      - cdata1-2:/cdata2
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - cdata2-1:/cdata1
+      - cdata2-2:/cdata2
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - cdata3-1:/cdata1
+      - cdata3-2:/cdata2
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - cdata4-1:/cdata1
+      - cdata4-2:/cdata2
+
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  cdata1-1:
+  cdata1-2:
+  cdata2-1:
+  cdata2-2:
+  cdata3-1:
+  cdata3-2:
+  cdata4-1:
+  cdata4-2:

.github/workflows/mint/minio-erasure.yaml

@@ -0,0 +1,53 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" edata{1...4}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_COMPRESS: "true"
+    MINIO_COMPRESS_MIMETYPES: "*"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - edata1-1:/edata1
+      - edata1-2:/edata2
+      - edata1-3:/edata3
+      - edata1-4:/edata4
+
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-1-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  edata1-1:
+  edata1-2:
+  edata1-3:
+  edata1-4:

.github/workflows/mint/minio-pools.yaml

@@ -0,0 +1,117 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - pdata1-1:/pdata1
+      - pdata1-2:/pdata2
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - pdata2-1:/pdata1
+      - pdata2-2:/pdata2
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - pdata3-1:/pdata1
+      - pdata3-2:/pdata2
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - pdata4-1:/pdata1
+      - pdata4-2:/pdata2
+
+  minio5:
+    <<: *minio-common
+    hostname: minio5
+    volumes:
+      - pdata5-1:/pdata1
+      - pdata5-2:/pdata2
+
+  minio6:
+    <<: *minio-common
+    hostname: minio6
+    volumes:
+      - pdata6-1:/pdata1
+      - pdata6-2:/pdata2
+
+  minio7:
+    <<: *minio-common
+    hostname: minio7
+    volumes:
+      - pdata7-1:/pdata1
+      - pdata7-2:/pdata2
+
+  minio8:
+    <<: *minio-common
+    hostname: minio8
+    volumes:
+      - pdata8-1:/pdata1
+      - pdata8-2:/pdata2
+      
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-8-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+      - minio5
+      - minio6
+      - minio7
+      - minio8
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  pdata1-1:
+  pdata1-2:
+  pdata2-1:
+  pdata2-2:
+  pdata3-1:
+  pdata3-2:
+  pdata4-1:
+  pdata4-2:
+  pdata5-1:
+  pdata5-2:
+  pdata6-1:
+  pdata6-2:
+  pdata7-1:
+  pdata7-2:
+  pdata8-1:
+  pdata8-2:

.github/workflows/mint/nginx-1-node.conf

@@ -0,0 +1,100 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/mint/nginx-4-node.conf

@@ -0,0 +1,106 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/mint/nginx-8-node.conf

@@ -0,0 +1,114 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+        server minio5:9000;
+        server minio6:9000;
+        server minio7:9000;
+        server minio8:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+        server minio5:9001;
+        server minio6:9001;
+        server minio7:9001;
+        server minio8:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/mint/nginx.conf

@@ -0,0 +1,106 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/run-mint.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -ex
+
+export MODE="$1"
+export ACCESS_KEY="$2"
+export SECRET_KEY="$3"
+export JOB_NAME="$4"
+export MINT_MODE="full"
+
+docker system prune -f
+docker volume prune -f
+
+## change working directory
+cd .github/workflows/mint
+
+docker-compose -f minio-${MODE}.yaml up -d
+sleep 5m
+
+docker run --rm --net=host \
+       --name="mint-${MODE}-${JOB_NAME}" \
+       -e SERVER_ENDPOINT="127.0.0.1:9000" \
+       -e ACCESS_KEY="${ACCESS_KEY}" \
+       -e SECRET_KEY="${SECRET_KEY}" \
+       -e ENABLE_HTTPS=0 \
+       -e MINT_MODE="${MINT_MODE}" \
+       docker.io/minio/mint:edge \
+	 aws-sdk-go   \
+	 aws-sdk-java \
+	 aws-sdk-php  \
+	 aws-sdk-ruby \
+	 awscli       \
+	 healthcheck  \
+	 mc           \
+	 minio-go     \
+	 minio-java   \
+	 minio-js     \
+	 minio-py     \
+	 s3cmd        \
+	 s3select     \
+	 versioning
+
+docker-compose -f minio-${MODE}.yaml down || true
+sleep 10s
+
+docker system prune -f || true
+docker volume prune -f || true
+
+## change working directory
+cd ../../../
+

CREDITS

@@ -3037,8 +3037,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 ================================================================
 
-github.com/aymanbagabas/go-osc52
-https://github.com/aymanbagabas/go-osc52
+github.com/aymanbagabas/go-osc52/v2
+https://github.com/aymanbagabas/go-osc52/v2
 ----------------------------------------------------------------
 MIT License
 
@@ -10042,6 +10042,214 @@ https://github.com/google/pprof
 
 ================================================================
 
+github.com/google/s2a-go
+https://github.com/google/s2a-go
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 github.com/google/shlex
 https://github.com/google/shlex
 ----------------------------------------------------------------
@@ -25435,6 +25643,744 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 ================================================================
 
+github.com/shoenig/go-m1cpu
+https://github.com/shoenig/go-m1cpu
+----------------------------------------------------------------
+Mozilla Public License, version 2.0
+
+1. Definitions
+
+1.1. "Contributor"
+
+     means each individual or legal entity that creates, contributes to the
+     creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+
+     means the combination of the Contributions of others (if any) used by a
+     Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+
+     means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+
+     means Source Code Form to which the initial Contributor has attached the
+     notice in Exhibit A, the Executable Form of such Source Code Form, and
+     Modifications of such Source Code Form, in each case including portions
+     thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+     means
+
+     a. that the initial Contributor has attached the notice described in
+        Exhibit B to the Covered Software; or
+
+     b. that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the terms of
+        a Secondary License.
+
+1.6. "Executable Form"
+
+     means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+
+     means a work that combines Covered Software with other material, in a
+     separate file or files, that is not Covered Software.
+
+1.8. "License"
+
+     means this document.
+
+1.9. "Licensable"
+
+     means having the right to grant, to the maximum extent possible, whether
+     at the time of the initial grant or subsequently, any and all of the
+     rights conveyed by this License.
+
+1.10. "Modifications"
+
+     means any of the following:
+
+     a. any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered Software; or
+
+     b. any new file in Source Code Form that contains any Covered Software.
+
+1.11. "Patent Claims" of a Contributor
+
+      means any patent claim(s), including without limitation, method,
+      process, and apparatus claims, in any patent Licensable by such
+      Contributor that would be infringed, but for the grant of the License,
+      by the making, using, selling, offering for sale, having made, import,
+      or transfer of either its Contributions or its Contributor Version.
+
+1.12. "Secondary License"
+
+      means either the GNU General Public License, Version 2.0, the GNU Lesser
+      General Public License, Version 2.1, the GNU Affero General Public
+      License, Version 3.0, or any later versions of those licenses.
+
+1.13. "Source Code Form"
+
+      means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+
+      means an individual or a legal entity exercising rights under this
+      License. For legal entities, "You" includes any entity that controls, is
+      controlled by, or is under common control with You. For purposes of this
+      definition, "control" means (a) the power, direct or indirect, to cause
+      the direction or management of such entity, whether by contract or
+      otherwise, or (b) ownership of more than fifty percent (50%) of the
+      outstanding shares or beneficial ownership of such entity.
+
+
+2. License Grants and Conditions
+
+2.1. Grants
+
+     Each Contributor hereby grants You a world-wide, royalty-free,
+     non-exclusive license:
+
+     a. under intellectual property rights (other than patent or trademark)
+        Licensable by such Contributor to use, reproduce, make available,
+        modify, display, perform, distribute, and otherwise exploit its
+        Contributions, either on an unmodified basis, with Modifications, or
+        as part of a Larger Work; and
+
+     b. under Patent Claims of such Contributor to make, use, sell, offer for
+        sale, have made, import, and otherwise transfer either its
+        Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+     The licenses granted in Section 2.1 with respect to any Contribution
+     become effective for each Contribution on the date the Contributor first
+     distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+     The licenses granted in this Section 2 are the only rights granted under
+     this License. No additional rights or licenses will be implied from the
+     distribution or licensing of Covered Software under this License.
+     Notwithstanding Section 2.1(b) above, no patent license is granted by a
+     Contributor:
+
+     a. for any code that a Contributor has removed from Covered Software; or
+
+     b. for infringements caused by: (i) Your and any other third party's
+        modifications of Covered Software, or (ii) the combination of its
+        Contributions with other software (except as part of its Contributor
+        Version); or
+
+     c. under Patent Claims infringed by Covered Software in the absence of
+        its Contributions.
+
+     This License does not grant any rights in the trademarks, service marks,
+     or logos of any Contributor (except as may be necessary to comply with
+     the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+     No Contributor makes additional grants as a result of Your choice to
+     distribute the Covered Software under a subsequent version of this
+     License (see Section 10.2) or under the terms of a Secondary License (if
+     permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+     Each Contributor represents that the Contributor believes its
+     Contributions are its original creation(s) or it has sufficient rights to
+     grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+     This License is not intended to limit any rights You have under
+     applicable copyright doctrines of fair use, fair dealing, or other
+     equivalents.
+
+2.7. Conditions
+
+     Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
+     Section 2.1.
+
+
+3. Responsibilities
+
+3.1. Distribution of Source Form
+
+     All distribution of Covered Software in Source Code Form, including any
+     Modifications that You create or to which You contribute, must be under
+     the terms of this License. You must inform recipients that the Source
+     Code Form of the Covered Software is governed by the terms of this
+     License, and how they can obtain a copy of this License. You may not
+     attempt to alter or restrict the recipients' rights in the Source Code
+     Form.
+
+3.2. Distribution of Executable Form
+
+     If You distribute Covered Software in Executable Form then:
+
+     a. such Covered Software must also be made available in Source Code Form,
+        as described in Section 3.1, and You must inform recipients of the
+        Executable Form how they can obtain a copy of such Source Code Form by
+        reasonable means in a timely manner, at a charge no more than the cost
+        of distribution to the recipient; and
+
+     b. You may distribute such Executable Form under the terms of this
+        License, or sublicense it under different terms, provided that the
+        license for the Executable Form does not attempt to limit or alter the
+        recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+     You may create and distribute a Larger Work under terms of Your choice,
+     provided that You also comply with the requirements of this License for
+     the Covered Software. If the Larger Work is a combination of Covered
+     Software with a work governed by one or more Secondary Licenses, and the
+     Covered Software is not Incompatible With Secondary Licenses, this
+     License permits You to additionally distribute such Covered Software
+     under the terms of such Secondary License(s), so that the recipient of
+     the Larger Work may, at their option, further distribute the Covered
+     Software under the terms of either this License or such Secondary
+     License(s).
+
+3.4. Notices
+
+     You may not remove or alter the substance of any license notices
+     (including copyright notices, patent notices, disclaimers of warranty, or
+     limitations of liability) contained within the Source Code Form of the
+     Covered Software, except that You may alter any license notices to the
+     extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+     You may choose to offer, and to charge a fee for, warranty, support,
+     indemnity or liability obligations to one or more recipients of Covered
+     Software. However, You may do so only on Your own behalf, and not on
+     behalf of any Contributor. You must make it absolutely clear that any
+     such warranty, support, indemnity, or liability obligation is offered by
+     You alone, and You hereby agree to indemnify every Contributor for any
+     liability incurred by such Contributor as a result of warranty, support,
+     indemnity or liability terms You offer. You may include additional
+     disclaimers of warranty and limitations of liability specific to any
+     jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+
+   If it is impossible for You to comply with any of the terms of this License
+   with respect to some or all of the Covered Software due to statute,
+   judicial order, or regulation then You must: (a) comply with the terms of
+   this License to the maximum extent possible; and (b) describe the
+   limitations and the code they affect. Such description must be placed in a
+   text file included with all distributions of the Covered Software under
+   this License. Except to the extent prohibited by statute or regulation,
+   such description must be sufficiently detailed for a recipient of ordinary
+   skill to be able to understand it.
+
+5. Termination
+
+5.1. The rights granted under this License will terminate automatically if You
+     fail to comply with any of its terms. However, if You become compliant,
+     then the rights granted under this License from a particular Contributor
+     are reinstated (a) provisionally, unless and until such Contributor
+     explicitly and finally terminates Your grants, and (b) on an ongoing
+     basis, if such Contributor fails to notify You of the non-compliance by
+     some reasonable means prior to 60 days after You have come back into
+     compliance. Moreover, Your grants from a particular Contributor are
+     reinstated on an ongoing basis if such Contributor notifies You of the
+     non-compliance by some reasonable means, this is the first time You have
+     received notice of non-compliance with this License from such
+     Contributor, and You become compliant prior to 30 days after Your receipt
+     of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+     infringement claim (excluding declaratory judgment actions,
+     counter-claims, and cross-claims) alleging that a Contributor Version
+     directly or indirectly infringes any patent, then the rights granted to
+     You by any and all Contributors for the Covered Software under Section
+     2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
+     license agreements (excluding distributors and resellers) which have been
+     validly granted by You or Your distributors under this License prior to
+     termination shall survive termination.
+
+6. Disclaimer of Warranty
+
+   Covered Software is provided under this License on an "as is" basis,
+   without warranty of any kind, either expressed, implied, or statutory,
+   including, without limitation, warranties that the Covered Software is free
+   of defects, merchantable, fit for a particular purpose or non-infringing.
+   The entire risk as to the quality and performance of the Covered Software
+   is with You. Should any Covered Software prove defective in any respect,
+   You (not any Contributor) assume the cost of any necessary servicing,
+   repair, or correction. This disclaimer of warranty constitutes an essential
+   part of this License. No use of  any Covered Software is authorized under
+   this License except under this disclaimer.
+
+7. Limitation of Liability
+
+   Under no circumstances and under no legal theory, whether tort (including
+   negligence), contract, or otherwise, shall any Contributor, or anyone who
+   distributes Covered Software as permitted above, be liable to You for any
+   direct, indirect, special, incidental, or consequential damages of any
+   character including, without limitation, damages for lost profits, loss of
+   goodwill, work stoppage, computer failure or malfunction, or any and all
+   other commercial damages or losses, even if such party shall have been
+   informed of the possibility of such damages. This limitation of liability
+   shall not apply to liability for death or personal injury resulting from
+   such party's negligence to the extent applicable law prohibits such
+   limitation. Some jurisdictions do not allow the exclusion or limitation of
+   incidental or consequential damages, so this exclusion and limitation may
+   not apply to You.
+
+8. Litigation
+
+   Any litigation relating to this License may be brought only in the courts
+   of a jurisdiction where the defendant maintains its principal place of
+   business and such litigation shall be governed by laws of that
+   jurisdiction, without reference to its conflict-of-law provisions. Nothing
+   in this Section shall prevent a party's ability to bring cross-claims or
+   counter-claims.
+
+9. Miscellaneous
+
+   This License represents the complete agreement concerning the subject
+   matter hereof. If any provision of this License is held to be
+   unenforceable, such provision shall be reformed only to the extent
+   necessary to make it enforceable. Any law or regulation which provides that
+   the language of a contract shall be construed against the drafter shall not
+   be used to construe this License against a Contributor.
+
+
+10. Versions of the License
+
+10.1. New Versions
+
+      Mozilla Foundation is the license steward. Except as provided in Section
+      10.3, no one other than the license steward has the right to modify or
+      publish new versions of this License. Each version will be given a
+      distinguishing version number.
+
+10.2. Effect of New Versions
+
+      You may distribute the Covered Software under the terms of the version
+      of the License under which You originally received the Covered Software,
+      or under the terms of any subsequent version published by the license
+      steward.
+
+10.3. Modified Versions
+
+      If you create software not governed by this License, and you want to
+      create a new license for such software, you may create and use a
+      modified version of this License if you rename the license and remove
+      any references to the name of the license steward (except to note that
+      such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+      Licenses If You choose to distribute Source Code Form that is
+      Incompatible With Secondary Licenses under the terms of this version of
+      the License, the notice described in Exhibit B of this License must be
+      attached.
+
+Exhibit A - Source Code Form License Notice
+
+      This Source Code Form is subject to the
+      terms of the Mozilla Public License, v.
+      2.0. If a copy of the MPL was not
+      distributed with this file, You can
+      obtain one at
+      http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular file,
+then You may include the notice in a location (such as a LICENSE file in a
+relevant directory) where a recipient would be likely to look for such a
+notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+
+      This Source Code Form is "Incompatible
+      With Secondary Licenses", as defined by
+      the Mozilla Public License, v. 2.0.
+
+
+================================================================
+
+github.com/shoenig/test
+https://github.com/shoenig/test
+----------------------------------------------------------------
+Mozilla Public License, version 2.0
+
+1. Definitions
+
+1.1. "Contributor"
+
+     means each individual or legal entity that creates, contributes to the
+     creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+
+     means the combination of the Contributions of others (if any) used by a
+     Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+
+     means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+
+     means Source Code Form to which the initial Contributor has attached the
+     notice in Exhibit A, the Executable Form of such Source Code Form, and
+     Modifications of such Source Code Form, in each case including portions
+     thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+     means
+
+     a. that the initial Contributor has attached the notice described in
+        Exhibit B to the Covered Software; or
+
+     b. that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the terms of
+        a Secondary License.
+
+1.6. "Executable Form"
+
+     means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+
+     means a work that combines Covered Software with other material, in a
+     separate file or files, that is not Covered Software.
+
+1.8. "License"
+
+     means this document.
+
+1.9. "Licensable"
+
+     means having the right to grant, to the maximum extent possible, whether
+     at the time of the initial grant or subsequently, any and all of the
+     rights conveyed by this License.
+
+1.10. "Modifications"
+
+     means any of the following:
+
+     a. any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered Software; or
+
+     b. any new file in Source Code Form that contains any Covered Software.
+
+1.11. "Patent Claims" of a Contributor
+
+      means any patent claim(s), including without limitation, method,
+      process, and apparatus claims, in any patent Licensable by such
+      Contributor that would be infringed, but for the grant of the License,
+      by the making, using, selling, offering for sale, having made, import,
+      or transfer of either its Contributions or its Contributor Version.
+
+1.12. "Secondary License"
+
+      means either the GNU General Public License, Version 2.0, the GNU Lesser
+      General Public License, Version 2.1, the GNU Affero General Public
+      License, Version 3.0, or any later versions of those licenses.
+
+1.13. "Source Code Form"
+
+      means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+
+      means an individual or a legal entity exercising rights under this
+      License. For legal entities, "You" includes any entity that controls, is
+      controlled by, or is under common control with You. For purposes of this
+      definition, "control" means (a) the power, direct or indirect, to cause
+      the direction or management of such entity, whether by contract or
+      otherwise, or (b) ownership of more than fifty percent (50%) of the
+      outstanding shares or beneficial ownership of such entity.
+
+
+2. License Grants and Conditions
+
+2.1. Grants
+
+     Each Contributor hereby grants You a world-wide, royalty-free,
+     non-exclusive license:
+
+     a. under intellectual property rights (other than patent or trademark)
+        Licensable by such Contributor to use, reproduce, make available,
+        modify, display, perform, distribute, and otherwise exploit its
+        Contributions, either on an unmodified basis, with Modifications, or
+        as part of a Larger Work; and
+
+     b. under Patent Claims of such Contributor to make, use, sell, offer for
+        sale, have made, import, and otherwise transfer either its
+        Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+     The licenses granted in Section 2.1 with respect to any Contribution
+     become effective for each Contribution on the date the Contributor first
+     distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+     The licenses granted in this Section 2 are the only rights granted under
+     this License. No additional rights or licenses will be implied from the
+     distribution or licensing of Covered Software under this License.
+     Notwithstanding Section 2.1(b) above, no patent license is granted by a
+     Contributor:
+
+     a. for any code that a Contributor has removed from Covered Software; or
+
+     b. for infringements caused by: (i) Your and any other third party's
+        modifications of Covered Software, or (ii) the combination of its
+        Contributions with other software (except as part of its Contributor
+        Version); or
+
+     c. under Patent Claims infringed by Covered Software in the absence of
+        its Contributions.
+
+     This License does not grant any rights in the trademarks, service marks,
+     or logos of any Contributor (except as may be necessary to comply with
+     the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+     No Contributor makes additional grants as a result of Your choice to
+     distribute the Covered Software under a subsequent version of this
+     License (see Section 10.2) or under the terms of a Secondary License (if
+     permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+     Each Contributor represents that the Contributor believes its
+     Contributions are its original creation(s) or it has sufficient rights to
+     grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+     This License is not intended to limit any rights You have under
+     applicable copyright doctrines of fair use, fair dealing, or other
+     equivalents.
+
+2.7. Conditions
+
+     Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
+     Section 2.1.
+
+
+3. Responsibilities
+
+3.1. Distribution of Source Form
+
+     All distribution of Covered Software in Source Code Form, including any
+     Modifications that You create or to which You contribute, must be under
+     the terms of this License. You must inform recipients that the Source
+     Code Form of the Covered Software is governed by the terms of this
+     License, and how they can obtain a copy of this License. You may not
+     attempt to alter or restrict the recipients' rights in the Source Code
+     Form.
+
+3.2. Distribution of Executable Form
+
+     If You distribute Covered Software in Executable Form then:
+
+     a. such Covered Software must also be made available in Source Code Form,
+        as described in Section 3.1, and You must inform recipients of the
+        Executable Form how they can obtain a copy of such Source Code Form by
+        reasonable means in a timely manner, at a charge no more than the cost
+        of distribution to the recipient; and
+
+     b. You may distribute such Executable Form under the terms of this
+        License, or sublicense it under different terms, provided that the
+        license for the Executable Form does not attempt to limit or alter the
+        recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+     You may create and distribute a Larger Work under terms of Your choice,
+     provided that You also comply with the requirements of this License for
+     the Covered Software. If the Larger Work is a combination of Covered
+     Software with a work governed by one or more Secondary Licenses, and the
+     Covered Software is not Incompatible With Secondary Licenses, this
+     License permits You to additionally distribute such Covered Software
+     under the terms of such Secondary License(s), so that the recipient of
+     the Larger Work may, at their option, further distribute the Covered
+     Software under the terms of either this License or such Secondary
+     License(s).
+
+3.4. Notices
+
+     You may not remove or alter the substance of any license notices
+     (including copyright notices, patent notices, disclaimers of warranty, or
+     limitations of liability) contained within the Source Code Form of the
+     Covered Software, except that You may alter any license notices to the
+     extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+     You may choose to offer, and to charge a fee for, warranty, support,
+     indemnity or liability obligations to one or more recipients of Covered
+     Software. However, You may do so only on Your own behalf, and not on
+     behalf of any Contributor. You must make it absolutely clear that any
+     such warranty, support, indemnity, or liability obligation is offered by
+     You alone, and You hereby agree to indemnify every Contributor for any
+     liability incurred by such Contributor as a result of warranty, support,
+     indemnity or liability terms You offer. You may include additional
+     disclaimers of warranty and limitations of liability specific to any
+     jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+
+   If it is impossible for You to comply with any of the terms of this License
+   with respect to some or all of the Covered Software due to statute,
+   judicial order, or regulation then You must: (a) comply with the terms of
+   this License to the maximum extent possible; and (b) describe the
+   limitations and the code they affect. Such description must be placed in a
+   text file included with all distributions of the Covered Software under
+   this License. Except to the extent prohibited by statute or regulation,
+   such description must be sufficiently detailed for a recipient of ordinary
+   skill to be able to understand it.
+
+5. Termination
+
+5.1. The rights granted under this License will terminate automatically if You
+     fail to comply with any of its terms. However, if You become compliant,
+     then the rights granted under this License from a particular Contributor
+     are reinstated (a) provisionally, unless and until such Contributor
+     explicitly and finally terminates Your grants, and (b) on an ongoing
+     basis, if such Contributor fails to notify You of the non-compliance by
+     some reasonable means prior to 60 days after You have come back into
+     compliance. Moreover, Your grants from a particular Contributor are
+     reinstated on an ongoing basis if such Contributor notifies You of the
+     non-compliance by some reasonable means, this is the first time You have
+     received notice of non-compliance with this License from such
+     Contributor, and You become compliant prior to 30 days after Your receipt
+     of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+     infringement claim (excluding declaratory judgment actions,
+     counter-claims, and cross-claims) alleging that a Contributor Version
+     directly or indirectly infringes any patent, then the rights granted to
+     You by any and all Contributors for the Covered Software under Section
+     2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
+     license agreements (excluding distributors and resellers) which have been
+     validly granted by You or Your distributors under this License prior to
+     termination shall survive termination.
+
+6. Disclaimer of Warranty
+
+   Covered Software is provided under this License on an "as is" basis,
+   without warranty of any kind, either expressed, implied, or statutory,
+   including, without limitation, warranties that the Covered Software is free
+   of defects, merchantable, fit for a particular purpose or non-infringing.
+   The entire risk as to the quality and performance of the Covered Software
+   is with You. Should any Covered Software prove defective in any respect,
+   You (not any Contributor) assume the cost of any necessary servicing,
+   repair, or correction. This disclaimer of warranty constitutes an essential
+   part of this License. No use of  any Covered Software is authorized under
+   this License except under this disclaimer.
+
+7. Limitation of Liability
+
+   Under no circumstances and under no legal theory, whether tort (including
+   negligence), contract, or otherwise, shall any Contributor, or anyone who
+   distributes Covered Software as permitted above, be liable to You for any
+   direct, indirect, special, incidental, or consequential damages of any
+   character including, without limitation, damages for lost profits, loss of
+   goodwill, work stoppage, computer failure or malfunction, or any and all
+   other commercial damages or losses, even if such party shall have been
+   informed of the possibility of such damages. This limitation of liability
+   shall not apply to liability for death or personal injury resulting from
+   such party's negligence to the extent applicable law prohibits such
+   limitation. Some jurisdictions do not allow the exclusion or limitation of
+   incidental or consequential damages, so this exclusion and limitation may
+   not apply to You.
+
+8. Litigation
+
+   Any litigation relating to this License may be brought only in the courts
+   of a jurisdiction where the defendant maintains its principal place of
+   business and such litigation shall be governed by laws of that
+   jurisdiction, without reference to its conflict-of-law provisions. Nothing
+   in this Section shall prevent a party's ability to bring cross-claims or
+   counter-claims.
+
+9. Miscellaneous
+
+   This License represents the complete agreement concerning the subject
+   matter hereof. If any provision of this License is held to be
+   unenforceable, such provision shall be reformed only to the extent
+   necessary to make it enforceable. Any law or regulation which provides that
+   the language of a contract shall be construed against the drafter shall not
+   be used to construe this License against a Contributor.
+
+
+10. Versions of the License
+
+10.1. New Versions
+
+      Mozilla Foundation is the license steward. Except as provided in Section
+      10.3, no one other than the license steward has the right to modify or
+      publish new versions of this License. Each version will be given a
+      distinguishing version number.
+
+10.2. Effect of New Versions
+
+      You may distribute the Covered Software under the terms of the version
+      of the License under which You originally received the Covered Software,
+      or under the terms of any subsequent version published by the license
+      steward.
+
+10.3. Modified Versions
+
+      If you create software not governed by this License, and you want to
+      create a new license for such software, you may create and use a
+      modified version of this License if you rename the license and remove
+      any references to the name of the license steward (except to note that
+      such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+      Licenses If You choose to distribute Source Code Form that is
+      Incompatible With Secondary Licenses under the terms of this version of
+      the License, the notice described in Exhibit B of this License must be
+      attached.
+
+Exhibit A - Source Code Form License Notice
+
+      This Source Code Form is subject to the
+      terms of the Mozilla Public License, v.
+      2.0. If a copy of the MPL was not
+      distributed with this file, You can
+      obtain one at
+      http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular file,
+then You may include the notice in a location (such as a LICENSE file in a
+relevant directory) where a recipient would be likely to look for such a
+notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+
+      This Source Code Form is "Incompatible
+      With Secondary Licenses", as defined by
+      the Mozilla Public License, v. 2.0.
+
+
+================================================================
+
 github.com/sirupsen/logrus
 https://github.com/sirupsen/logrus
 ----------------------------------------------------------------

Makefile

@@ -37,6 +37,10 @@ check-gen: ## check for updated autogenerated files
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 
 lint: getdeps ## runs golangci-lint suite of linters
+	@echo "Running $@ check"
+	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
+
+lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@echo "Running $@ check"
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
 

README.md

@@ -86,8 +86,6 @@ chmod +x minio
 ./minio server /data
 ```
 
-Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
-
 The following table lists supported architectures. Replace the `wget` URL with the architecture for your Linux host.
 
 | Architecture                   | URL                                                        |

buildscripts/verify-healing-with-root-disks.sh

@@ -45,16 +45,18 @@ function start_minio() {
 
 # Prepare fake disks with losetup
 function prepare_block_devices() {
-        mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
-        for i in 1 2 3 4; do
-                dd if=/dev/zero of=${WORK_DIR}/disks/img.$i bs=1M count=2048
-                mkfs.ext4 -F ${WORK_DIR}/disks/img.$i
-                sudo mknod /dev/minio-loopdisk$i b 7 $[256-$i]
-                sudo losetup /dev/minio-loopdisk$i ${WORK_DIR}/disks/img.$i
-                mkdir -p ${WORK_DIR}/mnt/disk$i/
-                sudo mount /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
-                sudo chown "$(id -u):$(id -g)" /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
-        done
+    set -e
+    mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
+    sudo modprobe loop
+    for i in 1 2 3 4; do
+        dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000
+        device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i})
+        sudo mkfs.ext4 -F ${device}
+        mkdir -p ${WORK_DIR}/mnt/disk${i}/
+        sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/
+        sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/
+    done
+    set +e
 }
 
 # Start a distributed MinIO setup, unmount one disk and check if it is formatted

cmd/admin-handlers-idp-config.go

@@ -86,7 +86,7 @@ func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.Resp
 		if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default {
 			// LDAP does not support multiple configurations. So cfgName must be
 			// empty or `madmin.Default`.
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPNonDefaultConfigName), r.URL)
 			return
 		}
 	}

cmd/admin-handlers-users.go

@@ -582,6 +582,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		AccountStatus: stsAccount.Status,
 		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
+		Expiration:    &stsAccount.Expiration,
 	}
 
 	data, err := json.Marshal(infoResp)
@@ -1956,7 +1957,7 @@ func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 			UserOrGroup: userOrGroup,
 			UserType:    int(userType),
 			IsGroup:     isGroup,
-			Policy:      strings.Join(policiesToDetach, ","),
+			Policy:      newPolicies,
 		},
 		UpdatedAt: updatedAt,
 	}))

cmd/admin-handlers.go

@@ -533,16 +533,19 @@ func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string)
 func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 	now := time.Now().UTC()
 	entryMap := make(map[string]*madmin.LockEntry)
+	toEntry := func(lri lockRequesterInfo) string {
+		return fmt.Sprintf("%s/%s", lri.Name, lri.UID)
+	}
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
 			continue
 		}
 		for k, v := range peerLock.Locks {
 			for _, lockReqInfo := range v {
-				if val, ok := entryMap[lockReqInfo.Name]; ok {
+				if val, ok := entryMap[toEntry(lockReqInfo)]; ok {
 					val.ServerList = append(val.ServerList, peerLock.Addr)
 				} else {
-					entryMap[lockReqInfo.Name] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr)
+					entryMap[toEntry(lockReqInfo)] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr)
 				}
 			}
 		}
@@ -1526,6 +1529,11 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Publish bootstrap events that have already occurred before client could subscribe.
+	if traceOpts.TraceTypes().Contains(madmin.TraceBootstrap) {
+		go globalBootstrapTracer.Publish(ctx, globalTrace)
+	}
+
 	for _, peer := range peers {
 		if peer == nil {
 			continue

cmd/admin-handlers_test.go

@@ -21,10 +21,12 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"sort"
 	"sync"
 	"testing"
 	"time"
@@ -380,3 +382,146 @@ func TestExtractHealInitParams(t *testing.T) {
 		}
 	}
 }
+
+type byResourceUID struct{ madmin.LockEntries }
+
+func (b byResourceUID) Less(i, j int) bool {
+	toUniqLock := func(entry madmin.LockEntry) string {
+		return fmt.Sprintf("%s/%s", entry.Resource, entry.ID)
+	}
+	return toUniqLock(b.LockEntries[i]) < toUniqLock(b.LockEntries[j])
+}
+
+func TestTopLockEntries(t *testing.T) {
+	locksHeld := make(map[string][]lockRequesterInfo)
+	var owners []string
+	for i := 0; i < 4; i++ {
+		owners = append(owners, fmt.Sprintf("node-%d", i))
+	}
+
+	// Simulate DeleteObjects of 10 objects in a single request. i.e same lock
+	// request UID, but 10 different resource names associated with it.
+	var lris []lockRequesterInfo
+	uuid := mustGetUUID()
+	for i := 0; i < 10; i++ {
+		resource := fmt.Sprintf("bucket/delete-object-%d", i)
+		lri := lockRequesterInfo{
+			Name:   resource,
+			Writer: true,
+			UID:    uuid,
+			Owner:  owners[i%len(owners)],
+			Group:  true,
+			Quorum: 3,
+		}
+		lris = append(lris, lri)
+		locksHeld[resource] = []lockRequesterInfo{lri}
+	}
+
+	// Add a few concurrent read locks to the mix
+	for i := 0; i < 50; i++ {
+		resource := fmt.Sprintf("bucket/get-object-%d", i)
+		lri := lockRequesterInfo{
+			Name:   resource,
+			UID:    mustGetUUID(),
+			Owner:  owners[i%len(owners)],
+			Quorum: 2,
+		}
+		lris = append(lris, lri)
+		locksHeld[resource] = append(locksHeld[resource], lri)
+		// concurrent read lock, same resource different uid
+		lri.UID = mustGetUUID()
+		lris = append(lris, lri)
+		locksHeld[resource] = append(locksHeld[resource], lri)
+	}
+
+	var peerLocks []*PeerLocks
+	for _, owner := range owners {
+		peerLocks = append(peerLocks, &PeerLocks{
+			Addr:  owner,
+			Locks: locksHeld,
+		})
+	}
+	var exp madmin.LockEntries
+	for _, lri := range lris {
+		lockType := func(lri lockRequesterInfo) string {
+			if lri.Writer {
+				return "WRITE"
+			}
+			return "READ"
+		}
+		exp = append(exp, madmin.LockEntry{
+			Resource:   lri.Name,
+			Type:       lockType(lri),
+			ServerList: owners,
+			Owner:      lri.Owner,
+			ID:         lri.UID,
+			Quorum:     lri.Quorum,
+		})
+	}
+
+	testCases := []struct {
+		peerLocks []*PeerLocks
+		expected  madmin.LockEntries
+	}{
+		{
+			peerLocks: peerLocks,
+			expected:  exp,
+		},
+	}
+
+	// printEntries := func(entries madmin.LockEntries) {
+	// 	for i, entry := range entries {
+	// 		fmt.Printf("%d: %s %s %s %s %v %d\n", i, entry.Resource, entry.ID, entry.Owner, entry.Type, entry.ServerList, entry.Elapsed)
+	// 	}
+	// }
+
+	check := func(exp, got madmin.LockEntries) (int, bool) {
+		if len(exp) != len(got) {
+			return 0, false
+		}
+		sort.Sort(byResourceUID{exp})
+		sort.Sort(byResourceUID{got})
+		// printEntries(exp)
+		// printEntries(got)
+		for i, e := range exp {
+			if !e.Timestamp.Equal(got[i].Timestamp) {
+				return i, false
+			}
+			// Skip checking elapsed since it's time sensitive.
+			// if e.Elapsed != got[i].Elapsed {
+			// 	return false
+			// }
+			if e.Resource != got[i].Resource {
+				return i, false
+			}
+			if e.Type != got[i].Type {
+				return i, false
+			}
+			if e.Source != got[i].Source {
+				return i, false
+			}
+			if e.Owner != got[i].Owner {
+				return i, false
+			}
+			if e.ID != got[i].ID {
+				return i, false
+			}
+			if len(e.ServerList) != len(got[i].ServerList) {
+				return i, false
+			}
+			for j := range e.ServerList {
+				if e.ServerList[j] != got[i].ServerList[j] {
+					return i, false
+				}
+			}
+		}
+		return 0, true
+	}
+
+	for i, tc := range testCases {
+		got := topLockEntries(tc.peerLocks, false)
+		if idx, ok := check(tc.expected, got); !ok {
+			t.Fatalf("%d: mismatch at %d \n expected %#v but got %#v", i, idx, tc.expected[idx], got[idx])
+		}
+	}
+}

cmd/admin-router.go

@@ -237,6 +237,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
 			gz(httpTraceHdrs(adminAPI.DescribeBatchJob)))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/cancel-job").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.CancelBatchJob)))
 
 		// Bucket migration operations
 		// ExportBucketMetaHandler

cmd/api-errors.go

@@ -38,6 +38,7 @@ import (
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
@@ -281,6 +282,7 @@ const (
 	ErrAdminConfigEnvOverridden
 	ErrAdminConfigDuplicateKeys
 	ErrAdminConfigInvalidIDPType
+	ErrAdminConfigLDAPNonDefaultConfigName
 	ErrAdminConfigLDAPValidation
 	ErrAdminConfigIDPCfgNameAlreadyExists
 	ErrAdminConfigIDPCfgNameDoesNotExist
@@ -1333,6 +1335,11 @@ var errorCodes = errorCodeMap{
 		Description:    fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigLDAPNonDefaultConfigName: {
+		Code:           "XMinioAdminConfigLDAPNonDefaultConfigName",
+		Description:    "Only a single LDAP configuration is supported - config name must be empty or `_`",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigLDAPValidation: {
 		Code:           "XMinioAdminConfigLDAPValidation",
 		Description:    "LDAP Configuration validation failed",
@@ -2311,6 +2318,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// any underlying errors if possible depending on
 		// their internal error types.
 		switch e := err.(type) {
+		case kms.Error:
+			apiErr = APIError{
+				Description:    e.Err.Error(),
+				Code:           e.APICode,
+				HTTPStatusCode: e.HTTPStatusCode,
+			}
 		case batchReplicationJobError:
 			apiErr = APIError(e)
 		case InvalidArgument:

cmd/api-response.go

@@ -35,6 +35,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 	xxml "github.com/minio/xxml"
 )
 
@@ -359,6 +360,7 @@ type Object struct {
 
 	// UserMetadata user-defined metadata
 	UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
+	UserTags     string    `xml:"UserTags,omitempty"`
 }
 
 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -492,7 +494,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 }
 
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
-func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
+func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
 
 	owner := Owner{
@@ -500,11 +502,19 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		DisplayName: "minio",
 	}
 	data := ListVersionsResponse{}
+	var lastObjMetaName string
+	var tagErr, metaErr APIErrorCode = -1, -1
 
 	for _, object := range resp.Objects {
 		if object.Name == "" {
 			continue
 		}
+		// Cache checks for the same object
+		if metadata != nil && lastObjMetaName != object.Name {
+			tagErr = metadata(object.Name, policy.GetObjectTaggingAction)
+			metaErr = metadata(object.Name, policy.GetObjectAction)
+			lastObjMetaName = object.Name
+		}
 		content := ObjectVersion{}
 		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
@@ -517,6 +527,32 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		} else {
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
+		if tagErr == ErrNone {
+			content.UserTags = object.UserTags
+		}
+		if metaErr == ErrNone {
+			content.UserMetadata = &Metadata{}
+			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
+			case crypto.S3:
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+			case crypto.S3KMS:
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
+			case crypto.SSEC:
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
+			}
+			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+					// Do not need to send any internal metadata
+					// values to client.
+					continue
+				}
+				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+					continue
+				}
+				content.UserMetadata.Set(k, v)
+			}
+		}
 		content.Owner = owner
 		content.VersionID = object.VersionID
 		if content.VersionID == "" {
@@ -600,7 +636,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata metaCheckFn) ListObjectsV2Response {
 	contents := make([]Object, 0, len(objects))
 	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
@@ -625,27 +661,32 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
 		content.Owner = owner
-		if metadata {
-			content.UserMetadata = &Metadata{}
-			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
-			case crypto.S3:
-				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
-			case crypto.S3KMS:
-				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
-			case crypto.SSEC:
-				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
+		if metadata != nil {
+			if metadata(object.Name, policy.GetObjectTaggingAction) == ErrNone {
+				content.UserTags = object.UserTags
 			}
-			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
-				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
-					// Do not need to send any internal metadata
-					// values to client.
-					continue
+			if metadata(object.Name, policy.GetObjectAction) == ErrNone {
+				content.UserMetadata = &Metadata{}
+				switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
+				case crypto.S3:
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+				case crypto.S3KMS:
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
+				case crypto.SSEC:
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 				}
-				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
-					continue
+				for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+					if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+						// Do not need to send any internal metadata
+						// values to client.
+						continue
+					}
+					// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+					if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+						continue
+					}
+					content.UserMetadata.Set(k, v)
 				}
-				content.UserMetadata.Set(k, v)
 			}
 		}
 		contents = append(contents, content)
@@ -673,6 +714,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	return data
 }
 
+type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
+
 // generates CopyObjectResponse from etag and lastModified time.
 func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
 	return CopyObjectResponse{

cmd/api-router.go

@@ -392,6 +392,9 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
 		// ListObjectVersions
+		router.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true")
+		// ListObjectVersions
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
 		// GetBucketPolicyStatus
@@ -434,8 +437,9 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodHead).HandlerFunc(
 			collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler)))))
 		// PostPolicy
-		router.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
-			collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
+		router.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
+			return isRequestPostPolicySignatureV4(r)
+		}).HandlerFunc(collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
 		// DeleteMultipleObjects
 		router.Methods(http.MethodPost).HandlerFunc(
 			collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "")

cmd/auth-handler.go

@@ -25,6 +25,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"io"
+	"mime"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -74,8 +75,11 @@ func isRequestPresignedSignatureV2(r *http.Request) bool {
 
 // Verify if request has AWS Post policy Signature Version '4'.
 func isRequestPostPolicySignatureV4(r *http.Request) bool {
-	return strings.Contains(r.Header.Get(xhttp.ContentType), "multipart/form-data") &&
-		r.Method == http.MethodPost
+	mediaType, _, err := mime.ParseMediaType(r.Header.Get(xhttp.ContentType))
+	if err != nil {
+		return false
+	}
+	return mediaType == "multipart/form-data" && r.Method == http.MethodPost
 }
 
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
@@ -207,7 +211,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 	// that clients cannot decode the token using the temp
 	// secret keys and generate an entirely new claim by essentially
 	// hijacking the policies. We need to make sure that this is
-	// based an admin credential such that token cannot be decoded
+	// based on admin credential such that token cannot be decoded
 	// on the client side and is treated like an opaque value.
 	claims, err := auth.ExtractClaims(token, secret)
 	if err != nil {
@@ -590,6 +594,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 				// All our internal APIs are sensitive towards Date
 				// header, for all requests where Date header is not
 				// present we will reject such clients.
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -603,6 +608,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = true
 				}
 
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -618,6 +624,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 			tc.ResponseRecorder.LogErrBody = true
 		}
 
+		defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
 		atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1)
 	})

cmd/background-newdisks-heal-ops.go

@@ -453,7 +453,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 					globalBackgroundHealState.setDiskHealingStatus(disk, true)
 					if err := healFreshDisk(ctx, z, disk); err != nil {
 						globalBackgroundHealState.setDiskHealingStatus(disk, false)
-						if !errors.Is(err, context.Canceled) {
+						timedout := OperationTimedOut{}
+						if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) {
 							printEndpointError(disk, err, false)
 						}
 						return

cmd/batch-handlers.go

@@ -28,6 +28,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"path"
 	"runtime"
 	"strconv"
 	"strings"
@@ -37,10 +38,14 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/workers"
@@ -189,6 +194,11 @@ type BatchJobReplicateCredentials struct {
 	SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"`
 }
 
+// Empty indicates if credentials are not set
+func (c BatchJobReplicateCredentials) Empty() bool {
+	return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == ""
+}
+
 // Validate validates if credentials are valid
 func (c BatchJobReplicateCredentials) Validate() error {
 	if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) {
@@ -227,6 +237,11 @@ type BatchJobReplicateV1 struct {
 	clnt *miniogo.Core `msg:"-"`
 }
 
+// RemoteToLocal returns true if source is remote and target is local
+func (r BatchJobReplicateV1) RemoteToLocal() bool {
+	return !r.Source.Creds.Empty()
+}
+
 // BatchJobRequest this is an internal data structure not for external consumption.
 type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
@@ -234,6 +249,8 @@ type BatchJobRequest struct {
 	Started   time.Time            `yaml:"-" json:"started"`
 	Location  string               `yaml:"-" json:"location"`
 	Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"`
+	KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"`
+	ctx       context.Context      `msg:"-"`
 }
 
 // Notify notifies notification endpoint if configured regarding job failure or success.
@@ -269,10 +286,360 @@ func (r BatchJobReplicateV1) Notify(ctx context.Context, body io.Reader) error {
 }
 
 // ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local.
-func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, c *miniogo.Core, srcObject string) error {
+func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool) error {
+	srcBucket := r.Source.Bucket
+	tgtBucket := r.Target.Bucket
+	srcObject := srcObjInfo.Name
+	tgtObject := srcObjInfo.Name
+	if r.Target.Prefix != "" {
+		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+	}
+
+	versioned := globalBucketVersioningSys.PrefixEnabled(tgtBucket, tgtObject)
+	versionSuspended := globalBucketVersioningSys.PrefixSuspended(tgtBucket, tgtObject)
+
+	if srcObjInfo.DeleteMarker {
+		_, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{
+			VersionID:          srcObjInfo.VersionID,
+			VersionSuspended:   versionSuspended,
+			Versioned:          versioned,
+			MTime:              srcObjInfo.ModTime,
+			DeleteMarker:       srcObjInfo.DeleteMarker,
+			ReplicationRequest: true,
+		})
+		return err
+	}
+
+	opts := ObjectOptions{
+		VersionID:        srcObjInfo.VersionID,
+		Versioned:        versioned,
+		VersionSuspended: versionSuspended,
+		MTime:            srcObjInfo.ModTime,
+		PreserveETag:     srcObjInfo.ETag,
+		UserDefined:      srcObjInfo.UserDefined,
+	}
+	if crypto.S3.IsEncrypted(srcObjInfo.UserDefined) {
+		opts.ServerSideEncryption = encrypt.NewSSE()
+	}
+	slc := strings.Split(srcObjInfo.ETag, "-")
+	if len(slc) == 2 {
+		partsCount, err := strconv.Atoi(slc[1])
+		if err != nil {
+			return err
+		}
+		return r.copyWithMultipartfromSource(ctx, api, core, srcObjInfo, opts, partsCount)
+	}
+	gopts := minio.GetObjectOptions{
+		VersionID: srcObjInfo.VersionID,
+	}
+	if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil {
+		return err
+	}
+	rd, objInfo, _, err := core.GetObject(ctx, srcBucket, srcObject, gopts)
+	if err != nil {
+		return err
+	}
+	defer rd.Close()
+
+	hr, err := hash.NewReader(rd, objInfo.Size, "", "", objInfo.Size)
+	if err != nil {
+		return err
+	}
+	pReader := NewPutObjReader(hr)
+	_, err = api.PutObject(ctx, tgtBucket, tgtObject, pReader, opts)
+	return err
+}
+
+func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, opts ObjectOptions, partsCount int) (err error) {
+	srcBucket := r.Source.Bucket
+	tgtBucket := r.Target.Bucket
+	srcObject := srcObjInfo.Name
+	tgtObject := srcObjInfo.Name
+	if r.Target.Prefix != "" {
+		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+	}
+
+	var uploadedParts []CompletePart
+	res, err := api.NewMultipartUpload(context.Background(), tgtBucket, tgtObject, opts)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			// block and abort remote upload upon failure.
+			attempts := 1
+			for attempts <= 3 {
+				aerr := api.AbortMultipartUpload(ctx, tgtBucket, tgtObject, res.UploadID, ObjectOptions{})
+				if aerr == nil {
+					return
+				}
+				logger.LogIf(ctx,
+					fmt.Errorf("trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
+						humanize.Ordinal(attempts), res.UploadID, tgtBucket, tgtObject, aerr))
+				attempts++
+				time.Sleep(time.Second)
+			}
+		}
+	}()
+
+	var (
+		hr    *hash.Reader
+		pInfo PartInfo
+	)
+
+	for i := 0; i < partsCount; i++ {
+		gopts := minio.GetObjectOptions{
+			VersionID:  srcObjInfo.VersionID,
+			PartNumber: i + 1,
+		}
+		if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil {
+			return err
+		}
+		rd, objInfo, _, err := c.GetObject(ctx, srcBucket, srcObject, gopts)
+		if err != nil {
+			return err
+		}
+		defer rd.Close()
+
+		hr, err = hash.NewReader(rd, objInfo.Size, "", "", objInfo.Size)
+		if err != nil {
+			return err
+		}
+		pReader := NewPutObjReader(hr)
+		opts.PreserveETag = ""
+		pInfo, err = api.PutObjectPart(ctx, tgtBucket, tgtObject, res.UploadID, i+1, pReader, opts)
+		if err != nil {
+			return err
+		}
+		if pInfo.Size != objInfo.Size {
+			return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, objInfo.Size)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+	_, err = api.CompleteMultipartUpload(ctx, tgtBucket, tgtObject, res.UploadID, uploadedParts, opts)
+	return err
+}
+
+// StartFromSource starts the batch replication job from remote source, resumes if there was a pending job via "job.ID"
+func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	ri := &batchJobInfo{
+		JobID:     job.ID,
+		JobType:   string(job.Type()),
+		StartTime: job.Started,
+	}
+	if err := ri.load(ctx, api, job); err != nil {
+		return err
+	}
+	globalBatchJobsMetrics.save(job.ID, ri)
+
+	delay := job.Replicate.Flags.Retry.Delay
+	if delay == 0 {
+		delay = batchReplJobDefaultRetryDelay
+	}
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	skip := func(oi ObjectInfo) (ok bool) {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan {
+			// skip all objects that are newer than specified older duration
+			return true
+		}
+
+		if r.Flags.Filter.NewerThan > 0 && time.Since(oi.ModTime) >= r.Flags.Filter.NewerThan {
+			// skip all objects that are older than specified newer duration
+			return true
+		}
+
+		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(oi.ModTime) {
+			// skip all objects that are created before the specified time.
+			return true
+		}
+
+		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(oi.ModTime) {
+			// skip all objects that are created after the specified time.
+			return true
+		}
+		if len(r.Flags.Filter.Tags) > 0 {
+			// Only parse object tags if tags filter is specified.
+			tagMap := map[string]string{}
+			tagStr := oi.UserTags
+			if len(tagStr) != 0 {
+				t, err := tags.ParseObjectTags(tagStr)
+				if err != nil {
+					return false
+				}
+				tagMap = t.ToMap()
+			}
+			for _, kv := range r.Flags.Filter.Tags {
+				for t, v := range tagMap {
+					if kv.Match(BatchJobReplicateKV{Key: t, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided tags filter match skip the object
+			return false
+		}
+
+		if len(r.Flags.Filter.Metadata) > 0 {
+			for _, kv := range r.Flags.Filter.Metadata {
+				for k, v := range oi.UserDefined {
+					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
+						continue
+					}
+					// We only need to match x-amz-meta or standardHeaders
+					if kv.Match(BatchJobReplicateKV{Key: k, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided metadata filters match skip the object.
+			return false
+		}
+
+		return false
+	}
+
+	u, err := url.Parse(r.Source.Endpoint)
+	if err != nil {
+		return err
+	}
+
+	cred := r.Source.Creds
+
+	c, err := miniogo.New(u.Host, &miniogo.Options{
+		Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+		Secure:    u.Scheme == "https",
+		Transport: getRemoteInstanceTransport,
+	})
+	if err != nil {
+		return err
+	}
+
+	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
+	core := &minio.Core{Client: c}
+
+	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+	if err != nil {
+		return err
+	}
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		// invalid worker size.
+		return err
+	}
+
+	retryAttempts := ri.RetryAttempts
+	retry := false
+	for attempts := 1; attempts <= retryAttempts; attempts++ {
+		attempts := attempts
+
+		ctx, cancel := context.WithCancel(ctx)
+		objInfoCh := c.ListObjects(ctx, r.Source.Bucket, minio.ListObjectsOptions{
+			Prefix:       r.Source.Prefix,
+			WithVersions: true,
+			Recursive:    true,
+			WithMetadata: true,
+		})
+		for obj := range objInfoCh {
+			oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
+			if skip(oi) {
+				continue
+			}
+			wk.Take()
+			go func() {
+				defer wk.Give()
+				stopFn := globalBatchJobsMetrics.trace(batchReplicationMetricObject, job.ID, attempts, oi)
+				success := true
+				if err := r.ReplicateFromSource(ctx, api, core, oi, retry); err != nil {
+					// object must be deleted concurrently, allow these failures but do not count them
+					if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
+						return
+					}
+					stopFn(err)
+					logger.LogIf(ctx, err)
+					success = false
+				} else {
+					stopFn(nil)
+				}
+				ri.trackCurrentBucketObject(r.Target.Bucket, oi, success)
+				globalBatchJobsMetrics.save(job.ID, ri)
+				// persist in-memory state to disk after every 10secs.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+			}()
+		}
+		wk.Wait()
+
+		ri.RetryAttempts = attempts
+		ri.Complete = ri.ObjectsFailed == 0
+		ri.Failed = ri.ObjectsFailed > 0
+
+		globalBatchJobsMetrics.save(job.ID, ri)
+		// persist in-memory state to disk.
+		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+
+		buf, _ := json.Marshal(ri)
+		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+		}
+
+		cancel()
+		if ri.Failed {
+			ri.ObjectsFailed = 0
+			ri.Bucket = ""
+			ri.Object = ""
+			ri.Objects = 0
+			ri.BytesFailed = 0
+			ri.BytesTransferred = 0
+			retry = true // indicate we are retrying..
+			time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+			continue
+		}
+
+		break
+	}
+
 	return nil
 }
 
+// toObjectInfo converts minio.ObjectInfo to ObjectInfo
+func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo {
+	tags, _ := tags.MapToObjectTags(objInfo.UserTags)
+	oi := ObjectInfo{
+		Bucket:                    bucket,
+		Name:                      object,
+		ModTime:                   objInfo.LastModified,
+		Size:                      objInfo.Size,
+		ETag:                      objInfo.ETag,
+		VersionID:                 objInfo.VersionID,
+		IsLatest:                  objInfo.IsLatest,
+		DeleteMarker:              objInfo.IsDeleteMarker,
+		ContentType:               objInfo.ContentType,
+		Expires:                   objInfo.Expires,
+		StorageClass:              objInfo.StorageClass,
+		ReplicationStatusInternal: objInfo.ReplicationStatus,
+		UserTags:                  tags.String(),
+	}
+	oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
+	for k, v := range objInfo.Metadata {
+		oi.UserDefined[k] = v[0]
+	}
+	ce, ok := oi.UserDefined[xhttp.ContentEncoding]
+	if !ok {
+		ce, ok = oi.UserDefined[strings.ToLower(xhttp.ContentEncoding)]
+	}
+	if ok {
+		oi.ContentEncoding = ce
+	}
+	return oi
+}
+
 // ReplicateToTarget read from source and replicate to configured target
 func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *miniogo.Core, srcObjInfo ObjectInfo, retry bool) error {
 	srcBucket := r.Source.Bucket
@@ -399,14 +766,34 @@ const (
 )
 
 func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
-	data, err := readConfig(ctx, api, pathJoin(job.Location, batchReplName))
+	var fileName string
+	var format, version uint16
+	switch {
+	case job.Replicate != nil:
+		fileName = batchReplName
+		version = batchReplVersionV1
+		format = batchReplFormat
+	case job.KeyRotate != nil:
+		fileName = batchKeyRotationName
+		version = batchKeyRotateVersionV1
+		format = batchKeyRotationFormat
+
+	}
+	data, err := readConfig(ctx, api, pathJoin(job.Location, fileName))
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) {
-			ri.Version = batchReplVersionV1
-			if job.Replicate.Flags.Retry.Attempts > 0 {
-				ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
-			} else {
+			ri.Version = int(version)
+			switch {
+			case job.Replicate != nil:
 				ri.RetryAttempts = batchReplJobDefaultRetries
+				if job.Replicate.Flags.Retry.Attempts > 0 {
+					ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
+				}
+			case job.KeyRotate != nil:
+				ri.RetryAttempts = batchKeyRotateJobDefaultRetries
+				if job.KeyRotate.Flags.Retry.Attempts > 0 {
+					ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
+				}
 			}
 			return nil
 		}
@@ -417,18 +804,18 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 		return nil
 	}
 	if len(data) <= 4 {
-		return fmt.Errorf("batchRepl: no data")
+		return fmt.Errorf("%s: no data", ri.JobType)
 	}
 	// Read header
 	switch binary.LittleEndian.Uint16(data[0:2]) {
-	case batchReplFormat:
+	case format:
 	default:
-		return fmt.Errorf("batchRepl: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+		return fmt.Errorf("%s: unknown format: %d", ri.JobType, binary.LittleEndian.Uint16(data[0:2]))
 	}
 	switch binary.LittleEndian.Uint16(data[2:4]) {
-	case batchReplVersion:
+	case version:
 	default:
-		return fmt.Errorf("batchRepl: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+		return fmt.Errorf("%s: unknown version: %d", ri.JobType, binary.LittleEndian.Uint16(data[2:4]))
 	}
 
 	ri.mu.Lock()
@@ -442,7 +829,7 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 	switch ri.Version {
 	case batchReplVersionV1:
 	default:
-		return fmt.Errorf("unexpected batch repl meta version: %d", ri.Version)
+		return fmt.Errorf("unexpected batch %s meta version: %d", ri.JobType, ri.Version)
 	}
 
 	return nil
@@ -491,31 +878,51 @@ func (ri *batchJobInfo) countItem(size int64, dmarker, success bool) {
 	}
 }
 
-func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, duration time.Duration, jobLocation string) error {
+func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, duration time.Duration, job BatchJobRequest) error {
 	if ri == nil {
 		return errInvalidArgument
 	}
 	now := UTCNow()
 	ri.mu.Lock()
+	var (
+		format, version  uint16
+		jobTyp, fileName string
+	)
+
 	if now.Sub(ri.LastUpdate) >= duration {
+		switch job.Type() {
+		case madmin.BatchJobReplicate:
+			format = batchReplFormat
+			version = batchReplVersion
+			jobTyp = string(job.Type())
+			fileName = batchReplName
+			ri.Version = batchReplVersionV1
+		case madmin.BatchJobKeyRotate:
+			format = batchKeyRotationFormat
+			version = batchKeyRotateVersion
+			jobTyp = string(job.Type())
+			fileName = batchKeyRotationName
+			ri.Version = batchKeyRotateVersionV1
+		default:
+			return errInvalidArgument
+		}
 		if serverDebugLog {
-			console.Debugf("batchReplicate: persisting batchReplication info on drive: threshold:%s, batchRepl:%#v\n", now.Sub(ri.LastUpdate), ri)
+			console.Debugf("%s: persisting info on drive: threshold:%s, %s:%#v\n", jobTyp, now.Sub(ri.LastUpdate), jobTyp, ri)
 		}
 		ri.LastUpdate = now
-		ri.Version = batchReplVersionV1
 
 		data := make([]byte, 4, ri.Msgsize()+4)
 
 		// Initialize the header.
-		binary.LittleEndian.PutUint16(data[0:2], batchReplFormat)
-		binary.LittleEndian.PutUint16(data[2:4], batchReplVersion)
+		binary.LittleEndian.PutUint16(data[0:2], format)
+		binary.LittleEndian.PutUint16(data[2:4], version)
 
 		buf, err := ri.MarshalMsg(data)
 		ri.mu.Unlock()
 		if err != nil {
 			return err
 		}
-		return saveConfig(ctx, api, pathJoin(jobLocation, batchReplName), buf)
+		return saveConfig(ctx, api, pathJoin(job.Location, fileName), buf)
 	}
 	ri.mu.Unlock()
 	return nil
@@ -689,7 +1096,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job.Location))
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 			}()
 		}
 		wk.Wait()
@@ -700,11 +1107,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 		globalBatchJobsMetrics.save(job.ID, ri)
 		// persist in-memory state to disk.
-		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job.Location))
+		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
 		buf, _ := json.Marshal(ri)
 		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to notify %v", err))
+			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
 		cancel()
@@ -750,13 +1157,18 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	if r.Source.Bucket == "" {
 		return errInvalidArgument
 	}
-
-	info, err := o.GetBucketInfo(ctx, r.Source.Bucket, BucketOptions{})
+	var isRemoteToLocal bool
+	localBkt := r.Source.Bucket
+	if r.Source.Endpoint != "" {
+		localBkt = r.Target.Bucket
+		isRemoteToLocal = true
+	}
+	info, err := o.GetBucketInfo(ctx, localBkt, BucketOptions{})
 	if err != nil {
 		if isErrBucketNotFound(err) {
 			return batchReplicationJobError{
 				Code:           "NoSuchSourceBucket",
-				Description:    "The specified source bucket does not exist",
+				Description:    fmt.Sprintf("The specified bucket %s does not exist", localBkt),
 				HTTPStatusCode: http.StatusNotFound,
 			}
 		}
@@ -766,8 +1178,20 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	if err := r.Source.Type.Validate(); err != nil {
 		return err
 	}
+	if r.Source.Creds.Empty() && r.Target.Creds.Empty() {
+		return errInvalidArgument
+	}
 
-	if r.Target.Endpoint == "" {
+	if !r.Source.Creds.Empty() {
+		if err := r.Source.Creds.Validate(); err != nil {
+			return err
+		}
+	}
+	if r.Target.Endpoint == "" && !r.Target.Creds.Empty() {
+		return errInvalidArgument
+	}
+
+	if r.Source.Endpoint == "" && !r.Source.Creds.Empty() {
 		return errInvalidArgument
 	}
 
@@ -775,8 +1199,14 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		return errInvalidArgument
 	}
 
-	if err := r.Target.Creds.Validate(); err != nil {
-		return err
+	if !r.Target.Creds.Empty() {
+		if err := r.Target.Creds.Validate(); err != nil {
+			return err
+		}
+	}
+
+	if r.Source.Creds.Empty() && r.Target.Creds.Empty() {
+		return errInvalidArgument
 	}
 
 	if err := r.Target.Type.Validate(); err != nil {
@@ -799,13 +1229,21 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		return err
 	}
 
-	u, err := url.Parse(r.Target.Endpoint)
+	remoteEp := r.Target.Endpoint
+	remoteBkt := r.Target.Bucket
+	cred := r.Target.Creds
+
+	if r.Source.Endpoint != "" {
+		remoteEp = r.Source.Endpoint
+		cred = r.Source.Creds
+		remoteBkt = r.Source.Bucket
+	}
+
+	u, err := url.Parse(remoteEp)
 	if err != nil {
 		return err
 	}
 
-	cred := r.Target.Creds
-
 	c, err := miniogo.NewCore(u.Host, &miniogo.Options{
 		Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 		Secure:    u.Scheme == "https",
@@ -816,7 +1254,7 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	}
 	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
-	vcfg, err := c.GetBucketVersioning(ctx, r.Target.Bucket)
+	vcfg, err := c.GetBucketVersioning(ctx, remoteBkt)
 	if err != nil {
 		if miniogo.ToErrorResponse(err).Code == "NoSuchBucket" {
 			return batchReplicationJobError{
@@ -827,8 +1265,8 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		}
 		return err
 	}
-
-	if info.Versioning && !vcfg.Enabled() {
+	// If source has versioning enabled, target must have versioning enabled
+	if (info.Versioning && !vcfg.Enabled() && !isRemoteToLocal) || (!info.Versioning && vcfg.Enabled() && isRemoteToLocal) {
 		return batchReplicationJobError{
 			Code: "InvalidBucketState",
 			Description: fmt.Sprintf("The source '%s' has versioning enabled, target '%s' must have versioning enabled",
@@ -843,8 +1281,11 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 
 // Type returns type of batch job, currently only supports 'replicate'
 func (j BatchJobRequest) Type() madmin.BatchJobType {
-	if j.Replicate != nil {
+	switch {
+	case j.Replicate != nil:
 		return madmin.BatchJobReplicate
+	case j.KeyRotate != nil:
+		return madmin.BatchJobKeyRotate
 	}
 	return madmin.BatchJobType("unknown")
 }
@@ -852,22 +1293,28 @@ func (j BatchJobRequest) Type() madmin.BatchJobType {
 // Validate validates the current job, used by 'save()' before
 // persisting the job request
 func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
-	if j.Replicate != nil {
+	switch {
+	case j.Replicate != nil:
 		return j.Replicate.Validate(ctx, j, o)
+	case j.KeyRotate != nil:
+		return j.KeyRotate.Validate(ctx, j, o)
 	}
 	return errInvalidArgument
 }
 
 func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
-	if j.Replicate != nil {
+	switch {
+	case j.Replicate != nil:
 		deleteConfig(ctx, api, pathJoin(j.Location, batchReplName))
+	case j.KeyRotate != nil:
+		deleteConfig(ctx, api, pathJoin(j.Location, batchKeyRotationName))
 	}
 	globalBatchJobsMetrics.delete(j.ID)
 	deleteConfig(ctx, api, j.Location)
 }
 
 func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
-	if j.Replicate == nil {
+	if j.Replicate == nil && j.KeyRotate == nil {
 		return errInvalidArgument
 	}
 
@@ -1061,6 +1508,34 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 	writeSuccessResponseJSON(w, buf)
 }
 
+// CancelBatchJob cancels a job in progress
+func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "CancelBatchJob")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CancelBatchJobAction)
+	if objectAPI == nil {
+		return
+	}
+	jobID := r.Form.Get("id")
+	if jobID == "" {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL)
+		return
+	}
+	if err := globalBatchJobPool.canceler(jobID, true); err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
+		return
+	}
+	j := BatchJobRequest{
+		ID:       jobID,
+		Location: pathJoin(batchJobPrefix, jobID),
+	}
+	j.delete(ctx, objectAPI)
+
+	writeSuccessNoContent(w)
+}
+
 //msgp:ignore BatchJobPool
 
 // BatchJobPool batch job pool
@@ -1070,6 +1545,8 @@ type BatchJobPool struct {
 	once         sync.Once
 	mu           sync.Mutex
 	jobCh        chan *BatchJobRequest
+	jmu          sync.Mutex // protects jobCancelers
+	jobCancelers map[string]context.CancelFunc
 	workerKillCh chan struct{}
 	workerSize   int
 }
@@ -1083,6 +1560,7 @@ func newBatchJobPool(ctx context.Context, o ObjectLayer, workers int) *BatchJobP
 		objLayer:     o,
 		jobCh:        make(chan *BatchJobRequest, 10000),
 		workerKillCh: make(chan struct{}, workers),
+		jobCancelers: make(map[string]context.CancelFunc),
 	}
 	jpool.ResizeWorkers(workers)
 	jpool.resume()
@@ -1098,6 +1576,10 @@ func (j *BatchJobPool) resume() {
 		return
 	}
 	for result := range results {
+		// ignore batch-replicate.bin and batch-rotate.bin entries
+		if strings.HasSuffix(result.Name, slashSeparator) {
+			continue
+		}
 		req := &BatchJobRequest{}
 		if err := req.load(ctx, j.objLayer, result.Name); err != nil {
 			logger.LogIf(ctx, err)
@@ -1124,15 +1606,36 @@ func (j *BatchJobPool) AddWorker() {
 				return
 			}
 			if job.Replicate != nil {
-				if err := job.Replicate.Start(j.ctx, j.objLayer, *job); err != nil {
+				if job.Replicate.RemoteToLocal() {
+					if err := job.Replicate.StartFromSource(job.ctx, j.objLayer, *job); err != nil {
+						if !isErrBucketNotFound(err) {
+							logger.LogIf(j.ctx, err)
+							j.canceler(job.ID, false)
+							continue
+						}
+						// Bucket not found proceed to delete such a job.
+					}
+				} else {
+					if err := job.Replicate.Start(job.ctx, j.objLayer, *job); err != nil {
+						if !isErrBucketNotFound(err) {
+							logger.LogIf(j.ctx, err)
+							j.canceler(job.ID, false)
+							continue
+						}
+						// Bucket not found proceed to delete such a job.
+					}
+				}
+			}
+			if job.KeyRotate != nil {
+				if err := job.KeyRotate.Start(job.ctx, j.objLayer, *job); err != nil {
 					if !isErrBucketNotFound(err) {
 						logger.LogIf(j.ctx, err)
 						continue
 					}
-					// Bucket not found proceed to delete such a job.
 				}
 			}
 			job.delete(j.ctx, j.objLayer)
+			j.canceler(job.ID, false)
 		case <-j.workerKillCh:
 			return
 		}
@@ -1162,6 +1665,12 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
 	if j == nil {
 		return errInvalidArgument
 	}
+	jctx, jcancel := context.WithCancel(j.ctx)
+	j.jmu.Lock()
+	j.jobCancelers[req.ID] = jcancel
+	j.jmu.Unlock()
+	req.ctx = jctx
+
 	select {
 	case <-j.ctx.Done():
 		j.once.Do(func() {
@@ -1174,6 +1683,22 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
 	return nil
 }
 
+// delete canceler from the map, cancel job if requested
+func (j *BatchJobPool) canceler(jobID string, cancel bool) error {
+	if j == nil {
+		return errInvalidArgument
+	}
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	if canceler, ok := j.jobCancelers[jobID]; ok {
+		if cancel {
+			canceler()
+		}
+	}
+	delete(j.jobCancelers, jobID)
+	return nil
+}
+
 //msgp:ignore batchJobMetrics
 type batchJobMetrics struct {
 	sync.RWMutex
@@ -1184,25 +1709,32 @@ var globalBatchJobsMetrics = batchJobMetrics{
 	metrics: make(map[string]*batchJobInfo),
 }
 
-//msgp:ignore batchReplicationMetric
-//go:generate stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric $GOFILE
-type batchReplicationMetric uint8
+//msgp:ignore batchJobMetric
+//go:generate stringer -type=batchJobMetric -trimprefix=batchJobMetric $GOFILE
+type batchJobMetric uint8
 
 const (
-	batchReplicationMetricObject batchReplicationMetric = iota
+	batchReplicationMetricObject batchJobMetric = iota
+	batchKeyRotationMetricObject
 )
 
-func batchReplicationTrace(d batchReplicationMetric, job string, startTime time.Time, duration time.Duration, info ObjectInfo, attempts int, err error) madmin.TraceInfo {
+func batchJobTrace(d batchJobMetric, job string, startTime time.Time, duration time.Duration, info ObjectInfo, attempts int, err error) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
 	}
-	funcName := fmt.Sprintf("batchReplication.%s (job-name=%s)", d.String(), job)
+	jobKind := "batchReplication"
+	traceType := madmin.TraceBatchReplication
+	if d == batchKeyRotationMetricObject {
+		jobKind = "batchKeyRotation"
+		traceType = madmin.TraceBatchKeyRotation
+	}
+	funcName := fmt.Sprintf("%s.%s (job-name=%s)", jobKind, d.String(), job)
 	if attempts > 0 {
-		funcName = fmt.Sprintf("batchReplication.%s (job-name=%s,attempts=%s)", d.String(), job, humanize.Ordinal(attempts))
+		funcName = fmt.Sprintf("%s.%s (job-name=%s,attempts=%s)", jobKind, d.String(), job, humanize.Ordinal(attempts))
 	}
 	return madmin.TraceInfo{
-		TraceType: madmin.TraceBatchReplication,
+		TraceType: traceType,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
 		FuncName:  funcName,
@@ -1234,6 +1766,12 @@ func (m *batchJobMetrics) report(jobID string) (metrics *madmin.BatchJobMetrics)
 				BytesTransferred: job.BytesTransferred,
 				BytesFailed:      job.BytesFailed,
 			},
+			KeyRotate: &madmin.KeyRotationInfo{
+				Bucket:        job.Bucket,
+				Object:        job.Object,
+				Objects:       job.Objects,
+				ObjectsFailed: job.ObjectsFailed,
+			},
 		}
 		if match {
 			break
@@ -1256,12 +1794,19 @@ func (m *batchJobMetrics) save(jobID string, ri *batchJobInfo) {
 	m.metrics[jobID] = ri.clone()
 }
 
-func (m *batchJobMetrics) trace(d batchReplicationMetric, job string, attempts int, info ObjectInfo) func(err error) {
+func (m *batchJobMetrics) trace(d batchJobMetric, job string, attempts int, info ObjectInfo) func(err error) {
 	startTime := time.Now()
 	return func(err error) {
 		duration := time.Since(startTime)
-		if globalTrace.NumSubscribers(madmin.TraceBatchReplication) > 0 {
-			globalTrace.Publish(batchReplicationTrace(d, job, startTime, duration, info, attempts, err))
+		switch d {
+		case batchReplicationMetricObject:
+			if globalTrace.NumSubscribers(madmin.TraceBatchReplication) > 0 {
+				globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err))
+			}
+		case batchKeyRotationMetricObject:
+			if globalTrace.NumSubscribers(madmin.TraceBatchKeyRotation) > 0 {
+				globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err))
+			}
 		}
 	}
 }

cmd/batch-handlers_gen.go

@@ -1538,6 +1538,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "KeyRotate":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "KeyRotate")
+					return
+				}
+				z.KeyRotate = nil
+			} else {
+				if z.KeyRotate == nil {
+					z.KeyRotate = new(BatchJobKeyRotateV1)
+				}
+				err = z.KeyRotate.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "KeyRotate")
+					return
+				}
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -1551,9 +1569,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 5
+	// map header, size 6
 	// write "ID"
-	err = en.Append(0x85, 0xa2, 0x49, 0x44)
+	err = en.Append(0x86, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -1609,15 +1627,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "KeyRotate"
+	err = en.Append(0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65)
+	if err != nil {
+		return
+	}
+	if z.KeyRotate == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.KeyRotate.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "KeyRotate")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
+	// map header, size 6
 	// string "ID"
-	o = append(o, 0x85, 0xa2, 0x49, 0x44)
+	o = append(o, 0x86, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -1639,6 +1674,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 			return
 		}
 	}
+	// string "KeyRotate"
+	o = append(o, 0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65)
+	if z.KeyRotate == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.KeyRotate.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "KeyRotate")
+			return
+		}
+	}
 	return
 }
 
@@ -1701,6 +1747,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "KeyRotate":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.KeyRotate = nil
+			} else {
+				if z.KeyRotate == nil {
+					z.KeyRotate = new(BatchJobKeyRotateV1)
+				}
+				bts, err = z.KeyRotate.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "KeyRotate")
+					return
+				}
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -1721,6 +1784,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
 	} else {
 		s += z.Replicate.Msgsize()
 	}
+	s += 10
+	if z.KeyRotate == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.KeyRotate.Msgsize()
+	}
 	return
 }
 

cmd/batch-rotate.go

@@ -0,0 +1,564 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/workers"
+	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/wildcard"
+)
+
+// keyrotate:
+//   apiVersion: v1
+//   bucket: BUCKET
+//   prefix: PREFIX
+//   encryption:
+//     type: sse-s3 # valid values are sse-s3 and sse-kms
+//     key: <new-kms-key> # valid only for sse-kms
+//     context: <new-kms-key-context> # valid only for sse-kms
+// # optional flags based filtering criteria
+// # for all objects
+// flags:
+//   filter:
+//     newerThan: "7d" # match objects newer than this value (e.g. 7d10h31s)
+//     olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+//     createdAfter: "date" # match objects created after "date"
+//     createdBefore: "date" # match objects created before "date"
+//     tags:
+//       - key: "name"
+//         value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
+//     metadata:
+//       - key: "content-type"
+//         value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
+//     kmskey: "key-id" # match objects with KMS key-id (applicable only for sse-kms)
+//   notify:
+//     endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
+//     token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+
+//   retry:
+//     attempts: 10 # number of retries for the job before giving up
+//     delay: "500ms" # least amount of delay between each retry
+
+//go:generate msgp -file $GOFILE -unexported
+
+// BatchKeyRotateKV is a datatype that holds key and values for filtering of objects
+// used by metadata filter as well as tags based filtering.
+type BatchKeyRotateKV struct {
+	Key   string `yaml:"key" json:"key"`
+	Value string `yaml:"value" json:"value"`
+}
+
+// Validate returns an error if key is empty
+func (kv BatchKeyRotateKV) Validate() error {
+	if kv.Key == "" {
+		return errInvalidArgument
+	}
+	return nil
+}
+
+// Empty indicates if kv is not set
+func (kv BatchKeyRotateKV) Empty() bool {
+	return kv.Key == "" && kv.Value == ""
+}
+
+// Match matches input kv with kv, value will be wildcard matched depending on the user input
+func (kv BatchKeyRotateKV) Match(ikv BatchKeyRotateKV) bool {
+	if kv.Empty() {
+		return true
+	}
+	if strings.EqualFold(kv.Key, ikv.Key) {
+		return wildcard.Match(kv.Value, ikv.Value)
+	}
+	return false
+}
+
+// BatchKeyRotateRetry datatype represents total retry attempts and delay between each retries.
+type BatchKeyRotateRetry struct {
+	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
+	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
+}
+
+// Validate validates input replicate retries.
+func (r BatchKeyRotateRetry) Validate() error {
+	if r.Attempts < 0 {
+		return errInvalidArgument
+	}
+
+	if r.Delay < 0 {
+		return errInvalidArgument
+	}
+
+	return nil
+}
+
+// BatchKeyRotationType defines key rotation type
+type BatchKeyRotationType string
+
+const (
+	sses3  BatchKeyRotationType = "sse-s3"
+	ssekms BatchKeyRotationType = "sse-kms"
+)
+
+// BatchJobKeyRotateEncryption defines key rotation encryption options passed
+type BatchJobKeyRotateEncryption struct {
+	Type       BatchKeyRotationType `yaml:"type" json:"type"`
+	Key        string               `yaml:"key" json:"key"`
+	Context    string               `yaml:"context" json:"context"`
+	kmsContext kms.Context          `msg:"-"`
+}
+
+// Validate validates input key rotation encryption options.
+func (e BatchJobKeyRotateEncryption) Validate() error {
+	if e.Type != sses3 && e.Type != ssekms {
+		return errInvalidArgument
+	}
+	spaces := strings.HasPrefix(e.Key, " ") || strings.HasSuffix(e.Key, " ")
+	if e.Type == ssekms && spaces {
+		return crypto.ErrInvalidEncryptionKeyID
+	}
+	if e.Type == ssekms && GlobalKMS != nil {
+		ctx := kms.Context{}
+		if e.Context != "" {
+			b, err := base64.StdEncoding.DecodeString(e.Context)
+			if err != nil {
+				return err
+			}
+
+			json := jsoniter.ConfigCompatibleWithStandardLibrary
+			if err := json.Unmarshal(b, &ctx); err != nil {
+				return err
+			}
+		}
+		e.kmsContext = kms.Context{}
+		for k, v := range ctx {
+			e.kmsContext[k] = v
+		}
+		ctx["MinIO batch API"] = "batchrotate" // Context for a test key operation
+		if _, err := GlobalKMS.GenerateKey(GlobalContext, e.Key, ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// BatchKeyRotateFilter holds all the filters currently supported for batch replication
+type BatchKeyRotateFilter struct {
+	NewerThan     time.Duration      `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     time.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time          `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchKeyRotateKV `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchKeyRotateKV `yaml:"metadata,omitempty" json:"metadata"`
+	KMSKeyID      string             `yaml:"kmskeyid" json:"kmskey"`
+}
+
+// BatchKeyRotateNotification success or failure notification endpoint for each job attempts
+type BatchKeyRotateNotification struct {
+	Endpoint string `yaml:"endpoint" json:"endpoint"`
+	Token    string `yaml:"token" json:"token"`
+}
+
+// BatchJobKeyRotateFlags various configurations for replication job definition currently includes
+// - filter
+// - notify
+// - retry
+type BatchJobKeyRotateFlags struct {
+	Filter BatchKeyRotateFilter       `yaml:"filter" json:"filter"`
+	Notify BatchKeyRotateNotification `yaml:"notify" json:"notify"`
+	Retry  BatchKeyRotateRetry        `yaml:"retry" json:"retry"`
+}
+
+// BatchJobKeyRotateV1 v1 of batch key rotation job
+type BatchJobKeyRotateV1 struct {
+	APIVersion string                      `yaml:"apiVersion" json:"apiVersion"`
+	Flags      BatchJobKeyRotateFlags      `yaml:"flags" json:"flags"`
+	Bucket     string                      `yaml:"bucket" json:"bucket"`
+	Prefix     string                      `yaml:"prefix" json:"prefix"`
+	Endpoint   string                      `yaml:"endpoint" json:"endpoint"`
+	Encryption BatchJobKeyRotateEncryption `yaml:"encryption" json:"encryption"`
+}
+
+// Notify notifies notification endpoint if configured regarding job failure or success.
+func (r BatchJobKeyRotateV1) Notify(ctx context.Context, body io.Reader) error {
+	if r.Flags.Notify.Endpoint == "" {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.Flags.Notify.Endpoint, body)
+	if err != nil {
+		return err
+	}
+
+	if r.Flags.Notify.Token != "" {
+		req.Header.Set("Authorization", r.Flags.Notify.Token)
+	}
+
+	clnt := http.Client{Transport: getRemoteInstanceTransport}
+	resp, err := clnt.Do(req)
+	if err != nil {
+		return err
+	}
+
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(resp.Status)
+	}
+
+	return nil
+}
+
+// KeyRotate rotates encryption key of an object
+func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error {
+	srcBucket := r.Bucket
+	srcObject := objInfo.Name
+
+	if objInfo.DeleteMarker || !objInfo.VersionPurgeStatus.Empty() {
+		return nil
+	}
+	sseKMS := crypto.S3KMS.IsEncrypted(objInfo.UserDefined)
+	sseS3 := crypto.S3.IsEncrypted(objInfo.UserDefined)
+	if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
+		return errInvalidEncryptionParameters
+	}
+	if sseKMS && r.Encryption.Type == sses3 { // previously encrypted with sse-kms, now sse-s3 disallowed
+		return errInvalidEncryptionParameters
+	}
+	versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
+	versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
+
+	lock := api.NewNSLock(r.Bucket, objInfo.Name)
+	lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return err
+	}
+	ctx = lkctx.Context()
+	defer lock.Unlock(lkctx)
+
+	opts := ObjectOptions{
+		VersionID:        objInfo.VersionID,
+		Versioned:        versioned,
+		VersionSuspended: versionSuspended,
+		NoLock:           true,
+	}
+	obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts)
+	if err != nil {
+		return err
+	}
+	oi := obj.Clone()
+	var (
+		newKeyID      string
+		newKeyContext kms.Context
+	)
+	encMetadata := make(map[string]string)
+	for k, v := range oi.UserDefined {
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			encMetadata[k] = v
+		}
+	}
+
+	if (sseKMS || sseS3) && r.Encryption.Type == ssekms {
+		if err = r.Encryption.Validate(); err != nil {
+			return err
+		}
+		newKeyID = strings.TrimPrefix(r.Encryption.Key, crypto.ARNPrefix)
+		newKeyContext = r.Encryption.kmsContext
+	}
+	if err = rotateKey(ctx, []byte{}, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext); err != nil {
+		return err
+	}
+
+	// Since we are rotating the keys, make sure to update the metadata.
+	oi.metadataOnly = true
+	oi.keyRotation = true
+	for k, v := range encMetadata {
+		oi.UserDefined[k] = v
+	}
+	if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{
+		VersionID: oi.VersionID,
+	}, ObjectOptions{
+		VersionID: oi.VersionID,
+		NoLock:    true,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+const (
+	batchKeyRotationName               = "batch-rotate.bin"
+	batchKeyRotationFormat             = 1
+	batchKeyRotateVersionV1            = 1
+	batchKeyRotateVersion              = batchKeyRotateVersionV1
+	batchKeyRotateAPIVersion           = "v1"
+	batchKeyRotateJobDefaultRetries    = 3
+	batchKeyRotateJobDefaultRetryDelay = 250 * time.Millisecond
+)
+
+// Start the batch key rottion job, resumes if there was a pending job via "job.ID"
+func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	ri := &batchJobInfo{
+		JobID:     job.ID,
+		JobType:   string(job.Type()),
+		StartTime: job.Started,
+	}
+	if err := ri.load(ctx, api, job); err != nil {
+		return err
+	}
+
+	globalBatchJobsMetrics.save(job.ID, ri)
+	lastObject := ri.Object
+
+	delay := job.KeyRotate.Flags.Retry.Delay
+	if delay == 0 {
+		delay = batchKeyRotateJobDefaultRetryDelay
+	}
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	skip := func(info FileInfo) (ok bool) {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
+			// skip all objects that are newer than specified older duration
+			return false
+		}
+
+		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan {
+			// skip all objects that are older than specified newer duration
+			return false
+		}
+
+		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(info.ModTime) {
+			// skip all objects that are created before the specified time.
+			return false
+		}
+
+		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(info.ModTime) {
+			// skip all objects that are created after the specified time.
+			return false
+		}
+
+		if len(r.Flags.Filter.Tags) > 0 {
+			// Only parse object tags if tags filter is specified.
+			tagMap := map[string]string{}
+			tagStr := info.Metadata[xhttp.AmzObjectTagging]
+			if len(tagStr) != 0 {
+				t, err := tags.ParseObjectTags(tagStr)
+				if err != nil {
+					return false
+				}
+				tagMap = t.ToMap()
+			}
+
+			for _, kv := range r.Flags.Filter.Tags {
+				for t, v := range tagMap {
+					if kv.Match(BatchKeyRotateKV{Key: t, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided tags filter match skip the object
+			return false
+		}
+
+		if len(r.Flags.Filter.Metadata) > 0 {
+			for _, kv := range r.Flags.Filter.Metadata {
+				for k, v := range info.Metadata {
+					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
+						continue
+					}
+					// We only need to match x-amz-meta or standardHeaders
+					if kv.Match(BatchKeyRotateKV{Key: k, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided metadata filters match skip the object.
+			return false
+		}
+		if r.Flags.Filter.KMSKeyID != "" {
+			if v, ok := info.Metadata[xhttp.AmzServerSideEncryptionKmsID]; ok && strings.TrimPrefix(v, crypto.ARNPrefix) != r.Flags.Filter.KMSKeyID {
+				return false
+			}
+		}
+		return true
+	}
+
+	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_KEYROTATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+	if err != nil {
+		return err
+	}
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		// invalid worker size.
+		return err
+	}
+
+	retryAttempts := ri.RetryAttempts
+	ctx, cancel := context.WithCancel(ctx)
+
+	results := make(chan ObjectInfo, 100)
+	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, ObjectOptions{
+		WalkMarker: lastObject,
+		WalkFilter: skip,
+	}); err != nil {
+		cancel()
+		// Do not need to retry if we can't list objects on source.
+		return err
+	}
+
+	for result := range results {
+		result := result
+		sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined)
+		sseS3 := crypto.S3.IsEncrypted(result.UserDefined)
+		if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
+			continue
+		}
+		wk.Take()
+		go func() {
+			defer wk.Give()
+			for attempts := 1; attempts <= retryAttempts; attempts++ {
+				attempts := attempts
+				stopFn := globalBatchJobsMetrics.trace(batchKeyRotationMetricObject, job.ID, attempts, result)
+				success := true
+				if err := r.KeyRotate(ctx, api, result); err != nil {
+					stopFn(err)
+					logger.LogIf(ctx, err)
+					success = false
+				} else {
+					stopFn(nil)
+				}
+				ri.trackCurrentBucketObject(r.Bucket, result, success)
+				ri.RetryAttempts = attempts
+				globalBatchJobsMetrics.save(job.ID, ri)
+				// persist in-memory state to disk after every 10secs.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				if success {
+					break
+				}
+			}
+		}()
+	}
+	wk.Wait()
+
+	ri.Complete = ri.ObjectsFailed == 0
+	ri.Failed = ri.ObjectsFailed > 0
+	globalBatchJobsMetrics.save(job.ID, ri)
+	// persist in-memory state to disk.
+	logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+
+	buf, _ := json.Marshal(ri)
+	if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+	}
+
+	cancel()
+	if ri.Failed {
+		ri.ObjectsFailed = 0
+		ri.Bucket = ""
+		ri.Object = ""
+		ri.Objects = 0
+		time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+	}
+
+	return nil
+}
+
+//msgp:ignore batchKeyRotationJobError
+type batchKeyRotationJobError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+func (e batchKeyRotationJobError) Error() string {
+	return e.Description
+}
+
+// Validate validates the job definition input
+func (r *BatchJobKeyRotateV1) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
+	if r == nil {
+		return nil
+	}
+
+	if r.APIVersion != batchKeyRotateAPIVersion {
+		return errInvalidArgument
+	}
+
+	if r.Bucket == "" {
+		return errInvalidArgument
+	}
+
+	if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil {
+		if isErrBucketNotFound(err) {
+			return batchKeyRotationJobError{
+				Code:           "NoSuchSourceBucket",
+				Description:    "The specified source bucket does not exist",
+				HTTPStatusCode: http.StatusNotFound,
+			}
+		}
+		return err
+	}
+	if GlobalKMS == nil {
+		return errKMSNotConfigured
+	}
+	if err := r.Encryption.Validate(); err != nil {
+		return err
+	}
+
+	for _, tag := range r.Flags.Filter.Tags {
+		if err := tag.Validate(); err != nil {
+			return err
+		}
+	}
+
+	for _, meta := range r.Flags.Filter.Metadata {
+		if err := meta.Validate(); err != nil {
+			return err
+		}
+	}
+
+	if err := r.Flags.Retry.Validate(); err != nil {
+		return err
+	}
+	return nil
+}

cmd/batch-rotate_gen_test.go

@@ -0,0 +1,801 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBatchJobKeyRotateEncryption(t *testing.T) {
+	v := BatchJobKeyRotateEncryption{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKeyRotateEncryption(t *testing.T) {
+	v := BatchJobKeyRotateEncryption{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateEncryption Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKeyRotateEncryption{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobKeyRotateFlags(t *testing.T) {
+	v := BatchJobKeyRotateFlags{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKeyRotateFlags(t *testing.T) {
+	v := BatchJobKeyRotateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateFlags Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKeyRotateFlags{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobKeyRotateV1(t *testing.T) {
+	v := BatchJobKeyRotateV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKeyRotateV1(t *testing.T) {
+	v := BatchJobKeyRotateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateV1 Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKeyRotateV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateFilter(t *testing.T) {
+	v := BatchKeyRotateFilter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateFilter(t *testing.T) {
+	v := BatchKeyRotateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateFilter Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateFilter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateKV(t *testing.T) {
+	v := BatchKeyRotateKV{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateKV(t *testing.T) {
+	v := BatchKeyRotateKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateKV Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateKV{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateNotification(t *testing.T) {
+	v := BatchKeyRotateNotification{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateNotification(t *testing.T) {
+	v := BatchKeyRotateNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateNotification Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateNotification{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateRetry(t *testing.T) {
+	v := BatchKeyRotateRetry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateRetry(t *testing.T) {
+	v := BatchKeyRotateRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateRetry Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateRetry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batchjobmetric_string.go

@@ -0,0 +1,24 @@
+// Code generated by "stringer -type=batchJobMetric -trimprefix=batchJobMetric batch-handlers.go"; DO NOT EDIT.
+
+package cmd
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[batchReplicationMetricObject-0]
+	_ = x[batchKeyRotationMetricObject-1]
+}
+
+const _batchJobMetric_name = "batchReplicationMetricObjectbatchKeyRotationMetricObject"
+
+var _batchJobMetric_index = [...]uint8{0, 28, 56}
+
+func (i batchJobMetric) String() string {
+	if i >= batchJobMetric(len(_batchJobMetric_index)-1) {
+		return "batchJobMetric(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _batchJobMetric_name[_batchJobMetric_index[i]:_batchJobMetric_index[i+1]]
+}

cmd/batchreplicationmetric_string.go

@@ -1,23 +0,0 @@
-// Code generated by "stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric batch-handlers.go"; DO NOT EDIT.
-
-package cmd
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[batchReplicationMetricObject-0]
-}
-
-const _batchReplicationMetric_name = "Object"
-
-var _batchReplicationMetric_index = [...]uint8{0, 6}
-
-func (i batchReplicationMetric) String() string {
-	if i >= batchReplicationMetric(len(_batchReplicationMetric_index)-1) {
-		return "batchReplicationMetric(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-	return _batchReplicationMetric_name[_batchReplicationMetric_index[i]:_batchReplicationMetric_index[i+1]]
-}

cmd/bootstrap-messages.go

@@ -0,0 +1,116 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio/internal/pubsub"
+)
+
+const bootstrapMsgsLimit = 4 << 10
+
+type bootstrapInfo struct {
+	msg    string
+	ts     time.Time
+	source string
+}
+type bootstrapTracer struct {
+	mu         sync.RWMutex
+	idx        int
+	info       [bootstrapMsgsLimit]bootstrapInfo
+	lastUpdate time.Time
+}
+
+var globalBootstrapTracer = &bootstrapTracer{}
+
+func (bs *bootstrapTracer) DropEvents() {
+	bs.mu.Lock()
+	defer bs.mu.Unlock()
+
+	if time.Now().UTC().Sub(bs.lastUpdate) > 24*time.Hour {
+		bs.info = [4096]bootstrapInfo{}
+		bs.idx = 0
+	}
+}
+
+func (bs *bootstrapTracer) Empty() bool {
+	var empty bool
+	bs.mu.RLock()
+	empty = bs.info[0].msg == ""
+	bs.mu.RUnlock()
+
+	return empty
+}
+
+func (bs *bootstrapTracer) Record(msg string, skip int) {
+	source := getSource(skip + 1)
+	bs.mu.Lock()
+	now := time.Now().UTC()
+	bs.info[bs.idx] = bootstrapInfo{
+		msg:    msg,
+		ts:     now,
+		source: source,
+	}
+	bs.lastUpdate = now
+	bs.idx = (bs.idx + 1) % bootstrapMsgsLimit
+	bs.mu.Unlock()
+}
+
+func (bs *bootstrapTracer) Events() []madmin.TraceInfo {
+	traceInfo := make([]madmin.TraceInfo, 0, bootstrapMsgsLimit)
+
+	// Add all messages in order
+	addAll := func(info []bootstrapInfo) {
+		for _, msg := range info {
+			if msg.ts.IsZero() {
+				continue // skip empty events
+			}
+			traceInfo = append(traceInfo, madmin.TraceInfo{
+				TraceType: madmin.TraceBootstrap,
+				Time:      msg.ts,
+				NodeName:  globalLocalNodeName,
+				FuncName:  "BOOTSTRAP",
+				Message:   fmt.Sprintf("%s %s", msg.source, msg.msg),
+			})
+		}
+	}
+
+	bs.mu.RLock()
+	addAll(bs.info[bs.idx:])
+	addAll(bs.info[:bs.idx])
+	bs.mu.RUnlock()
+	return traceInfo
+}
+
+func (bs *bootstrapTracer) Publish(ctx context.Context, trace *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) {
+	if bs.Empty() {
+		return
+	}
+	for _, bsEvent := range bs.Events() {
+		select {
+		case <-ctx.Done():
+		default:
+			trace.Publish(bsEvent)
+		}
+	}
+}

cmd/bootstrap-messages_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestBootstrap(t *testing.T) {
+	// Bootstrap events exceed bootstrap messages limit
+	bsTracer := &bootstrapTracer{}
+	for i := 0; i < bootstrapMsgsLimit+10; i++ {
+		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
+	}
+
+	traceInfos := bsTracer.Events()
+	if len(traceInfos) != bootstrapMsgsLimit {
+		t.Fatalf("Expected length of events %d but got %d", bootstrapMsgsLimit, len(traceInfos))
+	}
+
+	// Simulate the case where bootstrap events were updated a day ago
+	bsTracer.lastUpdate = time.Now().UTC().Add(-25 * time.Hour)
+	bsTracer.DropEvents()
+	if !bsTracer.Empty() {
+		t.Fatalf("Expected all bootstrap events to have been dropped, but found %d events", len(bsTracer.Events()))
+	}
+
+	// Fewer than 4K bootstrap events
+	for i := 0; i < 10; i++ {
+		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
+	}
+	events := bsTracer.Events()
+	if len(events) != 10 {
+		t.Fatalf("Expected length of events %d but got %d", 10, len(events))
+	}
+	for i, traceInfo := range bsTracer.Events() {
+		msg := fmt.Sprintf("msg-%d", i)
+		if !strings.HasSuffix(traceInfo.Message, msg) {
+			t.Fatalf("Expected %s but got %s", msg, traceInfo.Message)
+		}
+	}
+}

cmd/bootstrap-peer-server.go

@@ -101,10 +101,14 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 }
 
 var skipEnvs = map[string]struct{}{
-	"MINIO_OPTS":         {},
-	"MINIO_CERT_PASSWD":  {},
-	"MINIO_SERVER_DEBUG": {},
-	"MINIO_DSYNC_TRACE":  {},
+	"MINIO_OPTS":          {},
+	"MINIO_CERT_PASSWD":   {},
+	"MINIO_SERVER_DEBUG":  {},
+	"MINIO_DSYNC_TRACE":   {},
+	"MINIO_ROOT_USER":     {},
+	"MINIO_ROOT_PASSWORD": {},
+	"MINIO_ACCESS_KEY":    {},
+	"MINIO_SECRET_KEY":    {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {
@@ -118,7 +122,7 @@ func getServerSystemCfg() ServerSystemConfig {
 		if _, ok := skipEnvs[envK]; ok {
 			continue
 		}
-		envValues[envK] = env.Get(envK, "")
+		envValues[envK] = logger.HashString(env.Get(envK, ""))
 	}
 	return ServerSystemConfig{
 		MinioEndpoints: globalEndpoints,
@@ -126,11 +130,22 @@ func getServerSystemCfg() ServerSystemConfig {
 	}
 }
 
+func (b *bootstrapRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusForbidden)
+	w.Write([]byte(err.Error()))
+}
+
 // HealthHandler returns success if request is valid
 func (b *bootstrapRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {}
 
 func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "VerifyHandler")
+
+	if err := storageServerRequestValidate(r); err != nil {
+		b.writeErrorResponse(w, err)
+		return
+	}
+
 	cfg := getServerSystemCfg()
 	logger.LogIf(ctx, json.NewEncoder(w).Encode(&cfg))
 }

cmd/bucket-handlers.go

@@ -796,8 +796,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 				// Make sure to add Location information here only for bucket
-				w.Header().Set(xhttp.Location,
-					getObjectLocation(r, globalDomainNames, bucket, ""))
+				w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket))
 
 				writeSuccessResponseHeadersOnly(w)
 
@@ -848,9 +847,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	logger.LogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts))
 
 	// Make sure to add Location information here only for bucket
-	if cp := pathClean(r.URL.Path); cp != "" {
-		w.Header().Set(xhttp.Location, cp) // Clean any trailing slashes.
-	}
+	w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket))
 
 	writeSuccessResponseHeadersOnly(w)
 
@@ -1123,7 +1120,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
 
-	w.Header().Set(xhttp.Location, getObjectLocation(r, globalDomainNames, bucket, object))
+	if obj := getObjectLocation(r, globalDomainNames, bucket, object); obj != "" {
+		w.Header().Set(xhttp.Location, obj)
+	}
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{

cmd/bucket-lifecycle.go

@@ -24,12 +24,15 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
@@ -38,6 +41,8 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
+	"github.com/minio/minio/internal/workers"
+	"github.com/minio/pkg/env"
 )
 
 const (
@@ -66,6 +71,30 @@ func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
 
+func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string) madmin.TraceInfo {
+	return madmin.TraceInfo{
+		TraceType: madmin.TraceILM,
+		Time:      startTime,
+		NodeName:  globalLocalNodeName,
+		FuncName:  event,
+		Duration:  duration,
+		Path:      pathJoin(oi.Bucket, oi.Name),
+		Error:     "",
+		Message:   getSource(4),
+		Custom:    map[string]string{"version-id": oi.VersionID},
+	}
+}
+
+func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string) {
+	startTime := time.Now()
+	return func(event string) {
+		duration := time.Since(startTime)
+		if globalTrace.NumSubscribers(madmin.TraceILM) > 0 {
+			globalTrace.Publish(ilmTrace(startTime, duration, oi, event))
+		}
+	}
+}
+
 type expiryTask struct {
 	objInfo        ObjectInfo
 	versionExpiry  bool
@@ -116,26 +145,49 @@ var globalExpiryState *expiryState
 
 func newExpiryState() *expiryState {
 	return &expiryState{
-		byDaysCh:            make(chan expiryTask, 10000),
-		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 10000),
+		byDaysCh:            make(chan expiryTask, 100000),
+		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 100000),
 	}
 }
 
 func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 	globalExpiryState = newExpiryState()
+
+	workerSize, _ := strconv.Atoi(env.Get("_MINIO_ILM_EXPIRY_WORKERS", strconv.Itoa((runtime.GOMAXPROCS(0)+1)/2)))
+
+	ewk, err := workers.New(workerSize)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
+	nwk, err := workers.New(workerSize)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
 	go func() {
 		for t := range globalExpiryState.byDaysCh {
-			if t.objInfo.TransitionedObject.Status != "" {
-				applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
-			} else {
-				applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
-			}
+			ewk.Take()
+			go func(t expiryTask) {
+				defer ewk.Give()
+				if t.objInfo.TransitionedObject.Status != "" {
+					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
+				} else {
+					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
+				}
+			}(t)
 		}
+		ewk.Wait()
 	}()
 	go func() {
 		for t := range globalExpiryState.byNewerNoncurrentCh {
-			deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+			nwk.Take()
+			go func(t newerNoncurrentTask) {
+				defer nwk.Give()
+				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+			}(t)
 		}
+		nwk.Wait()
 	}()
 }
 
@@ -333,6 +385,7 @@ const (
 // 1. when a restored (via PostRestoreObject API) object expires.
 // 2. when a transitioned object expires (based on an ILM rule).
 func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error {
+	traceFn := globalLifecycleSys.trace(*oi)
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name)
 	opts.VersionID = lcOpts.VersionID
@@ -358,7 +411,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		}
 
 		// Send audit for the lifecycle delete operation
-		auditLogLifecycle(ctx, *oi, ILMExpiry)
+		defer auditLogLifecycle(ctx, *oi, ILMExpiry, traceFn)
 
 		eventName := event.ObjectRemovedDelete
 		if lcOpts.DeleteMarker {
@@ -374,7 +427,8 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 			EventName:  eventName,
 			BucketName: oi.Bucket,
 			Object:     objInfo,
-			Host:       "Internal: [ILM-Expiry]",
+			UserAgent:  "Internal: [ILM-Expiry]",
+			Host:       globalLocalNodeName,
 		})
 
 	case expireRestoredObj:

cmd/bucket-listobjects-handlers.go

@@ -59,10 +59,18 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max
 	return ErrNone
 }
 
-// ListObjectVersions - GET Bucket Object versions
+func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+	api.listObjectVersionsHandler(w, r, false)
+}
+
+func (api objectAPIHandlers) ListObjectVersionsMHandler(w http.ResponseWriter, r *http.Request) {
+	api.listObjectVersionsHandler(w, r, true)
+}
+
+// ListObjectVersionsHandler - GET Bucket Object versions
 // You can use the versions subresource to list metadata about all
 // of the versions of objects in a bucket.
-func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) listObjectVersionsHandler(w http.ResponseWriter, r *http.Request, metadata bool) {
 	ctx := newContext(r, w, "ListObjectVersions")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -80,7 +88,12 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
-
+	var checkObjMeta metaCheckFn
+	if metadata {
+		checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
+			return checkRequestAuthType(ctx, r, action, bucket, name)
+		}
+	}
 	urlValues := r.Form
 
 	// Extract all the listBucketVersions query params to their native values.
@@ -111,8 +124,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
-	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
+	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo, checkObjMeta)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponseList(response))
@@ -128,64 +140,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 // MinIO continues to support ListObjectsV1 and V2 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2M")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-
-	urlValues := r.Form
-
-	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
-	if errCode != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
-		return
-	}
-
-	// Validate the query params before beginning to serve the request.
-	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-
-	listObjectsV2 := objectAPI.ListObjectsV2
-
-	// Inititate a list objects operation based on the input params.
-	// On success would return back ListObjectsInfo object to be
-	// marshaled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	// The next continuation token has id@node_index format to optimize paginated listing
-	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-
-	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
-		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
-		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true)
-
-	// Write success response.
-	writeSuccessResponseXML(w, encodeResponseList(response))
+	api.listObjectsV2Handler(ctx, w, r, true)
 }
 
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
@@ -198,7 +153,11 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 // MinIO continues to support ListObjectsV1 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
+	api.listObjectsV2Handler(ctx, w, r, false)
+}
 
+// listObjectsV2Handler performs listing either with or without extra metadata.
+func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.ResponseWriter, r *http.Request, metadata bool) {
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
@@ -215,6 +174,12 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
+	var checkObjMeta metaCheckFn
+	if metadata {
+		checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
+			return checkRequestAuthType(ctx, r, action, bucket, name)
+		}
+	}
 	urlValues := r.Form
 
 	// Extract all the listObjectsV2 query params to their native values.
@@ -257,7 +222,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
-		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
+		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, checkObjMeta)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponseList(response))

cmd/bucket-replication-handlers.go

@@ -277,6 +277,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 				Bucket: bucket,
 				Err:    fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
 			}), r.URL)
+			return
 		}
 	}
 	resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24))
@@ -347,6 +348,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		}
+		return
 	}
 	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	if err != nil {

cmd/bucket-replication.go

@@ -387,7 +387,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -403,7 +404,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -424,7 +426,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -449,7 +452,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 					VersionID:    versionID,
 					DeleteMarker: dobj.DeleteMarker,
 				},
-				Host:      "Internal: [Replication]",
+				UserAgent: "Internal: [Replication]",
+				Host:      globalLocalNodeName,
 				EventName: event.ObjectReplicationNotTracked,
 			})
 			continue
@@ -508,7 +512,6 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
 	if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on.
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %s", bucket, dobj.ObjectName, versionID, err))
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -517,14 +520,16 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: eventName,
 		})
 	} else {
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object:     dobjInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 			EventName:  eventName,
 		})
 	}
@@ -561,7 +566,8 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 				VersionID:    dobj.VersionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		if dobj.VersionID == "" {
@@ -921,7 +927,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -939,7 +946,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
 		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, ri.TargetArn))
@@ -959,7 +967,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			continue
 		}
@@ -1005,10 +1014,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			},
 		}
 
-		if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w",
-				bucket, objInfo.Name, objInfo.VersionID, err))
-		}
+		_, _ = objectAPI.PutObjectMetadata(ctx, bucket, object, popts)
 		opType := replication.MetadataReplicationType
 		if rinfos.Action() == replicateAll {
 			opType = replication.ObjectReplicationType
@@ -1024,7 +1030,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		EventName:  eventName,
 		BucketName: bucket,
 		Object:     objInfo,
-		Host:       "Internal: [Replication]",
+		UserAgent:  "Internal: [Replication]",
+		Host:       globalLocalNodeName,
 	})
 
 	// re-queue failures once more - keep a retry count to avoid flooding the queue if
@@ -1069,7 +1076,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1088,7 +1096,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
@@ -1105,7 +1114,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1116,7 +1126,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return rinfo
 	}
@@ -1141,7 +1152,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1212,7 +1224,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1231,7 +1244,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
@@ -1255,7 +1269,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1266,7 +1281,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return rinfo
 	}
@@ -1296,7 +1312,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 					EventName:  event.ObjectReplicationNotTracked,
 					BucketName: bucket,
 					Object:     objInfo,
-					Host:       "Internal: [Replication]",
+					UserAgent:  "Internal: [Replication]",
+					Host:       globalLocalNodeName,
 				})
 			}
 			// object with same VersionID already exists, replication kicked off by
@@ -1344,7 +1361,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			return
 		}
@@ -1424,7 +1442,12 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		if err != nil {
 			return err
 		}
-		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, "", "", opts.ServerSideEncryption)
+
+		popts := minio.PutObjectPartOptions{
+			SSE: opts.ServerSideEncryption,
+		}
+
+		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
 		if err != nil {
 			return err
 		}

cmd/bucket-targets.go

@@ -208,7 +208,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	}
 	clnt, err := sys.getRemoteTargetClient(tgt)
 	if err != nil {
-		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket, Err: err}
 	}
 	// validate if target credentials are ok
 	exists, err := clnt.BucketExists(ctx, tgt.TargetBucket)

cmd/common-main.go

@@ -257,6 +257,9 @@ func initConsoleServer() (*restapi.Server, error) {
 		Path: globalCertsCADir.Get(),
 	}
 
+	// set certs before other console initialization
+	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
+
 	swaggerSpec, err := loads.Embedded(restapi.SwaggerJSON, restapi.FlatSwaggerJSON)
 	if err != nil {
 		return nil, err
@@ -283,8 +286,6 @@ func initConsoleServer() (*restapi.Server, error) {
 	// register all APIs
 	server.ConfigureAPI()
 
-	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
-
 	consolePort, _ := strconv.Atoi(globalMinioConsolePort)
 
 	server.Host = globalMinioConsoleHost

cmd/data-scanner.go

@@ -957,6 +957,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 // applyTierObjSweep removes remote object pending deletion and the free-version
 // tracking this information.
 func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
+	traceFn := globalLifecycleSys.trace(oi)
 	if !oi.TransitionedObject.FreeVersion {
 		// nothing to be done
 		return
@@ -982,7 +983,7 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 		InclFreeVersions: true,
 	})
 	if err == nil {
-		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete)
+		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, traceFn)
 	}
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
@@ -991,27 +992,35 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 
 // applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
 // Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
-func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
-	if i.lifeCycle == nil {
-		return fivs, nil
-	}
+func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
 	done := globalScannerMetrics.time(scannerMetricApplyNonCurrent)
 	defer done()
 
-	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
-	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
-		return fivs, nil
-	}
-
-	overflowVersions := fivs[lim+1:]
-	// current version + most recent lim noncurrent versions
-	fivs = fivs[:lim+1]
-
 	rcfg, _ := globalBucketObjectLockSys.Get(i.bucket)
 	vcfg, _ := globalBucketVersioningSys.Get(i.bucket)
 
 	versioned := vcfg != nil && vcfg.Versioned(i.objectPath())
 
+	// current version + most recent lim noncurrent versions
+	objectInfos := make([]ObjectInfo, 0, len(fivs))
+
+	if i.lifeCycle == nil {
+		for _, fi := range fivs {
+			objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
+		}
+		return objectInfos, nil
+	}
+
+	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
+		for _, fi := range fivs {
+			objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
+		}
+		return objectInfos, nil
+	}
+
+	overflowVersions := fivs[lim+1:]
+
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
 		obj := fi.ToObjectInfo(i.bucket, i.objectPath(), versioned)
@@ -1026,7 +1035,7 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 			}
 			// add this version back to remaining versions for
 			// subsequent lifecycle policy applications
-			fivs = append(fivs, fi)
+			objectInfos = append(objectInfos, obj)
 			continue
 		}
 
@@ -1040,19 +1049,19 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 
 		toDel = append(toDel, ObjectToDelete{
 			ObjectV: ObjectV{
-				ObjectName: fi.Name,
-				VersionID:  fi.VersionID,
+				ObjectName: obj.Name,
+				VersionID:  obj.VersionID,
 			},
 		})
 	}
 
 	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel)
-	return fivs, nil
+	return objectInfos, nil
 }
 
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
-func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
+func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
 	if i.heal.enabled {
 		if healDeleteDangling {
 			done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
@@ -1063,13 +1072,14 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			}
 		}
 	}
-	fivs, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+
+	objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
 	if err != nil {
-		return fivs, err
+		return nil, err
 	}
 
 	// Check if we have many versions after applyNewerNoncurrentVersionLimit.
-	if len(fivs) > dataScannerExcessiveVersionsThreshold {
+	if len(objInfos) > dataScannerExcessiveVersionsThreshold {
 		// Notify object accessed via a GET request.
 		sendEvent(eventArgs{
 			EventName:  event.ObjectManyVersions,
@@ -1077,13 +1087,13 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			Object: ObjectInfo{
 				Name: i.objectPath(),
 			},
-			UserAgent:    "scanner",
-			Host:         globalMinioHost,
+			UserAgent:    "Internal: [Scanner]",
+			Host:         globalLocalNodeName,
 			RespElements: map[string]string{"x-minio-versions": strconv.Itoa(len(fivs))},
 		})
 	}
 
-	return fivs, nil
+	return objInfos, nil
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -1168,6 +1178,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 }
 
 func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
+	traceFn := globalLifecycleSys.trace(obj)
 	opts := ObjectOptions{
 		Expiration: ExpirationOptions{Expire: true},
 	}
@@ -1191,7 +1202,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	}
 
 	// Send audit for the lifecycle delete operation
-	auditLogLifecycle(ctx, obj, ILMExpiry)
+	auditLogLifecycle(ctx, obj, ILMExpiry, traceFn)
 
 	eventName := event.ObjectRemovedDelete
 	if obj.DeleteMarker {
@@ -1203,7 +1214,8 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		EventName:  eventName,
 		BucketName: obj.Bucket,
 		Object:     obj,
-		Host:       "Internal: [ILM-Expiry]",
+		UserAgent:  "Internal: [ILM-Expiry]",
+		Host:       globalLocalNodeName,
 	})
 
 	return true
@@ -1433,7 +1445,7 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, traceFn func(event string)) {
 	var apiName string
 	switch event {
 	case ILMExpiry:
@@ -1450,4 +1462,5 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
 		Object:    oi.Name,
 		VersionID: oi.VersionID,
 	})
+	traceFn(event)
 }

cmd/erasure-healing.go

@@ -294,6 +294,26 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 	return false
 }
 
+const xMinIOHealing = ReservedMetadataPrefix + "healing"
+
+// SetHealing marks object (version) as being healed.
+// Note: this is to be used only from healObject
+func (fi *FileInfo) SetHealing() {
+	if fi.Metadata == nil {
+		fi.Metadata = make(map[string]string)
+	}
+	fi.Metadata[xMinIOHealing] = "true"
+}
+
+// Healing returns true if object is being healed (i.e fi is being passed down
+// from healObject)
+func (fi FileInfo) Healing() bool {
+	if _, ok := fi.Metadata[xMinIOHealing]; ok {
+		return true
+	}
+	return false
+}
+
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
 	dryRun := opts.DryRun
@@ -505,8 +525,34 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	migrateDataDir := mustGetUUID()
 
 	// Reorder so that we have data disks first and parity disks next.
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(availableDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from available disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, availableDisks, bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
+
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(outDatedDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from outdated disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, outDatedDisks, bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution)
+
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(partsMetadata) {
+		err := fmt.Errorf("unexpected file distribution (%v) from metadata entries (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, len(partsMetadata), bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution)
 
 	copyPartsMetadata := make([]FileInfo, len(partsMetadata))
@@ -638,6 +684,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		partsMetadata[i].Erasure.Index = i + 1
 
 		// Attempt a rename now from healed data to final location.
+		partsMetadata[i].SetHealing()
 		if _, err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
 			logger.LogIf(ctx, err)
 			return result, err

cmd/erasure-object.go

@@ -658,6 +658,13 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		return fi, nil, nil, err
 	}
 
+	if !fi.Deleted && len(fi.Erasure.Distribution) != len(onlineDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			fi.Erasure.Distribution, onlineDisks, bucket, object, opts.VersionID)
+		logger.LogIf(ctx, err)
+		return fi, nil, nil, toObjectErr(err, bucket, object, opts.VersionID)
+	}
+
 	filterOnlineDisksInplace(fi, metaArr, onlineDisks)
 
 	// if one of the disk is offline, return right here no need
@@ -972,17 +979,14 @@ func healObjectVersionsDisparity(bucket string, entry metaCacheEntry) error {
 
 	fivs, err := entry.fileInfoVersions(bucket)
 	if err != nil {
-		go healObject(bucket, entry.name, "", madmin.HealNormalScan)
+		go healObject(bucket, entry.name, "", madmin.HealDeepScan)
 		return err
 	}
 
-	if len(fivs.Versions) > 2 {
-		return fmt.Errorf("bucket(%s)/object(%s) object with %d versions needs healing, allowing lazy healing via scanner instead here for versions greater than 2",
-			bucket, entry.name, len(fivs.Versions))
-	}
-
-	for _, version := range fivs.Versions {
-		go healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
+	if len(fivs.Versions) <= 2 {
+		for _, version := range fivs.Versions {
+			go healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
+		}
 	}
 
 	return nil
@@ -994,7 +998,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	if opts.CheckPrecondFn != nil {
 		obj, err := er.getObjectInfo(ctx, bucket, object, opts)
-		if err != nil && !isErrVersionNotFound(err) {
+		if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
 			return objInfo, err
 		}
 		if opts.CheckPrecondFn(obj) {
@@ -1474,8 +1478,11 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 
 	// Check failed deletes across multiple objects
 	for i, dobj := range dobjects {
-		// This object errored, no need to attempt a heal.
-		if errs[i] != nil {
+		// This object errored, we should attempt a heal just in case.
+		if errs[i] != nil && !isErrVersionNotFound(errs[i]) && !isErrObjectNotFound(errs[i]) {
+			// all other direct versionId references we should
+			// ensure no dangling file is left over.
+			er.addPartial(bucket, dobj.ObjectName, dobj.VersionID, -1)
 			continue
 		}
 
@@ -1644,6 +1651,18 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	}
 	fvID := mustGetUUID()
 
+	defer func() {
+		// attempt a heal before returning if there are offline disks
+		// for both del marker and permanent delete situations.
+		for _, disk := range storageDisks {
+			if disk != nil && disk.IsOnline() {
+				continue
+			}
+			er.addPartial(bucket, object, opts.VersionID, -1)
+			break
+		}
+	}()
+
 	if markDelete && (opts.Versioned || opts.VersionSuspended) {
 		if !deleteMarker {
 			// versioning suspended means we add `null` version as
@@ -1691,14 +1710,6 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 
-	for _, disk := range storageDisks {
-		if disk != nil && disk.IsOnline() {
-			continue
-		}
-		er.addPartial(bucket, object, opts.VersionID, -1)
-		break
-	}
-
 	return dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
@@ -1942,6 +1953,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 			return toObjectErr(err, bucket, object)
 		}
 	}
+	traceFn := globalLifecycleSys.trace(fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended))
 
 	destObj, err := genTransitionObjName(bucket)
 	if err != nil {
@@ -1986,9 +1998,10 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		EventName:  eventName,
 		BucketName: bucket,
 		Object:     objInfo,
-		Host:       "Internal: [ILM-Transition]",
+		UserAgent:  "Internal: [ILM-Transition]",
+		Host:       globalLocalNodeName,
 	})
-	auditLogLifecycle(ctx, objInfo, ILMTransition)
+	auditLogLifecycle(ctx, objInfo, ILMTransition, traceFn)
 	return err
 }
 
@@ -2092,3 +2105,46 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 	})
 	return setRestoreHeaderFn(oi, err)
 }
+
+// DecomTieredObject - moves tiered object to another pool during decommissioning.
+func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
+	if opts.UserDefined == nil {
+		opts.UserDefined = make(map[string]string)
+	}
+	// overlay Erasure info for this set of disks
+	storageDisks := er.getDisks()
+	// Get parity and data drive count based on storage class metadata
+	parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
+	if parityDrives < 0 {
+		parityDrives = er.defaultParityCount
+	}
+	dataDrives := len(storageDisks) - parityDrives
+
+	// we now know the number of blocks this object needs for data and parity.
+	// writeQuorum is dataBlocks + 1
+	writeQuorum := dataDrives
+	if dataDrives == parityDrives {
+		writeQuorum++
+	}
+
+	// Initialize parts metadata
+	partsMetadata := make([]FileInfo, len(storageDisks))
+
+	fi2 := newFileInfo(pathJoin(bucket, object), dataDrives, parityDrives)
+	fi.Erasure = fi2.Erasure
+	// Initialize erasure metadata.
+	for index := range partsMetadata {
+		partsMetadata[index] = fi
+		partsMetadata[index].Erasure.Index = index + 1
+	}
+
+	// Order disks according to erasure distribution
+	var onlineDisks []StorageAPI
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi)
+
+	if _, err := writeUniqueFileInfo(ctx, onlineDisks, bucket, object, partsMetadata, writeQuorum); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	return nil
+}

cmd/erasure-server-pool-decom.go

@@ -736,14 +736,16 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
 			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
-			if evt.Action.Delete() {
+			switch {
+			case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
+				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt.Action.DeleteRestored())
+				return false
+			case evt.Action.Delete():
 				globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
-				if !evt.Action.DeleteRestored() {
-					return true
-				}
+				return true
+			default:
+				return false
 			}
-
-			return false
 		}
 
 		decommissionEntry := func(entry metaCacheEntry) {
@@ -767,12 +769,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 			var decommissionedCount int
 			for _, version := range fivs.Versions {
-				// TODO: Skip transitioned objects for now.
-				if version.IsRemote() {
-					logger.LogIf(ctx, fmt.Errorf("found %s/%s transitioned object, transitioned object won't be decommissioned", bi.Name, version.Name))
-					continue
-				}
-
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bi.Name, version.Name, version) {
 					logger.LogIf(ctx, fmt.Errorf("found %s/%s (%s) expired object based on ILM rules, skipping and scheduled for deletion", bi.Name, version.Name, version.VersionID))
@@ -821,6 +817,22 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				var failure, ignore bool
 				// gr.Close() is ensured by decommissionObject().
 				for try := 0; try < 3; try++ {
+					if version.IsRemote() {
+						stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
+						if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
+							VersionID:   version.VersionID,
+							MTime:       version.ModTime,
+							UserDefined: version.Metadata,
+						}); err != nil {
+							stopFn(err)
+							failure = true
+							logger.LogIf(ctx, err)
+							continue
+						}
+						stopFn(nil)
+						failure = false
+						break
+					}
 					gr, err := set.GetObjectNInfo(ctx,
 						bi.Name,
 						encodeDirObject(version.Name),
@@ -1268,17 +1280,6 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, indices ...i
 		z.HealBucket(ctx, bucket.Name, madmin.HealOpts{})
 	}
 
-	// TODO: Support decommissioning transition tiers.
-	for _, bucket := range decomBuckets {
-		if lc, err := globalLifecycleSys.Get(bucket.Name); err == nil {
-			if lc.HasTransition() {
-				return decomError{
-					Err: fmt.Sprintf("Bucket is part of transitioned tier %s: decommission is not allowed in Tier'd setups", bucket.Name),
-				}
-			}
-		}
-	}
-
 	// Create .minio.sys/conifg, .minio.sys/buckets paths if missing,
 	// this code is present to avoid any missing meta buckets on other
 	// pools.

cmd/erasure-server-pool.go

@@ -346,11 +346,17 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 	g.Wait()
 
 	for i, zinfo := range storageInfos {
-		var available uint64
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(zinfo, size) {
+		if zinfo == nil {
 			serverPools[i] = poolAvailableSpace{Index: i}
 			continue
 		}
+		var available uint64
+		if !isMinioMetaBucketName(bucket) {
+			if avail, err := hasSpaceFor(zinfo, size); err != nil && !avail {
+				serverPools[i] = poolAvailableSpace{Index: i}
+				continue
+			}
+		}
 		var maxUsedPct int
 		for _, disk := range zinfo {
 			if disk == nil || disk.Total == 0 {
@@ -426,30 +432,37 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
 	for _, pinfo := range poolObjInfos {
 		// skip all objects from suspended pools if asked by the
 		// caller.
-		if z.IsSuspended(pinfo.Index) && opts.SkipDecommissioned {
+		if opts.SkipDecommissioned && z.IsSuspended(pinfo.Index) {
 			continue
 		}
 		// Skip object if it's from pools participating in a rebalance operation.
 		if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) {
 			continue
 		}
-
-		if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
-			return pinfo, pinfo.Err
+		if pinfo.Err == nil {
+			// found a pool
+			return pinfo, nil
+		}
+		if isErrReadQuorum(pinfo.Err) {
+			// read quorum is returned when the object is visibly
+			// present but its unreadable, we simply ask the writes to
+			// schedule to this pool instead. If there is no quorum
+			// it will fail anyways, however if there is quorum available
+			// with enough disks online but sufficiently inconsistent to
+			// break parity threshold, allow them to be overwritten
+			// or allow new versions to be added.
+			return pinfo, nil
 		}
 		defPool = pinfo
-		if isErrObjectNotFound(pinfo.Err) {
-			// No object exists or its a delete marker,
-			// check objInfo to confirm.
-			if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
-				return pinfo, nil
-			}
-			// objInfo is not valid, truly the object doesn't
-			// exist proceed to next pool.
-			continue
+		if !isErrObjectNotFound(pinfo.Err) {
+			return pinfo, pinfo.Err
 		}
 
-		return pinfo, nil
+		// No object exists or its a delete marker,
+		// check objInfo to confirm.
+		if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
+			return pinfo, nil
+		}
 	}
 	if opts.ReplicationRequest && opts.DeleteMarker && defPool.Index >= 0 {
 		// If the request is a delete marker replication request, return a default pool
@@ -934,8 +947,15 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 	object = encodeDirObject(object)
 
 	if z.SinglePool() {
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), data.Size()) {
-			return ObjectInfo{}, toObjectErr(errDiskFull)
+		if !isMinioMetaBucketName(bucket) {
+			avail, err := hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), data.Size())
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return ObjectInfo{}, toObjectErr(errErasureWriteQuorum)
+			}
+			if !avail {
+				return ObjectInfo{}, toObjectErr(errDiskFull)
+			}
 		}
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
 	}
@@ -1399,8 +1419,15 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}
 
 	if z.SinglePool() {
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), -1) {
-			return nil, toObjectErr(errDiskFull)
+		if !isMinioMetaBucketName(bucket) {
+			avail, err := hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), -1)
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return nil, toObjectErr(errErasureWriteQuorum)
+			}
+			if !avail {
+				return nil, toObjectErr(errDiskFull)
+			}
 		}
 		return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
 	}
@@ -1638,14 +1665,14 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 	}
 
 	err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
-	if err == nil || errors.Is(err, errVolumeNotFound) {
+	if err == nil || isErrBucketNotFound(err) {
 		// If site replication is configured, hold on to deleted bucket state until sites sync
 		if opts.SRDeleteOp == MarkDelete {
 			z.s3Peer.MakeBucket(context.Background(), pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket), MakeBucketOptions{})
 		}
 	}
 
-	if err != nil && !errors.Is(err, errVolumeNotFound) {
+	if err != nil && !isErrBucketNotFound(err) {
 		if !opts.NoRecreate {
 			z.s3Peer.MakeBucket(ctx, bucket, MakeBucketOptions{})
 		}
@@ -2077,9 +2104,14 @@ func (z *erasureServerPools) getPoolAndSet(id string) (poolIdx, setIdx, diskIdx
 	return -1, -1, -1, fmt.Errorf("DriveID(%s) %w", id, errDiskNotFound)
 }
 
+const (
+	vmware = "VMWare"
+)
+
 // HealthOptions takes input options to return sepcific information
 type HealthOptions struct {
-	Maintenance bool
+	Maintenance    bool
+	DeploymentType string
 }
 
 // HealthResult returns the current state of the system, also
@@ -2165,7 +2197,8 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	}
 
 	var aggHealStateResult madmin.BgHealState
-	if opts.Maintenance {
+	// Check if disks are healing on in-case of VMware vsphere deployments.
+	if opts.Maintenance && opts.DeploymentType == vmware {
 		// check if local disks are being healed, if they are being healed
 		// we need to tell healthy status as 'false' so that this server
 		// is not taken down for maintenance
@@ -2348,3 +2381,27 @@ func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, ob
 	}
 	return nil
 }
+
+// DecomTieredObject - moves tiered object to another pool during decommissioning.
+func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
+	object = encodeDirObject(object)
+	if z.SinglePool() {
+		return fmt.Errorf("error decommissioning %s/%s", bucket, object)
+	}
+	if !opts.NoLock {
+		ns := z.NewNSLock(bucket, object)
+		lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return err
+		}
+		ctx = lkctx.Context()
+		defer ns.Unlock(lkctx)
+		opts.NoLock = true
+	}
+	idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size)
+	if err != nil {
+		return err
+	}
+
+	return z.serverPools[idx].DecomTieredObject(ctx, bucket, object, fi, opts)
+}

cmd/erasure-sets.go

@@ -1215,6 +1215,12 @@ func (s *erasureSets) PutObjectMetadata(ctx context.Context, bucket, object stri
 	return er.PutObjectMetadata(ctx, bucket, object, opts)
 }
 
+// DecomTieredObject - moves tiered object to another pool during decommissioning.
+func (s *erasureSets) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
+	er := s.getHashedSet(object)
+	return er.DecomTieredObject(ctx, bucket, object, fi, opts)
+}
+
 // PutObjectTags - replace or add tags to an existing object
 func (s *erasureSets) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) {
 	er := s.getHashedSet(object)

cmd/erasure.go

@@ -207,6 +207,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				Scanning:       info.Scanning,
 				State:          diskErrToDriveState(err),
 				FreeInodes:     info.FreeInodes,
+				UsedInodes:     info.UsedInodes,
 			}
 			di.PoolIndex, di.SetIndex, di.DiskIndex = disks[index].GetDiskLoc()
 			if info.Healing {

cmd/generic-handlers.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"net/http"
 	"path"
+	"path/filepath"
 	"runtime/debug"
 	"strings"
 	"sync/atomic"
@@ -35,7 +36,6 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/http/stats"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
 )
@@ -111,6 +111,7 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrUnsupportedMetadata), r.URL)
 			return
 		}
@@ -121,6 +122,7 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsHeader, 1)
 			return
@@ -133,9 +135,8 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 
 // Reserved bucket.
 const (
-	minioReservedBucket              = "minio"
-	minioReservedBucketPath          = SlashSeparator + minioReservedBucket
-	minioReservedBucketPathWithSlash = SlashSeparator + minioReservedBucket + SlashSeparator
+	minioReservedBucket     = "minio"
+	minioReservedBucketPath = SlashSeparator + minioReservedBucket
 
 	loginPathPrefix = SlashSeparator + "login"
 )
@@ -277,60 +278,6 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
 	return time.Time{}, ErrMissingDateHeader
 }
 
-// splitStr splits a string into n parts, empty strings are added
-// if we are not able to reach n elements
-func splitStr(path, sep string, n int) []string {
-	splits := strings.SplitN(path, sep, n)
-	// Add empty strings if we found elements less than nr
-	for i := n - len(splits); i > 0; i-- {
-		splits = append(splits, "")
-	}
-	return splits
-}
-
-func url2Bucket(p string) (bucket string) {
-	tokens := splitStr(p, SlashSeparator, 3)
-	return tokens[1]
-}
-
-// setHttpStatsHandler sets a http Stats handler to gather HTTP statistics
-func setHTTPStatsHandler(h http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Meters s3 connection stats.
-		meteredRequest := &stats.IncomingTrafficMeter{ReadCloser: r.Body}
-		meteredResponse := &stats.OutgoingTrafficMeter{ResponseWriter: w}
-
-		// Execute the request
-		r.Body = meteredRequest
-		h.ServeHTTP(meteredResponse, r)
-
-		if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
-			strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
-			strings.HasPrefix(r.URL.Path, peerS3Prefix) ||
-			strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
-			globalConnStats.incInputBytes(meteredRequest.BytesRead())
-			globalConnStats.incOutputBytes(meteredResponse.BytesWritten())
-			return
-		}
-		if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
-			globalConnStats.incAdminInputBytes(meteredRequest.BytesRead())
-			globalConnStats.incAdminOutputBytes(meteredResponse.BytesWritten())
-			return
-		}
-
-		globalConnStats.incS3InputBytes(meteredRequest.BytesRead())
-		globalConnStats.incS3OutputBytes(meteredResponse.BytesWritten())
-
-		if r.URL != nil {
-			bucket := url2Bucket(r.URL.Path)
-			if bucket != "" && bucket != minioReservedBucket {
-				globalBucketConnStats.incS3InputBytes(bucket, meteredRequest.BytesRead())
-				globalBucketConnStats.incS3OutputBytes(bucket, meteredResponse.BytesWritten())
-			}
-		}
-	})
-}
-
 // Bad path components to be rejected by the path validity handler.
 const (
 	dotdotComponent = ".."
@@ -349,7 +296,7 @@ func hasBadHost(host string) error {
 // Check if the incoming path has bad path components,
 // such as ".." and "."
 func hasBadPathComponent(path string) bool {
-	path = strings.TrimSpace(path)
+	path = filepath.ToSlash(strings.TrimSpace(path)) // For windows '\' must be converted to '/'
 	for _, p := range strings.Split(path, SlashSeparator) {
 		switch strings.TrimSpace(p) {
 		case dotdotComponent:
@@ -388,6 +335,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (%s)", invalidReq.Description, err)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -402,6 +350,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 			return
@@ -415,6 +364,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 						tc.ResponseRecorder.LogErrBody = true
 					}
 
+					defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 					writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 					atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 					return
@@ -427,6 +377,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (request has multiple authentication types, please use one)", invalidReq.Description)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -441,6 +392,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 					tc.FuncName = "handler.ValidRequest"
 					tc.ResponseRecorder.LogErrBody = true
 				}
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL)
 				return
 			}
@@ -453,6 +405,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = false
 				}
 
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest))
 			} else {
 				if ok {
@@ -460,6 +413,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = true
 				}
 
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL)
 			}
 			return
@@ -510,6 +464,7 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 		}
 		sr, err := globalDNSConfig.Get(bucket)
 		if err != nil {
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			if err == dns.ErrNoEntriesFound {
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrNoSuchBucket), r.URL)
 			} else {
@@ -554,7 +509,7 @@ func addCustomHeaders(h http.Handler) http.Handler {
 		if globalLocalNodeName != "" {
 			w.Header().Set(xhttp.AmzRequestHostID, globalLocalNodeNameHex)
 		}
-		h.ServeHTTP(xhttp.NewResponseRecorder(w), r)
+		h.ServeHTTP(w, r)
 	})
 }
 
@@ -592,6 +547,7 @@ func setUploadForwardingHandler(h http.Handler) http.Handler {
 			h.ServeHTTP(w, r)
 			return
 		}
+
 		bucket, object := request2BucketObjectName(r)
 		uploadID := r.Form.Get(xhttp.UploadID)
 
@@ -608,6 +564,7 @@ func setUploadForwardingHandler(h http.Handler) http.Handler {
 			}
 			// forward request to peer handling this upload
 			if globalBucketTargetSys.isOffline(remote.EndpointURL) {
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrReplicationRemoteConnectionError), r.URL)
 				return
 			}

cmd/handler-api.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/internal/config/api"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/mcontext"
 )
 
 type apiConfig struct {
@@ -272,6 +273,10 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 
 		globalHTTPStats.addRequestsInQueue(1)
 
+		if tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt); ok {
+			tc.FuncName = "s3.MaxClients"
+		}
+
 		deadlineTimer := time.NewTimer(deadline)
 		defer deadlineTimer.Stop()
 
@@ -288,6 +293,10 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 			globalHTTPStats.addRequestsInQueue(-1)
 			return
 		case <-r.Context().Done():
+			// When the client disconnects before getting the S3 handler
+			// status code response, set the status code to 499 so this request
+			// will be properly audited and traced.
+			w.WriteHeader(499)
 			globalHTTPStats.addRequestsInQueue(-1)
 			return
 		}

cmd/handler-utils.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/mcontext"
 	xnet "github.com/minio/pkg/net"
 )
 
@@ -355,11 +356,45 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 		globalHTTPStats.currentS3Requests.Inc(api)
 		defer globalHTTPStats.currentS3Requests.Dec(api)
 
-		statsWriter := xhttp.NewResponseRecorder(w)
+		f.ServeHTTP(w, r)
 
-		f.ServeHTTP(statsWriter, r)
+		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
+		if !ok {
+			return
+		}
 
-		globalHTTPStats.updateStats(api, r, statsWriter)
+		if tc != nil {
+			if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
+				strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
+				strings.HasPrefix(r.URL.Path, peerS3Prefix) ||
+				strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
+				globalConnStats.incInputBytes(int64(tc.RequestRecorder.Size()))
+				globalConnStats.incOutputBytes(int64(tc.ResponseRecorder.Size()))
+				return
+			}
+
+			if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
+				globalConnStats.incAdminInputBytes(int64(tc.RequestRecorder.Size()))
+				globalConnStats.incAdminOutputBytes(int64(tc.ResponseRecorder.Size()))
+				return
+			}
+
+			globalHTTPStats.updateStats(api, tc.ResponseRecorder)
+			globalConnStats.incS3InputBytes(int64(tc.RequestRecorder.Size()))
+			globalConnStats.incS3OutputBytes(int64(tc.ResponseRecorder.Size()))
+
+			resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
+			if err != nil {
+				logger.LogIf(r.Context(), fmt.Errorf("Unable to get the actual resource in the incoming request: %v", err))
+				return
+			}
+
+			bucket, _ := path2BucketObject(resource)
+			if bucket != "" && bucket != minioReservedBucket {
+				globalBucketConnStats.incS3InputBytes(bucket, int64(tc.RequestRecorder.Size()))
+				globalBucketConnStats.incS3OutputBytes(bucket, int64(tc.ResponseRecorder.Size()))
+			}
+		}
 	}
 }
 

cmd/healthcheck-handler.go

@@ -47,7 +47,10 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
 	defer cancel()
 
-	opts := HealthOptions{Maintenance: r.Form.Get("maintenance") == "true"}
+	opts := HealthOptions{
+		Maintenance:    r.Form.Get("maintenance") == "true",
+		DeploymentType: r.Form.Get("deployment-type"),
+	}
 	result := objLayer.Health(ctx, opts)
 	if result.WriteQuorum > 0 {
 		w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum))

cmd/http-stats.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"net/http"
-	"strings"
 	"sync"
 	"sync/atomic"
 
@@ -299,12 +298,7 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 }
 
 // Update statistics from http request and response data
-func (st *HTTPStats) updateStats(api string, r *http.Request, w *xhttp.ResponseRecorder) {
-	// Ignore non S3 requests
-	if strings.HasSuffix(r.URL.Path, minioReservedBucketPathWithSlash) {
-		return
-	}
-
+func (st *HTTPStats) updateStats(api string, w *xhttp.ResponseRecorder) {
 	st.totalS3Requests.Inc(api)
 
 	// Increment the prometheus http request response histogram with appropriate label

cmd/http-tracer.go

@@ -39,7 +39,7 @@ var ldapPwdRegex = regexp.MustCompile("(^.*?)LDAPPassword=([^&]*?)(&(.*?))?$")
 // redact LDAP password if part of string
 func redactLDAPPwd(s string) string {
 	parts := ldapPwdRegex.FindStringSubmatch(s)
-	if len(parts) > 0 {
+	if len(parts) > 3 {
 		return parts[1] + "LDAPPassword=*REDACTED*" + parts[3]
 	}
 	return s
@@ -67,41 +67,37 @@ func getOpName(name string) (op string) {
 // otherwise, generate a trace event with request information but no response.
 func httpTracer(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if globalTrace.NumSubscribers(madmin.TraceS3|madmin.TraceInternal) == 0 {
-			h.ServeHTTP(w, r)
-			return
-		}
+		// Setup a http request response recorder - this is needed for
+		// http stats requests and audit if enabled.
+		respRecorder := xhttp.NewResponseRecorder(w)
+
+		// Setup a http request body recorder
+		reqRecorder := &xhttp.RequestRecorder{Reader: r.Body}
+		r.Body = reqRecorder
 
 		// Create tracing data structure and associate it to the request context
 		tc := mcontext.TraceCtxt{
-			AmzReqID: r.Header.Get(xhttp.AmzRequestID),
+			AmzReqID:         r.Header.Get(xhttp.AmzRequestID),
+			RequestRecorder:  reqRecorder,
+			ResponseRecorder: respRecorder,
 		}
 
-		ctx := context.WithValue(r.Context(), mcontext.ContextTraceKey, &tc)
-		r = r.WithContext(ctx)
-
-		// Setup a http request and response body recorder
-		reqRecorder := &xhttp.RequestRecorder{Reader: r.Body}
-		respRecorder := xhttp.NewResponseRecorder(w)
-
-		tc.RequestRecorder = reqRecorder
-		tc.ResponseRecorder = respRecorder
-
-		// Execute call.
-		r.Body = reqRecorder
+		r = r.WithContext(context.WithValue(r.Context(), mcontext.ContextTraceKey, &tc))
 
 		reqStartTime := time.Now().UTC()
 		h.ServeHTTP(respRecorder, r)
 		reqEndTime := time.Now().UTC()
 
+		if globalTrace.NumSubscribers(madmin.TraceS3|madmin.TraceInternal) == 0 {
+			// no subscribers nothing to trace.
+			return
+		}
+
 		tt := madmin.TraceInternal
 		if strings.HasPrefix(tc.FuncName, "s3.") {
 			tt = madmin.TraceS3
 		}
-		// No need to continue if no subscribers for actual type...
-		if globalTrace.NumSubscribers(tt) == 0 {
-			return
-		}
+
 		// Calculate input body size with headers
 		reqHeaders := r.Header.Clone()
 		reqHeaders.Set("Host", r.Host)
@@ -110,7 +106,7 @@ func httpTracer(h http.Handler) http.Handler {
 		} else {
 			reqHeaders.Set("Transfer-Encoding", strings.Join(r.TransferEncoding, ","))
 		}
-		inputBytes := reqRecorder.BodySize()
+		inputBytes := reqRecorder.Size()
 		for k, v := range reqHeaders {
 			inputBytes += len(k) + len(v)
 		}

cmd/iam-etcd-store.go

@@ -248,6 +248,13 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
+			if u.Credentials.IsTemp() {
+				// We should delete such that the client can re-request
+				// for the expiring credentials.
+				deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
+				deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
+				return nil
+			}
 			return err
 		}
 		u.Credentials.Claims = jwtClaims.Map()

cmd/iam-object-store.go

@@ -187,7 +187,15 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
+			if u.Credentials.IsTemp() {
+				// We should delete such that the client can re-request
+				// for the expiring credentials.
+				iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
+				iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
+				return nil
+			}
 			return err
+
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}

cmd/local-locker.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/minio/minio/internal/dsync"
@@ -50,7 +51,7 @@ func isWriteLock(lri []lockRequesterInfo) bool {
 
 // localLocker implements Dsync.NetLocker
 type localLocker struct {
-	mutex   chan struct{}
+	mutex   sync.Mutex
 	lockMap map[string][]lockRequesterInfo
 	lockUID map[string]string // UUID -> resource map.
 }
@@ -69,41 +70,13 @@ func (l *localLocker) canTakeLock(resources ...string) bool {
 	return true
 }
 
-// lockMu locks the "mutex" of the local locker.
-// If "ctx" is canceled before the lock can be obtained false is returned.
-// If "true" is returned unlockMu MUST be called.
-// Behavior is similar to sync.Mutex.
-func (l *localLocker) lockMu(ctx context.Context) (ok bool) {
-	select {
-	case l.mutex <- struct{}{}:
-		return true
-	case <-ctx.Done():
-		return false
-	}
-}
-
-// lockMuBlock will block while getting the mutex.
-// When returned unlockMu *must* be called.
-// Behavior is similar to sync.Mutex.
-func (l *localLocker) lockMuBlock() {
-	l.mutex <- struct{}{}
-}
-
-// unlockMu unlocks an acquired mutex.
-// This may only be called once after successfully getting a mutex.
-func (l *localLocker) unlockMu() {
-	<-l.mutex
-}
-
 func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
 	if len(args.Resources) > maxDeleteList {
 		return false, fmt.Errorf("internal error: localLocker.Lock called with more than %d resources", maxDeleteList)
 	}
 
-	if !l.lockMu(ctx) {
-		return false, ctx.Err()
-	}
-	defer l.unlockMu()
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 
 	if !l.canTakeLock(args.Resources...) {
 		// Not all locks can be taken on resources,
@@ -142,9 +115,8 @@ func (l *localLocker) Unlock(_ context.Context, args dsync.LockArgs) (reply bool
 		return false, fmt.Errorf("internal error: localLocker.Unlock called with more than %d resources", maxDeleteList)
 	}
 
-	l.lockMuBlock()
-	defer l.unlockMu()
-
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 	err = nil
 
 	for _, resource := range args.Resources {
@@ -191,11 +163,8 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 		return false, fmt.Errorf("internal error: localLocker.RLock called with more than one resource")
 	}
 
-	if !l.lockMu(ctx) {
-		return false, ctx.Err()
-	}
-	defer l.unlockMu()
-
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 	resource := args.Resources[0]
 	lrInfo := lockRequesterInfo{
 		Name:            resource,
@@ -227,9 +196,8 @@ func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply boo
 		return false, fmt.Errorf("internal error: localLocker.RUnlock called with more than one resource")
 	}
 
-	l.lockMuBlock()
-	defer l.unlockMu()
-
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 	var lri []lockRequesterInfo
 
 	resource := args.Resources[0]
@@ -245,12 +213,41 @@ func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply boo
 	return reply, nil
 }
 
+type lockStats struct {
+	Total  int
+	Writes int
+	Reads  int
+}
+
+func (l *localLocker) stats() lockStats {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+
+	st := lockStats{Total: len(l.lockMap)}
+	for _, v := range l.lockMap {
+		if len(v) == 0 {
+			continue
+		}
+		entry := v[0]
+		if entry.Writer {
+			st.Writes++
+		} else {
+			st.Reads += len(v)
+		}
+	}
+	return st
+}
+
 func (l *localLocker) DupLockMap() map[string][]lockRequesterInfo {
-	l.lockMuBlock()
-	defer l.unlockMu()
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 
 	lockCopy := make(map[string][]lockRequesterInfo, len(l.lockMap))
 	for k, v := range l.lockMap {
+		if len(v) == 0 {
+			delete(l.lockMap, k)
+			continue
+		}
 		lockCopy[k] = append(make([]lockRequesterInfo, 0, len(v)), v...)
 	}
 	return lockCopy
@@ -271,90 +268,97 @@ func (l *localLocker) IsLocal() bool {
 }
 
 func (l *localLocker) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
-	l.lockMuBlock()
-	defer l.unlockMu()
-
-	if len(args.UID) == 0 {
-		for _, resource := range args.Resources {
-			lris, ok := l.lockMap[resource]
-			if !ok {
-				continue
-			}
-			// Collect uids, so we don't mutate while we delete
-			uids := make([]string, 0, len(lris))
-			for _, lri := range lris {
-				uids = append(uids, lri.UID)
-			}
-
-			// Delete collected uids:
-			for _, uid := range uids {
+	select {
+	case <-ctx.Done():
+		return false, ctx.Err()
+	default:
+		l.mutex.Lock()
+		defer l.mutex.Unlock()
+		if len(args.UID) == 0 {
+			for _, resource := range args.Resources {
 				lris, ok := l.lockMap[resource]
 				if !ok {
-					// Just to be safe, delete uuids.
-					for idx := 0; idx < maxDeleteList; idx++ {
-						mapID := formatUUID(uid, idx)
-						if _, ok := l.lockUID[mapID]; !ok {
-							break
-						}
-						delete(l.lockUID, mapID)
-					}
 					continue
 				}
-				l.removeEntry(resource, dsync.LockArgs{UID: uid}, &lris)
-			}
-		}
-		return true, nil
-	}
+				// Collect uids, so we don't mutate while we delete
+				uids := make([]string, 0, len(lris))
+				for _, lri := range lris {
+					uids = append(uids, lri.UID)
+				}
 
-	idx := 0
-	for {
-		mapID := formatUUID(args.UID, idx)
-		resource, ok := l.lockUID[mapID]
-		if !ok {
-			return idx > 0, nil
+				// Delete collected uids:
+				for _, uid := range uids {
+					lris, ok := l.lockMap[resource]
+					if !ok {
+						// Just to be safe, delete uuids.
+						for idx := 0; idx < maxDeleteList; idx++ {
+							mapID := formatUUID(uid, idx)
+							if _, ok := l.lockUID[mapID]; !ok {
+								break
+							}
+							delete(l.lockUID, mapID)
+						}
+						continue
+					}
+					l.removeEntry(resource, dsync.LockArgs{UID: uid}, &lris)
+				}
+			}
+			return true, nil
 		}
-		lris, ok := l.lockMap[resource]
-		if !ok {
-			// Unexpected  inconsistency, delete.
-			delete(l.lockUID, mapID)
+
+		idx := 0
+		for {
+			mapID := formatUUID(args.UID, idx)
+			resource, ok := l.lockUID[mapID]
+			if !ok {
+				return idx > 0, nil
+			}
+			lris, ok := l.lockMap[resource]
+			if !ok {
+				// Unexpected  inconsistency, delete.
+				delete(l.lockUID, mapID)
+				idx++
+				continue
+			}
+			reply = true
+			l.removeEntry(resource, dsync.LockArgs{UID: args.UID}, &lris)
 			idx++
-			continue
 		}
-		reply = true
-		l.removeEntry(resource, dsync.LockArgs{UID: args.UID}, &lris)
-		idx++
 	}
 }
 
 func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refreshed bool, err error) {
-	if !l.lockMu(ctx) {
+	select {
+	case <-ctx.Done():
 		return false, ctx.Err()
-	}
-	defer l.unlockMu()
+	default:
+		l.mutex.Lock()
+		defer l.mutex.Unlock()
 
-	// Check whether uid is still active.
-	resource, ok := l.lockUID[formatUUID(args.UID, 0)]
-	if !ok {
-		return false, nil
-	}
-	idx := 0
-	for {
-		lris, ok := l.lockMap[resource]
+		// Check whether uid is still active.
+		resource, ok := l.lockUID[formatUUID(args.UID, 0)]
 		if !ok {
-			// Inconsistent. Delete UID.
-			delete(l.lockUID, formatUUID(args.UID, idx))
-			return idx > 0, nil
+			return false, nil
 		}
-		for i := range lris {
-			if lris[i].UID == args.UID {
-				lris[i].TimeLastRefresh = UTCNow()
+		idx := 0
+		for {
+			lris, ok := l.lockMap[resource]
+			if !ok {
+				// Inconsistent. Delete UID.
+				delete(l.lockUID, formatUUID(args.UID, idx))
+				return idx > 0, nil
+			}
+			for i := range lris {
+				if lris[i].UID == args.UID {
+					lris[i].TimeLastRefresh = UTCNow()
+				}
+			}
+			idx++
+			resource, ok = l.lockUID[formatUUID(args.UID, idx)]
+			if !ok {
+				// No more resources for UID, but we did update at least one.
+				return true, nil
 			}
-		}
-		idx++
-		resource, ok = l.lockUID[formatUUID(args.UID, idx)]
-		if !ok {
-			// No more resources for UID, but we did update at least one.
-			return true, nil
 		}
 	}
 }
@@ -362,8 +366,8 @@ func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refresh
 // Similar to removeEntry but only removes an entry only if the lock entry exists in map.
 // Caller must hold 'l.mutex' lock.
 func (l *localLocker) expireOldLocks(interval time.Duration) {
-	l.lockMuBlock()
-	defer l.unlockMu()
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 
 	for k, lris := range l.lockMap {
 		modified := false
@@ -394,7 +398,6 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
 
 func newLocker() *localLocker {
 	return &localLocker{
-		mutex:   make(chan struct{}, 1),
 		lockMap: make(map[string][]lockRequesterInfo, 1000),
 		lockUID: make(map[string]string, 1000),
 	}

cmd/lock-rest-server-common_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"os"
 	"reflect"
+	"sync"
 	"testing"
 
 	"github.com/minio/minio/internal/dsync"
@@ -37,7 +38,10 @@ func createLockTestServer(ctx context.Context, t *testing.T) (string, *lockRESTS
 	}
 
 	locker := &lockRESTServer{
-		ll: newLocker(),
+		ll: &localLocker{
+			mutex:   sync.Mutex{},
+			lockMap: make(map[string][]lockRequesterInfo),
+		},
 	}
 	creds := globalActiveCred
 	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")

cmd/metacache-set.go

@@ -397,6 +397,11 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 	retries := 0
 	rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
 
+	const (
+		retryDelay    = 50 * time.Millisecond
+		retryDelay250 = 250 * time.Millisecond
+	)
+
 	for {
 		if contextCanceled(ctx) {
 			return entries, ctx.Err()
@@ -411,7 +416,6 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			retries = 1
 		}
 
-		const retryDelay = 250 * time.Millisecond
 		// All operations are performed without locks, so we must be careful and allow for failures.
 		// Read metadata associated with the object from a disk.
 		if retries > 0 {
@@ -425,7 +429,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				_, err := disk.ReadVersion(ctx, minioMetaBucket,
 					o.objectPath(0), "", false)
 				if err != nil {
-					time.Sleep(retryDelay)
+					time.Sleep(retryDelay250)
 					retries++
 					continue
 				}
@@ -440,11 +444,19 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			switch toObjectErr(err, minioMetaBucket, o.objectPath(0)).(type) {
 			case ObjectNotFound:
 				retries++
-				time.Sleep(retryDelay)
+				if retries == 1 {
+					time.Sleep(retryDelay)
+				} else {
+					time.Sleep(retryDelay250)
+				}
 				continue
 			case InsufficientReadQuorum:
 				retries++
-				time.Sleep(retryDelay)
+				if retries == 1 {
+					time.Sleep(retryDelay)
+				} else {
+					time.Sleep(retryDelay250)
+				}
 				continue
 			default:
 				return entries, fmt.Errorf("reading first part metadata: %w", err)
@@ -463,7 +475,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				retries = -1
 			}
 			retries++
-			time.Sleep(retryDelay)
+			time.Sleep(retryDelay250)
 			continue
 		case errors.Is(err, io.EOF):
 			return entries, io.EOF
@@ -497,7 +509,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 						_, err := disk.ReadVersion(ctx, minioMetaBucket,
 							o.objectPath(partN), "", false)
 						if err != nil {
-							time.Sleep(retryDelay)
+							time.Sleep(retryDelay250)
 							retries++
 							continue
 						}
@@ -508,7 +520,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				// Load partN metadata...
 				fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{}, true)
 				if err != nil {
-					time.Sleep(retryDelay)
+					time.Sleep(retryDelay250)
 					retries++
 					continue
 				}
@@ -530,9 +542,9 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			}()
 
 			tmp := newMetacacheReader(pr)
-			defer tmp.Close()
 			e, err := tmp.filter(o)
 			pr.CloseWithError(err)
+			tmp.Close()
 			entries.o = append(entries.o, e.o...)
 			if o.Limit > 0 && entries.len() > o.Limit {
 				entries.truncate(o.Limit)
@@ -546,11 +558,11 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				switch toObjectErr(err, minioMetaBucket, o.objectPath(partN)).(type) {
 				case ObjectNotFound:
 					retries++
-					time.Sleep(retryDelay)
+					time.Sleep(retryDelay250)
 					continue
 				case InsufficientReadQuorum:
 					retries++
-					time.Sleep(retryDelay)
+					time.Sleep(retryDelay250)
 					continue
 				default:
 					logger.LogIf(ctx, err)

cmd/metacache-walk.go

@@ -323,13 +323,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				}
 			case isSysErrNotDir(err):
 				// skip
-			default:
-				// It is totally possible that xl.meta was overwritten
-				// while being concurrently listed at the same time in
-				// such scenarios the 'xl.meta' might get truncated
-				if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) {
-					logger.LogIf(ctx, err)
-				}
 			}
 		}
 

cmd/metrics-v2.go

@@ -77,7 +77,7 @@ func init() {
 		return allMetrics
 	}()
 
-	nodeCollector = newMinioCollectorNode([]*MetricsGroup{
+	nodeGroups := []*MetricsGroup{
 		getNodeHealthMetrics(),
 		getLocalDriveStorageMetrics(),
 		getCacheMetrics(),
@@ -86,7 +86,10 @@ func init() {
 		getMinioVersionMetrics(),
 		getS3TTFBMetric(),
 		getNotificationMetrics(),
-	})
+		getDistLockMetrics(),
+	}
+
+	nodeCollector = newMinioCollectorNode(nodeGroups)
 	clusterCollector = newMinioClusterCollector(allMetricsGroups)
 }
 
@@ -152,6 +155,7 @@ const (
 	waitingTotal   MetricName = "waiting_total"
 	incomingTotal  MetricName = "incoming_total"
 	objectTotal    MetricName = "object_total"
+	versionTotal   MetricName = "version_total"
 	offlineTotal   MetricName = "offline_total"
 	onlineTotal    MetricName = "online_total"
 	openTotal      MetricName = "open_total"
@@ -506,6 +510,16 @@ func getBucketUsageObjectsTotalMD() MetricDescription {
 	}
 }
 
+func getBucketUsageVersionsTotalMD() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: usageSubsystem,
+		Name:      versionTotal,
+		Help:      "Total number of versions",
+		Type:      gaugeMetric,
+	}
+}
+
 func getBucketRepLatencyMD() MetricDescription {
 	return MetricDescription{
 		Namespace: bucketMetricNamespace,
@@ -1663,6 +1677,53 @@ func getCacheMetrics() *MetricsGroup {
 	return mg
 }
 
+func getDistLockMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: 1 * time.Second,
+	}
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		if !globalIsDistErasure {
+			return []Metric{}
+		}
+
+		st := globalLockServer.stats()
+
+		metrics := make([]Metric, 0, 3)
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: minioNamespace,
+				Subsystem: "locks",
+				Name:      "total",
+				Help:      "Number of current locks on this peer",
+				Type:      gaugeMetric,
+			},
+			Value: float64(st.Total),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: minioNamespace,
+				Subsystem: "locks",
+				Name:      "write_total",
+				Help:      "Number of current WRITE locks on this peer",
+				Type:      gaugeMetric,
+			},
+			Value: float64(st.Writes),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: minioNamespace,
+				Subsystem: "locks",
+				Name:      "read_total",
+				Help:      "Number of current READ locks on this peer",
+				Type:      gaugeMetric,
+			},
+			Value: float64(st.Reads),
+		})
+		return metrics
+	})
+	return mg
+}
+
 func getNotificationMetrics() *MetricsGroup {
 	mg := &MetricsGroup{
 		cacheInterval: 10 * time.Second,
@@ -1945,6 +2006,12 @@ func getBucketUsageMetrics() *MetricsGroup {
 				VariableLabels: map[string]string{"bucket": bucket},
 			})
 
+			metrics = append(metrics, Metric{
+				Description:    getBucketUsageVersionsTotalMD(),
+				Value:          float64(usage.VersionsCount),
+				VariableLabels: map[string]string{"bucket": bucket},
+			})
+
 			metrics = append(metrics, Metric{
 				Description:    getBucketRepReceivedBytesMD(),
 				Value:          float64(stats.ReplicaSize),

cmd/object-api-errors.go

@@ -674,6 +674,12 @@ func isErrBucketNotFound(err error) bool {
 	return errors.As(err, &bkNotFound)
 }
 
+// isErrReadQuorum check if the error type is InsufficentReadQuorum
+func isErrReadQuorum(err error) bool {
+	var rquorum InsufficientReadQuorum
+	return errors.As(err, &rquorum)
+}
+
 // isErrObjectNotFound - Check if error type is ObjectNotFound.
 func isErrObjectNotFound(err error) bool {
 	var objNotFound ObjectNotFound

cmd/object-api-interface.go

@@ -254,6 +254,7 @@ type ObjectLayer interface {
 
 	// Metadata operations
 	PutObjectMetadata(context.Context, string, string, ObjectOptions) (ObjectInfo, error)
+	DecomTieredObject(context.Context, string, string, FileInfo, ObjectOptions) error
 
 	// ObjectTagging operations
 	PutObjectTags(context.Context, string, string, string, ObjectOptions) (ObjectInfo, error)

cmd/object-api-multipart_test.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -32,6 +33,9 @@ import (
 
 // Wrapper for calling NewMultipartUpload tests for both Erasure multiple disks and single node setup.
 func TestObjectNewMultipartUpload(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip()
+	}
 	ExecObjectLayerTest(t, testObjectNewMultipartUpload)
 }
 
@@ -110,9 +114,18 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
 		{"--", object, uploadID, BucketNotFound{}},
 		{"foo", object, uploadID, BucketNotFound{}},
 		{bucket, object, "foo-foo", InvalidUploadID{}},
-		{bucket, "\\", uploadID, InvalidUploadID{}},
 		{bucket, object, uploadID, nil},
 	}
+
+	if runtime.GOOS != globalWindowsOSName {
+		abortTestCases = append(abortTestCases, struct {
+			bucketName      string
+			objName         string
+			uploadID        string
+			expectedErrType error
+		}{bucket, "\\", uploadID, InvalidUploadID{}})
+	}
+
 	// Iterating over creatPartCases to generate multipart chunks.
 	for i, testCase := range abortTestCases {
 		err = obj.AbortMultipartUpload(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, opts)

cmd/object-api-options.go

@@ -232,7 +232,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 	mtimeStr := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime))
 	mtime := UTCNow()
 	if mtimeStr != "" {
-		mtime, err = time.Parse(time.RFC3339, mtimeStr)
+		mtime, err = time.Parse(time.RFC3339Nano, mtimeStr)
 		if err != nil {
 			return opts, InvalidArgument{
 				Bucket: bucket,
@@ -353,7 +353,7 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object
 	mtimeStr := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime))
 	mtime := UTCNow()
 	if mtimeStr != "" {
-		mtime, err = time.Parse(time.RFC3339, mtimeStr)
+		mtime, err = time.Parse(time.RFC3339Nano, mtimeStr)
 		if err != nil {
 			return opts, InvalidArgument{
 				Bucket: bucket,

cmd/object-api-utils.go

@@ -174,10 +174,7 @@ func IsValidObjectPrefix(object string) bool {
 	// work with file systems, we will reject here
 	// to return object name invalid rather than
 	// a cryptic error from the file system.
-	if strings.ContainsRune(object, 0) {
-		return false
-	}
-	return true
+	return !strings.ContainsRune(object, 0)
 }
 
 // checkObjectNameForLengthAndSlash -check for the validity of object name length and prefis as slash
@@ -199,7 +196,7 @@ func checkObjectNameForLengthAndSlash(bucket, object string) error {
 	if runtime.GOOS == globalWindowsOSName {
 		// Explicitly disallowed characters on windows.
 		// Avoids most problematic names.
-		if strings.ContainsAny(object, `:*?"|<>`) {
+		if strings.ContainsAny(object, `\:*?"|<>`) {
 			return ObjectNameInvalid{
 				Bucket: bucket,
 				Object: object,
@@ -1064,7 +1061,7 @@ func getDiskInfos(ctx context.Context, disks ...StorageAPI) []*DiskInfo {
 }
 
 // hasSpaceFor returns whether the disks in `di` have space for and object of a given size.
-func hasSpaceFor(di []*DiskInfo, size int64) bool {
+func hasSpaceFor(di []*DiskInfo, size int64) (bool, error) {
 	// We multiply the size by 2 to account for erasure coding.
 	size *= 2
 	if size < 0 {
@@ -1085,8 +1082,8 @@ func hasSpaceFor(di []*DiskInfo, size int64) bool {
 		available += disk.Total - disk.Used
 	}
 
-	if nDisks == 0 {
-		return false
+	if nDisks < len(di)/2 {
+		return false, fmt.Errorf("not enough online disks to calculate the available space, expected (%d)/(%d)", (len(di)/2)+1, nDisks)
 	}
 
 	// Check we have enough on each disk, ignoring diskFillFraction.
@@ -1096,13 +1093,13 @@ func hasSpaceFor(di []*DiskInfo, size int64) bool {
 			continue
 		}
 		if int64(disk.Free) <= perDisk {
-			return false
+			return false, nil
 		}
 	}
 
 	// Make sure we can fit "size" on to the disk without getting above the diskFillFraction
 	if available < uint64(size) {
-		return false
+		return false, nil
 	}
 
 	// How much will be left after adding the file.
@@ -1110,5 +1107,5 @@ func hasSpaceFor(di []*DiskInfo, size int64) bool {
 
 	// wantLeft is how much space there at least must be left.
 	wantLeft := uint64(float64(total) * (1.0 - diskFillFraction))
-	return available > wantLeft
+	return available > wantLeft, nil
 }

cmd/object-api-utils_test.go

@@ -19,19 +19,75 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"net/http"
+	"net/http/httptest"
 	"reflect"
+	"runtime"
 	"strconv"
 	"testing"
 
 	"github.com/klauspost/compress/s2"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/compress"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/pkg/trie"
 )
 
+// Wrapper
+func TestPathTraversalExploit(t *testing.T) {
+	if runtime.GOOS != globalWindowsOSName {
+		t.Skip()
+	}
+	defer DetectTestLeak(t)()
+	ExecExtendedObjectLayerAPITest(t, testPathTraversalExploit, []string{"PutObject"})
+}
+
+// testPathTraversal exploit test, exploits path traversal on windows
+// with following object names "\\../.minio.sys/config/iam/${username}/identity.json"
+// #16852
+func testPathTraversalExploit(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials auth.Credentials, t *testing.T,
+) {
+	if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil {
+		t.Fatalf("Initializing config.json failed")
+	}
+
+	objectName := `\../.minio.sys/config/hello.txt`
+
+	// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+	rec := httptest.NewRecorder()
+	// construct HTTP request for Get Object end point.
+	req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, objectName),
+		int64(5), bytes.NewReader([]byte("hello")), credentials.AccessKey, credentials.SecretKey, map[string]string{})
+	if err != nil {
+		t.Fatalf("failed to create HTTP request for Put Object: <ERROR> %v", err)
+	}
+
+	// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+	// Call the ServeHTTP to execute the handler.
+	apiRouter.ServeHTTP(rec, req)
+
+	ctx, cancel := context.WithCancel(GlobalContext)
+	defer cancel()
+
+	// Now check if we actually wrote to backend (regardless of the response
+	// returned by the server).
+	z := obj.(*erasureServerPools)
+	xl := z.serverPools[0].sets[0]
+	erasureDisks := xl.getDisks()
+	parts, errs := readAllFileInfo(ctx, erasureDisks, bucketName, objectName, "", false)
+	for i := range parts {
+		if errs[i] == nil {
+			if parts[i].Name == objectName {
+				t.Errorf("path traversal allowed to allow writing to minioMetaBucket: %s", instanceType)
+			}
+		}
+	}
+}
+
 // Tests validate bucket name.
 func TestIsValidBucketName(t *testing.T) {
 	testCases := []struct {

cmd/object-handlers-common.go

@@ -367,6 +367,18 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
 				continue
 			}
 			dobj := deletedObjs[i]
+			oi := ObjectInfo{
+				Bucket:    bucket,
+				Name:      dobj.ObjectName,
+				VersionID: dobj.VersionID,
+			}
+			traceFn := globalLifecycleSys.trace(oi)
+			// Send audit for the lifecycle delete operation
+			auditLogLifecycle(
+				ctx,
+				oi,
+				ILMExpiry, traceFn)
+
 			sendEvent(eventArgs{
 				EventName:  event.ObjectRemovedDelete,
 				BucketName: bucket,
@@ -374,7 +386,8 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
 					Name:      dobj.ObjectName,
 					VersionID: dobj.VersionID,
 				},
-				Host: "Internal: [ILM-Expiry]",
+				UserAgent: "Internal: [ILM-Expiry]",
+				Host:      globalLocalNodeName,
 			})
 		}
 	}

cmd/object-multipart-handlers.go

@@ -30,6 +30,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/amztime"
@@ -416,7 +417,11 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			return
 		}
 
-		partInfo, err := core.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, gr, length, "", "", dstOpts.ServerSideEncryption)
+		popts := minio.PutObjectPartOptions{
+			SSE: dstOpts.ServerSideEncryption,
+		}
+
+		partInfo, err := core.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, gr, length, popts)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return

cmd/os_windows.go

@@ -21,8 +21,8 @@
 package cmd
 
 import (
-	"io"
 	"os"
+	"path/filepath"
 	"syscall"
 )
 
@@ -39,38 +39,34 @@ func osMkdirAll(dirPath string, perm os.FileMode) error {
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.
 func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
-	f, err := Open(dirPath)
+	// Ensure we don't pick up files as directories.
+	globAll := filepath.Clean(dirPath) + `\*`
+	globAllP, err := syscall.UTF16PtrFromString(globAll)
 	if err != nil {
-		if osErrToFileErr(err) == errFileNotFound {
+		return errInvalidArgument
+	}
+	data := &syscall.Win32finddata{}
+	handle, err := syscall.FindFirstFile(globAllP, data)
+	if err != nil {
+		if err = syscallErrToFileErr(dirPath, err); err == errFileNotFound {
 			return nil
 		}
-		return osErrToFileErr(err)
+		return err
 	}
-	defer f.Close()
+	defer syscall.CloseHandle(handle)
 
-	// Check if file or dir. This is the quickest way.
-	// Do not remove this check, on windows syscall.FindNextFile
-	// would throw an exception if Fd() points to a file
-	// instead of a directory, we need to quickly fail
-	// in such situations - this workadound is expected.
-	if _, err = f.Seek(0, io.SeekStart); err == nil {
-		return errFileNotFound
-	}
-
-	data := &syscall.Win32finddata{}
-	for {
-		e := syscall.FindNextFile(syscall.Handle(f.Fd()), data)
-		if e != nil {
-			if e == syscall.ERROR_NO_MORE_FILES {
+	for ; ; err = syscall.FindNextFile(handle, data) {
+		if err != nil {
+			if err == syscall.ERROR_NO_MORE_FILES {
 				break
 			} else {
-				if isSysErrPathNotFound(e) {
+				if isSysErrPathNotFound(err) {
 					return nil
 				}
 				err = osErrToFileErr(&os.PathError{
 					Op:   "FindNextFile",
 					Path: dirPath,
-					Err:  e,
+					Err:  err,
 				})
 				if err == errFileNotFound {
 					return nil
@@ -109,7 +105,7 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
 			typ = os.ModeDir
 		}
 
-		if e = filter(name, typ); e == errDoneForNow {
+		if err = filter(name, typ); err == errDoneForNow {
 			// filtering requested to return by caller.
 			return nil
 		}
@@ -120,35 +116,30 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
 
 // Return N entries at the directory dirPath.
 func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
-	f, err := Open(dirPath)
+	// Ensure we don't pick up files as directories.
+	globAll := filepath.Clean(dirPath) + `\*`
+	globAllP, err := syscall.UTF16PtrFromString(globAll)
 	if err != nil {
-		return nil, osErrToFileErr(err)
+		return nil, errInvalidArgument
 	}
-	defer f.Close()
-
-	// Check if file or dir. This is the quickest way.
-	// Do not remove this check, on windows syscall.FindNextFile
-	// would throw an exception if Fd() points to a file
-	// instead of a directory, we need to quickly fail
-	// in such situations - this workadound is expected.
-	if _, err = f.Seek(0, io.SeekStart); err == nil {
-		return nil, errFileNotFound
-	}
-
 	data := &syscall.Win32finddata{}
-	handle := syscall.Handle(f.Fd())
+	handle, err := syscall.FindFirstFile(globAllP, data)
+	if err != nil {
+		return nil, syscallErrToFileErr(dirPath, err)
+	}
+
+	defer syscall.CloseHandle(handle)
 
 	count := opts.count
-	for count != 0 {
-		e := syscall.FindNextFile(handle, data)
-		if e != nil {
-			if e == syscall.ERROR_NO_MORE_FILES {
+	for ; count != 0; err = syscall.FindNextFile(handle, data) {
+		if err != nil {
+			if err == syscall.ERROR_NO_MORE_FILES {
 				break
 			} else {
 				return nil, osErrToFileErr(&os.PathError{
 					Op:   "FindNextFile",
 					Path: dirPath,
-					Err:  e,
+					Err:  err,
 				})
 			}
 		}
@@ -192,3 +183,21 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
 func globalSync() {
 	// no-op on windows
 }
+
+func syscallErrToFileErr(dirPath string, err error) error {
+	switch err {
+	case nil:
+		return nil
+	case syscall.ERROR_FILE_NOT_FOUND:
+		return errFileNotFound
+	case syscall.ERROR_ACCESS_DENIED:
+		return errFileAccessDenied
+	default:
+		// Fails on file not found and when not a directory.
+		return osErrToFileErr(&os.PathError{
+			Op:   "FindNextFile",
+			Path: dirPath,
+			Err:  err,
+		})
+	}
+}

cmd/peer-rest-server.go

@@ -977,6 +977,12 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		s.writeErrorResponse(w, err)
 		return
 	}
+
+	// Publish bootstrap events that have already occurred before client could subscribe.
+	if traceOpts.TraceTypes().Contains(madmin.TraceBootstrap) {
+		go globalBootstrapTracer.Publish(r.Context(), globalTrace)
+	}
+
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 
@@ -1187,6 +1193,7 @@ func (s *peerRESTServer) GetBandwidth(w http.ResponseWriter, r *http.Request) {
 func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
 	}
 
 	enc := gob.NewEncoder(w)

cmd/peer-s3-client.go

@@ -137,16 +137,7 @@ func (sys *S3PeerSys) ListBuckets(ctx context.Context, opts BucketOptions) (resu
 func (sys *S3PeerSys) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (binfo BucketInfo, err error) {
 	g := errgroup.WithNErrs(len(sys.peerClients))
 
-	bucketInfos := make([]BucketInfo, len(sys.peerClients)+1)
-
-	bucketInfo, err := getBucketInfoLocal(ctx, bucket, opts)
-	if err != nil {
-		return BucketInfo{}, err
-	}
-
-	errs := []error{nil}
-	bucketInfos[0] = bucketInfo
-
+	bucketInfos := make([]BucketInfo, len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		idx := idx
 		client := client
@@ -158,26 +149,29 @@ func (sys *S3PeerSys) GetBucketInfo(ctx context.Context, bucket string, opts Buc
 			if err != nil {
 				return err
 			}
-			bucketInfos[idx+1] = bucketInfo
+			bucketInfos[idx] = bucketInfo
 			return nil
 		}, idx)
 	}
 
-	errs = append(errs, g.Wait()...)
+	errs := g.Wait()
+
+	bucketInfo, err := getBucketInfoLocal(ctx, bucket, opts)
+	errs = append(errs, err)
+	bucketInfos = append(bucketInfos, bucketInfo)
 
 	quorum := (len(sys.allPeerClients) / 2)
 	if err = reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum); err != nil {
-		return BucketInfo{}, err
+		return BucketInfo{}, toObjectErr(err, bucket)
 	}
 
 	for i, err := range errs {
 		if err == nil {
-			bucketInfo = bucketInfos[i]
-			break
+			return bucketInfos[i], nil
 		}
 	}
 
-	return bucketInfo, nil
+	return BucketInfo{}, toObjectErr(errVolumeNotFound, bucket)
 }
 
 func (client *peerS3Client) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) {
@@ -266,12 +260,9 @@ func (sys *S3PeerSys) DeleteBucket(ctx context.Context, bucket string, opts Dele
 	errs := g.Wait()
 	errs = append(errs, deleteBucketLocal(ctx, bucket, opts))
 
-	for _, err := range errs {
-		if err != nil {
-			return err
-		}
-	}
-	return nil
+	quorum := (len(sys.allPeerClients) / 2) + 1
+	err := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum)
+	return toObjectErr(err, bucket)
 }
 
 // DeleteBucket deletes bucket on a peer

cmd/post-policy_test.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"strings"
 	"testing"
 	"time"
 
@@ -114,6 +115,65 @@ func newPostPolicyBytesV2(bucketName, objectKey string, expiration time.Time) []
 	return []byte(retStr)
 }
 
+// Wrapper
+func TestPostPolicyReservedBucketExploit(t *testing.T) {
+	ExecObjectLayerTestWithDirs(t, testPostPolicyReservedBucketExploit)
+}
+
+// testPostPolicyReservedBucketExploit is a test for the exploit fixed in PR
+// #16849
+func testPostPolicyReservedBucketExploit(obj ObjectLayer, instanceType string, dirs []string, t TestErrHandler) {
+	if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil {
+		t.Fatalf("Initializing config.json failed")
+	}
+
+	// Register the API end points with Erasure/FS object layer.
+	apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"})
+
+	credentials := globalActiveCred
+	bucketName := minioMetaBucket
+	objectName := "config/x"
+
+	// This exploit needs browser to be enabled.
+	if !globalBrowserEnabled {
+		globalBrowserEnabled = true
+		defer func() { globalBrowserEnabled = false }()
+	}
+
+	// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
+	rec := httptest.NewRecorder()
+	req, perr := newPostRequestV4("", bucketName, objectName, []byte("pwned"), credentials.AccessKey, credentials.SecretKey)
+	if perr != nil {
+		t.Fatalf("Test %s: Failed to create HTTP request for PostPolicyHandler: <ERROR> %v", instanceType, perr)
+	}
+
+	contentTypeHdr := req.Header.Get("Content-Type")
+	contentTypeHdr = strings.Replace(contentTypeHdr, "multipart/form-data", "multipart/form-datA", 1)
+	req.Header.Set("Content-Type", contentTypeHdr)
+	req.Header.Set("User-Agent", "Mozilla")
+
+	// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic ofthe handler.
+	// Call the ServeHTTP to execute the handler.
+	apiRouter.ServeHTTP(rec, req)
+
+	ctx, cancel := context.WithCancel(GlobalContext)
+	defer cancel()
+
+	// Now check if we actually wrote to backend (regardless of the response
+	// returned by the server).
+	z := obj.(*erasureServerPools)
+	xl := z.serverPools[0].sets[0]
+	erasureDisks := xl.getDisks()
+	parts, errs := readAllFileInfo(ctx, erasureDisks, bucketName, objectName+"/upload.txt", "", false)
+	for i := range parts {
+		if errs[i] == nil {
+			if parts[i].Name == objectName+"/upload.txt" {
+				t.Errorf("Test %s: Failed to stop post policy handler from writing to minioMetaBucket", instanceType)
+			}
+		}
+	}
+}
+
 // Wrapper for calling TestPostPolicyBucketHandler tests for both Erasure multiple disks and single node setup.
 func TestPostPolicyBucketHandler(t *testing.T) {
 	ExecObjectLayerTest(t, testPostPolicyBucketHandler)

cmd/routers.go

@@ -58,8 +58,6 @@ var globalHandlers = []mux.MiddlewareFunc{
 	setCrossDomainPolicy,
 	// Limits all body and header sizes to a maximum fixed limit
 	setRequestLimitHandler,
-	// Network statistics
-	setHTTPStatsHandler,
 	// Validate all the incoming requests.
 	setRequestValidityHandler,
 	// set x-amz-request-id header.

cmd/server-main.go

@@ -352,6 +352,8 @@ func configRetriableErrors(err error) bool {
 }
 
 func bootstrapTrace(msg string) {
+	globalBootstrapTracer.Record(msg, 2)
+
 	if globalTrace.NumSubscribers(madmin.TraceBootstrap) == 0 {
 		return
 	}

cmd/site-replication-utils.go

@@ -170,6 +170,7 @@ func (sm *siteResyncMetrics) save(ctx context.Context) {
 		case <-sTimer.C:
 			if globalSiteReplicationSys.isEnabled() {
 				sm.Lock()
+				wg := sync.WaitGroup{}
 				for dID, rs := range sm.peerResyncMap {
 					st, ok := sm.resyncStatus[rs.resyncID]
 					if ok {
@@ -179,9 +180,14 @@ func (sm *siteResyncMetrics) save(ctx context.Context) {
 						}
 						rs.LastSaved = UTCNow()
 						sm.peerResyncMap[dID] = rs
-						go saveSiteResyncMetadata(ctx, st, newObjectLayerFn())
+						wg.Add(1)
+						go func() {
+							defer wg.Done()
+							saveSiteResyncMetadata(ctx, st, newObjectLayerFn())
+						}()
 					}
 				}
+				wg.Wait()
 				sm.Unlock()
 			}
 			sTimer.Reset(siteResyncSaveInterval)

cmd/site-replication.go

@@ -896,6 +896,9 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 				AccessKey: creds.AccessKey,
 				SecretKey: creds.SecretKey,
 			}
+			if !peer.SyncState.Empty() {
+				targetToUpdate.ReplicationSync = (peer.SyncState == madmin.SyncEnabled)
+			}
 			err := globalBucketTargetSys.SetTarget(ctx, bucket, &targetToUpdate, true)
 			if err != nil {
 				return c.annotatePeerErr(peer.Name, "Bucket target update error", err)
@@ -927,7 +930,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 				API:             "s3v4",
 				Type:            madmin.ReplicationService,
 				Region:          "",
-				ReplicationSync: false,
+				ReplicationSync: peer.SyncState == madmin.SyncEnabled,
 			}
 			var exists bool // true if ARN already exists
 			bucketTarget.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &bucketTarget, peer.DeploymentID)
@@ -3443,12 +3446,17 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe
 	)
 
 	for _, v := range sites.Sites {
-		if peer.Endpoint == v.Endpoint {
-			return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("Endpoint %s entered for deployment id %s already configured in site replication", v.Endpoint, v.DeploymentID))
-		}
 		if peer.DeploymentID == v.DeploymentID {
 			found = true
-			if peer.Endpoint == v.Endpoint {
+			if !peer.SyncState.Empty() {
+				if globalDeploymentID == peer.DeploymentID {
+					return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("invalid sync setting for endpoint %s (deployment id %s)", v.Endpoint, v.DeploymentID))
+				}
+				if peer.Endpoint == "" { // peer.Endpoint may be "" if only sync state is being updated
+					break
+				}
+			}
+			if peer.Endpoint == v.Endpoint && peer.SyncState.Empty() {
 				return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("Endpoint %s entered for deployment id %s already configured in site replication", v.Endpoint, v.DeploymentID))
 			}
 			admClient, err = c.getAdminClientWithEndpoint(ctx, v.DeploymentID, peer.Endpoint)
@@ -3469,44 +3477,52 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe
 	if !found {
 		return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("%s not found in existing replicated sites", peer.DeploymentID))
 	}
-
-	errs := make(map[string]error, len(c.state.Peers))
-	var wg sync.WaitGroup
-
 	pi := c.state.Peers[peer.DeploymentID]
 	prevPeerInfo := pi
-	pi.Endpoint = peer.Endpoint
+	if !peer.SyncState.Empty() { // update replication to peer to be sync/async
+		pi.SyncState = peer.SyncState
+		c.state.Peers[peer.DeploymentID] = pi
+	}
+	if peer.Endpoint != "" { // `admin replicate update` requested an endpoint change
+		pi.Endpoint = peer.Endpoint
+	}
 
-	for i, v := range sites.Sites {
-		if v.DeploymentID == globalDeploymentID {
-			c.state.Peers[peer.DeploymentID] = pi
-			continue
-		}
-		wg.Add(1)
-		go func(pi madmin.PeerInfo, i int) {
-			defer wg.Done()
-			v := sites.Sites[i]
-			admClient, err := c.getAdminClient(ctx, v.DeploymentID)
-			if v.DeploymentID == peer.DeploymentID {
-				admClient, err = c.getAdminClientWithEndpoint(ctx, v.DeploymentID, peer.Endpoint)
+	if admClient != nil {
+		errs := make(map[string]error, len(c.state.Peers))
+		var wg sync.WaitGroup
+
+		for i, v := range sites.Sites {
+			if v.DeploymentID == globalDeploymentID {
+				c.state.Peers[peer.DeploymentID] = pi
+				continue
 			}
+			wg.Add(1)
+			go func(pi madmin.PeerInfo, i int) {
+				defer wg.Done()
+				v := sites.Sites[i]
+				admClient, err := c.getAdminClient(ctx, v.DeploymentID)
+				if v.DeploymentID == peer.DeploymentID {
+					admClient, err = c.getAdminClientWithEndpoint(ctx, v.DeploymentID, peer.Endpoint)
+				}
+				if err != nil {
+					errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
+					return
+				}
+				if err = admClient.SRPeerEdit(ctx, pi); err != nil {
+					errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", v.Name, err))
+					return
+				}
+			}(pi, i)
+		}
+
+		wg.Wait()
+		for dID, err := range errs {
 			if err != nil {
-				errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
-				return
+				return madmin.ReplicateEditStatus{}, errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", c.state.Peers[dID].Name, err))
 			}
-			if err = admClient.SRPeerEdit(ctx, pi); err != nil {
-				errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", v.Name, err))
-				return
-			}
-		}(pi, i)
-	}
-
-	wg.Wait()
-	for dID, err := range errs {
-		if err != nil {
-			return madmin.ReplicateEditStatus{}, errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", c.state.Peers[dID].Name, err))
 		}
 	}
+
 	// we can now save the cluster replication configuration state.
 	if err = c.saveToDisk(ctx, c.state); err != nil {
 		return madmin.ReplicateEditStatus{
@@ -3514,7 +3530,7 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe
 			ErrDetail: fmt.Sprintf("unable to save cluster-replication state on local: %v", err),
 		}, nil
 	}
-	if err = c.updateTargetEndpoints(ctx, prevPeerInfo, peer); err != nil {
+	if err = c.updateTargetEndpoints(ctx, prevPeerInfo, pi); err != nil {
 		return madmin.ReplicateEditStatus{
 			Status:    madmin.ReplicateAddStatusPartial,
 			ErrDetail: fmt.Sprintf("unable to update peer targets on local: %v", err),
@@ -3556,6 +3572,9 @@ func (c *SiteReplicationSys) updateTargetEndpoints(ctx context.Context, prevInfo
 				bucketTarget := target
 				bucketTarget.Secure = ep.Scheme == "https"
 				bucketTarget.Endpoint = ep.Host
+				if !peer.SyncState.Empty() {
+					bucketTarget.ReplicationSync = (peer.SyncState == madmin.SyncEnabled)
+				}
 				err := globalBucketTargetSys.SetTarget(ctx, bucket, &bucketTarget, true)
 				if err != nil {
 					logger.LogIf(ctx, c.annotatePeerErr(peer.Name, "Bucket target creation error", err))

cmd/storage-rest-client.go

@@ -731,6 +731,7 @@ func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path
 		return stat, err
 	}
 	rd := msgpNewReader(respReader)
+	defer readMsgpReaderPool.Put(rd)
 	for {
 		var st StatInfo
 		err = st.DecodeMsg(rd)

cmd/sts-errors.go

@@ -27,19 +27,9 @@ import (
 )
 
 // writeSTSErrorRespone writes error headers
-func writeSTSErrorResponse(ctx context.Context, w http.ResponseWriter, isErrCodeSTS bool, errCode STSErrorCode, errCtxt error) {
-	var err STSError
-	if isErrCodeSTS {
-		err = stsErrCodes.ToSTSErr(errCode)
-	}
-	if err.Code == "InternalError" || !isErrCodeSTS {
-		aerr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, errCtxt))
-		if aerr.Code != "InternalError" {
-			err.Code = aerr.Code
-			err.Description = aerr.Description
-			err.HTTPStatusCode = aerr.HTTPStatusCode
-		}
-	}
+func writeSTSErrorResponse(ctx context.Context, w http.ResponseWriter, errCode STSErrorCode, errCtxt error) {
+	err := stsErrCodes.ToSTSErr(errCode)
+
 	// Generate error response.
 	stsErrorResponse := STSErrorResponse{}
 	stsErrorResponse.Error.Code = err.Code

cmd/sts-handlers.go

@@ -137,32 +137,45 @@ func registerSTSRouter(router *mux.Router) {
 		Queries(stsVersion, stsAPIVersion)
 }
 
-func checkAssumeRoleAuth(ctx context.Context, r *http.Request) (user auth.Credentials, isErrCodeSTS bool, stsErr STSErrorCode) {
+func apiToSTSError(authErr APIErrorCode) (stsErrCode STSErrorCode) {
+	switch authErr {
+	case ErrSignatureDoesNotMatch, ErrInvalidAccessKeyID, ErrAccessKeyDisabled:
+		return ErrSTSAccessDenied
+	case ErrServerNotInitialized:
+		return ErrSTSNotInitialized
+	case ErrInternalError:
+		return ErrSTSInternalError
+	default:
+		return ErrSTSAccessDenied
+	}
+}
+
+func checkAssumeRoleAuth(ctx context.Context, r *http.Request) (auth.Credentials, APIErrorCode) {
 	if !isRequestSignatureV4(r) {
-		return user, true, ErrSTSAccessDenied
+		return auth.Credentials{}, ErrAccessDenied
 	}
 
 	s3Err := isReqAuthenticated(ctx, r, globalSite.Region, serviceSTS)
 	if s3Err != ErrNone {
-		return user, false, STSErrorCode(s3Err)
+		return auth.Credentials{}, s3Err
 	}
 
-	user, _, s3Err = getReqAccessKeyV4(r, globalSite.Region, serviceSTS)
+	user, _, s3Err := getReqAccessKeyV4(r, globalSite.Region, serviceSTS)
 	if s3Err != ErrNone {
-		return user, false, STSErrorCode(s3Err)
+		return auth.Credentials{}, s3Err
 	}
 
 	// Temporary credentials or Service accounts cannot generate further temporary credentials.
 	if user.IsTemp() || user.IsServiceAccount() {
-		return user, true, ErrSTSAccessDenied
+		return auth.Credentials{}, ErrAccessDenied
 	}
 
 	// Session tokens are not allowed in STS AssumeRole requests.
 	if getSessionToken(r) != "" {
-		return user, true, ErrSTSAccessDenied
+		return auth.Credentials{}, ErrAccessDenied
 	}
 
-	return user, true, ErrSTSNone
+	return user, ErrNone
 }
 
 func parseForm(r *http.Request) error {
@@ -190,15 +203,15 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	// the call to `parseForm` below), but return failure only after we are
 	// able to validate that it is a valid STS request, so that we are able
 	// to send an appropriate audit log.
-	user, isErrCodeSTS, stsErr := checkAssumeRoleAuth(ctx, r)
+	user, apiErrCode := checkAssumeRoleAuth(ctx, r)
 
 	if err := parseForm(r); err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
 	if r.Form.Get(stsVersion) != stsAPIVersion {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get(stsVersion), stsAPIVersion))
+		writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get(stsVersion), stsAPIVersion))
 		return
 	}
 
@@ -206,16 +219,17 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	switch action {
 	case assumeRole:
 	default:
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
 		return
 	}
 
 	ctx = newContext(r, w, action)
 
-	// Validate the authentication result here so that failures will be
-	// audit-logged.
-	if stsErr != ErrSTSNone {
-		writeSTSErrorResponse(ctx, w, isErrCodeSTS, stsErr, nil)
+	// Validate the authentication result here so that failures will be audit-logged.
+	if apiErrCode != ErrNone {
+		stsErr := apiToSTSError(apiErrCode)
+		// Borrow the description error from the API error code
+		writeSTSErrorResponse(ctx, w, stsErr, fmt.Errorf(errorCodes[apiErrCode].Description))
 		return
 	}
 
@@ -224,27 +238,27 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	// The plain text that you use for both inline and managed session
 	// policies shouldn't exceed 2048 characters.
 	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy shouldn't exceed 2048 characters"))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy shouldn't exceed 2048 characters"))
 		return
 	}
 
 	if len(sessionPolicyStr) > 0 {
 		sessionPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
 		if err != nil {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 			return
 		}
 
 		// Version in policy must not be empty
 		if sessionPolicy.Version == "" {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Version cannot be empty expecting '2012-10-17'"))
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Version cannot be empty expecting '2012-10-17'"))
 			return
 		}
 	}
 
 	duration, err := openid.GetDefaultExpiration(r.Form.Get(stsDurationSeconds))
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
@@ -254,7 +268,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	// Validate that user.AccessKey's policies can be retrieved - it may not
 	// be in case the user is disabled.
 	if _, err = globalIAMSys.PolicyDBGet(user.AccessKey, false); err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
@@ -265,7 +279,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	secret := globalActiveCred.SecretKey
 	cred, err := auth.GetNewCredentialsWithMetadata(claims, secret)
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 
@@ -276,7 +290,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	// Set the newly generated credentials.
 	updatedAt, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, "")
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 
@@ -312,12 +326,12 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 
 	// Parse the incoming form data.
 	if err := parseForm(r); err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
 	if r.Form.Get(stsVersion) != stsAPIVersion {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
+		writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
 		return
 	}
 
@@ -328,7 +342,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 		return
 	case clientGrants, webIdentity:
 	default:
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
 		return
 	}
 
@@ -354,7 +368,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 		var err error
 		roleArn, _, err = globalIAMSys.GetRolePolicy(roleArnStr)
 		if err != nil {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue,
 				fmt.Errorf("Error processing %s parameter: %v", stsRoleArn, err))
 			return
 		}
@@ -366,16 +380,16 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 		case openid.ErrTokenExpired:
 			switch action {
 			case clientGrants:
-				writeSTSErrorResponse(ctx, w, true, ErrSTSClientGrantsExpiredToken, err)
+				writeSTSErrorResponse(ctx, w, ErrSTSClientGrantsExpiredToken, err)
 			case webIdentity:
-				writeSTSErrorResponse(ctx, w, true, ErrSTSWebIdentityExpiredToken, err)
+				writeSTSErrorResponse(ctx, w, ErrSTSWebIdentityExpiredToken, err)
 			}
 			return
 		case auth.ErrInvalidDuration:
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 			return
 		}
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
@@ -397,11 +411,11 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 
 		if newGlobalAuthZPluginFn() == nil {
 			if !ok {
-				writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+				writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue,
 					fmt.Errorf("%s claim missing from the JWT token, credentials will not be generated", iamPolicyClaimNameOpenID()))
 				return
 			} else if policyName == "" {
-				writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+				writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue,
 					fmt.Errorf("None of the given policies (`%s`) are defined, credentials will not be generated", policies))
 				return
 			}
@@ -414,20 +428,20 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	// The plain text that you use for both inline and managed session
 	// policies shouldn't exceed 2048 characters.
 	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy should not exceed 2048 characters"))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy should not exceed 2048 characters"))
 		return
 	}
 
 	if len(sessionPolicyStr) > 0 {
 		sessionPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
 		if err != nil {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 			return
 		}
 
 		// Version in policy must not be empty
 		if sessionPolicy.Version == "" {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid session policy version"))
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid session policy version"))
 			return
 		}
 
@@ -437,7 +451,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	secret := globalActiveCred.SecretKey
 	cred, err := auth.GetNewCredentialsWithMetadata(claims, secret)
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 
@@ -452,7 +466,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	}
 
 	if subFromToken == "" {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue,
 			errors.New("STS JWT Token has `sub` claim missing, `sub` claim is mandatory"))
 		return
 	}
@@ -477,7 +491,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	// Set the newly generated credentials.
 	updatedAt, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, policyName)
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 
@@ -549,12 +563,12 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 
 	// Parse the incoming form data.
 	if err := parseForm(r); err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
 	if r.Form.Get(stsVersion) != stsAPIVersion {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter,
+		writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter,
 			fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
 		return
 	}
@@ -563,7 +577,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 	ldapPassword := r.Form.Get(stsLDAPPassword)
 
 	if ldapUsername == "" || ldapPassword == "" {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, fmt.Errorf("LDAPUsername and LDAPPassword cannot be empty"))
+		writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, fmt.Errorf("LDAPUsername and LDAPPassword cannot be empty"))
 		return
 	}
 
@@ -571,7 +585,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 	switch action {
 	case ldapIdentity:
 	default:
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
 		return
 	}
 
@@ -580,20 +594,20 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 	// The plain text that you use for both inline and managed session
 	// policies shouldn't exceed 2048 characters.
 	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy should not exceed 2048 characters"))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy should not exceed 2048 characters"))
 		return
 	}
 
 	if len(sessionPolicyStr) > 0 {
 		sessionPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
 		if err != nil {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 			return
 		}
 
 		// Version in policy must not be empty
 		if sessionPolicy.Version == "" {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Version needs to be specified in session policy"))
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Version needs to be specified in session policy"))
 			return
 		}
 	}
@@ -601,14 +615,14 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 	ldapUserDN, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(ldapUsername, ldapPassword)
 	if err != nil {
 		err = fmt.Errorf("LDAP server error: %w", err)
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
 	// Check if this user or their groups have a policy applied.
 	ldapPolicies, _ := globalIAMSys.PolicyDBGet(ldapUserDN, false, groupDistNames...)
 	if len(ldapPolicies) == 0 && newGlobalAuthZPluginFn() == nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue,
 			fmt.Errorf("expecting a policy to be set for user `%s` or one of their groups: `%s` - rejecting this request",
 				ldapUserDN, strings.Join(groupDistNames, "`,`")))
 		return
@@ -616,7 +630,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 
 	expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
@@ -631,7 +645,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 	secret := globalActiveCred.SecretKey
 	cred, err := auth.GetNewCredentialsWithMetadata(claims, secret)
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 
@@ -648,7 +662,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 	// mapping.
 	updatedAt, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, "")
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 
@@ -687,7 +701,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	defer logger.AuditLog(ctx, w, r, claims)
 
 	if !globalIAMSys.STSTLSConfig.Enabled {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCertificate' is disabled"))
+		writeSTSErrorResponse(ctx, w, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCertificate' is disabled"))
 		return
 	}
 
@@ -696,7 +710,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	// Otherwise, we don't have a certificate to verify or
 	// the policy lookup would ambigious.
 	if r.TLS == nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInsecureConnection, errors.New("No TLS connection attempt"))
+		writeSTSErrorResponse(ctx, w, ErrSTSInsecureConnection, errors.New("No TLS connection attempt"))
 		return
 	}
 
@@ -718,11 +732,11 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	// Now, we have to check that the client has provided exactly one leaf
 	// certificate that we can map to a policy.
 	if len(r.TLS.PeerCertificates) == 0 {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, errors.New("No client certificate provided"))
+		writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, errors.New("No client certificate provided"))
 		return
 	}
 	if len(r.TLS.PeerCertificates) > 1 {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, errors.New("More than one client certificate provided"))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, errors.New("More than one client certificate provided"))
 		return
 	}
 
@@ -735,7 +749,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 			Roots: globalRootCAs,
 		})
 		if err != nil {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidClientCertificate, err)
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidClientCertificate, err)
 			return
 		}
 	} else {
@@ -760,7 +774,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 			}
 		}
 		if !validKeyUsage {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, errors.New("certificate is not valid for client authentication"))
+			writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, errors.New("certificate is not valid for client authentication"))
 			return
 		}
 	}
@@ -772,13 +786,13 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	//
 	// Group mapping is not possible with standard X.509 certificates.
 	if certificate.Subject.CommonName == "" {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, errors.New("certificate subject CN cannot be empty"))
+		writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, errors.New("certificate subject CN cannot be empty"))
 		return
 	}
 
 	expiry, err := globalIAMSys.STSTLSConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds))
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSMissingParameter, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, err)
 		return
 	}
 
@@ -801,7 +815,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 
 	tmpCredentials, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 
@@ -809,7 +823,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *h
 	policyName := certificate.Subject.CommonName
 	updatedAt, err := globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, policyName)
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 
@@ -845,19 +859,19 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 
 	authn := newGlobalAuthNPluginFn()
 	if authn == nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCustomToken' is disabled"))
+		writeSTSErrorResponse(ctx, w, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCustomToken' is disabled"))
 		return
 	}
 
 	action := r.Form.Get(stsAction)
 	if action != customTokenIdentity {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action))
 		return
 	}
 
 	token := r.Form.Get(stsToken)
 	if token == "" {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid empty `Token` parameter provided"))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid empty `Token` parameter provided"))
 		return
 	}
 
@@ -867,7 +881,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 		var err error
 		requestedDuration, err = strconv.Atoi(durationParam)
 		if err != nil {
-			writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid requested duration: %s", durationParam))
+			writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid requested duration: %s", durationParam))
 			return
 		}
 	}
@@ -875,26 +889,26 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 	roleArnStr := r.Form.Get(stsRoleArn)
 	roleArn, _, err := globalIAMSys.GetRolePolicy(roleArnStr)
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue,
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue,
 			fmt.Errorf("Error processing parameter %s: %v", stsRoleArn, err))
 		return
 	}
 
 	res, err := authn.Authenticate(roleArn, token)
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err)
 		return
 	}
 
 	// If authentication failed, return the error message to the user.
 	if res.Failure != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSUpstreamError, errors.New(res.Failure.Reason))
+		writeSTSErrorResponse(ctx, w, ErrSTSUpstreamError, errors.New(res.Failure.Reason))
 		return
 	}
 
 	// It is required that parent user be set.
 	if res.Success.User == "" {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSUpstreamError, errors.New("A valid user was not returned by the authenticator."))
+		writeSTSErrorResponse(ctx, w, ErrSTSUpstreamError, errors.New("A valid user was not returned by the authenticator."))
 		return
 	}
 
@@ -923,14 +937,14 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 
 	tmpCredentials, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 
 	tmpCredentials.ParentUser = parentUser
 	updatedAt, err := globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, "")
 	if err != nil {
-		writeSTSErrorResponse(ctx, w, true, ErrSTSInternalError, err)
+		writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err)
 		return
 	}
 

cmd/test-utils_test.go

@@ -1885,21 +1885,29 @@ func ExecObjectLayerTestWithDirs(t TestErrHandler, objTest objTestTypeWithDirs)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	if localMetacacheMgr != nil {
+		localMetacacheMgr.deleteAll()
+	}
+
+	initAllSubsystems(ctx)
 	objLayer, fsDirs, err := prepareErasure16(ctx)
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
 	}
-	defer objLayer.Shutdown(ctx)
-
-	// initialize the server and obtain the credentials and root.
-	// credentials are necessary to sign the HTTP request.
-	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
-		t.Fatal("Unexpected error", err)
-	}
+	setObjectLayer(objLayer)
+	initConfigSubsystem(ctx, objLayer)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
 
 	// Executing the object layer tests for Erasure.
 	objTest(objLayer, ErasureTestStr, fsDirs, t)
-	defer removeRoots(fsDirs)
+
+	objLayer.Shutdown(context.Background())
+	if localMetacacheMgr != nil {
+		localMetacacheMgr.deleteAll()
+	}
+	setObjectLayer(newObjectLayerFn())
+	cancel()
+	removeRoots(fsDirs)
 }
 
 // ExecObjectLayerDiskAlteredTest - executes object layer tests while altering

cmd/untar.go

@@ -144,7 +144,8 @@ func untar(ctx context.Context, r io.Reader, putObject func(reader io.Reader, in
 	case formatS2:
 		r = s2.NewReader(bf)
 	case formatZstd:
-		dec, err := zstd.NewReader(bf)
+		// Limit to 16 MiB per stream.
+		dec, err := zstd.NewReader(bf, zstd.WithDecoderMaxWindow(16<<20))
 		if err != nil {
 			return err
 		}

cmd/update.go

@@ -518,6 +518,11 @@ func downloadBinary(u *url.URL, mode string) (readerReturn []byte, err error) {
 	return binaryFile, nil
 }
 
+const (
+	// Update this whenever the official minisign pubkey is rotated.
+	defaultMinisignPubkey = "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
+)
+
 func verifyBinary(u *url.URL, sha256Sum []byte, releaseInfo string, mode string, reader []byte) (err error) {
 	if !atomic.CompareAndSwapUint32(&updateInProgress, 0, 1) {
 		return errors.New("update already in progress")
@@ -538,7 +543,7 @@ func verifyBinary(u *url.URL, sha256Sum []byte, releaseInfo string, mode string,
 		}
 	}
 
-	minisignPubkey := env.Get(envMinisignPubKey, "")
+	minisignPubkey := env.Get(envMinisignPubKey, defaultMinisignPubkey)
 	if minisignPubkey != "" {
 		v := selfupdate.NewVerifier()
 		u.Path = path.Dir(u.Path) + slashSeparator + releaseInfo + ".minisig"

cmd/utils.go

@@ -639,6 +639,7 @@ const defaultDialTimeout = 5 * time.Second
 // NewHTTPTransportWithTimeout allows setting a timeout.
 func NewHTTPTransportWithTimeout(timeout time.Duration) *http.Transport {
 	return xhttp.ConnSettings{
+		DialContext: newCustomDialContext(),
 		DNSCache:    globalDNSCache,
 		DialTimeout: defaultDialTimeout,
 		RootCAs:     globalRootCAs,
@@ -674,6 +675,7 @@ func newCustomDialContext() dialContext {
 func NewRemoteTargetHTTPTransport() func() *http.Transport {
 	return xhttp.ConnSettings{
 		DialContext: newCustomDialContext(),
+		DNSCache:    globalDNSCache,
 		RootCAs:     globalRootCAs,
 		EnableHTTP2: false,
 	}.NewRemoteTargetHTTPTransport()

cmd/xl-storage-disk-id-check.go

@@ -175,10 +175,12 @@ func (p *xlStorageDiskIDCheck) Healing() *healingTracker {
 
 func (p *xlStorageDiskIDCheck) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (dataUsageCache, error) {
 	if contextCanceled(ctx) {
+		close(updates)
 		return dataUsageCache{}, ctx.Err()
 	}
 
 	if err := p.checkDiskStale(); err != nil {
+		close(updates)
 		return dataUsageCache{}, err
 	}
 	return p.storage.NSScanner(ctx, cache, updates, scanMode)

cmd/xl-storage-format-v2.go

@@ -1583,8 +1583,9 @@ func (x *xlMetaV2) AddVersion(fi FileInfo) error {
 			if len(k) > len(ReservedMetadataPrefixLower) && strings.EqualFold(k[:len(ReservedMetadataPrefixLower)], ReservedMetadataPrefixLower) {
 				// Skip tierFVID, tierFVMarker keys; it's used
 				// only for creating free-version.
+				// Skip xMinIOHealing, it's used only in RenameData
 				switch k {
-				case tierFVIDKey, tierFVMarkerKey:
+				case tierFVIDKey, tierFVMarkerKey, xMinIOHealing:
 					continue
 				}
 

cmd/xl-storage-format-v2_test.go

@@ -1010,3 +1010,51 @@ func Test_mergeXLV2Versions2(t *testing.T) {
 		})
 	}
 }
+
+func TestXMinIOHealingSkip(t *testing.T) {
+	xl := xlMetaV2{}
+	failOnErr := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatalf("Test failed with %v", err)
+		}
+	}
+
+	fi := FileInfo{
+		Volume:    "volume",
+		Name:      "object-name",
+		VersionID: "756100c6-b393-4981-928a-d49bbc164741",
+		IsLatest:  true,
+		Deleted:   false,
+		ModTime:   time.Now(),
+		Size:      1 << 10,
+		Mode:      0,
+		Erasure: ErasureInfo{
+			Algorithm:    ReedSolomon.String(),
+			DataBlocks:   4,
+			ParityBlocks: 2,
+			BlockSize:    10000,
+			Index:        1,
+			Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8},
+			Checksums: []ChecksumInfo{{
+				PartNumber: 1,
+				Algorithm:  HighwayHash256S,
+				Hash:       nil,
+			}},
+		},
+		NumVersions: 1,
+	}
+
+	fi.SetHealing()
+	failOnErr(xl.AddVersion(fi))
+
+	var err error
+	fi, err = xl.ToFileInfo(fi.Volume, fi.Name, fi.VersionID, false)
+	if err != nil {
+		t.Fatalf("xl.ToFileInfo failed with %v", err)
+	}
+
+	if fi.Healing() {
+		t.Fatal("Expected fi.Healing to be false")
+	}
+}

cmd/xl-storage.go

@@ -435,11 +435,19 @@ func (s *xlStorage) readMetadata(ctx context.Context, itemPath string) ([]byte,
 func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (dataUsageCache, error) {
 	atomic.AddInt32(&s.scanning, 1)
 	defer atomic.AddInt32(&s.scanning, -1)
+	var err error
+	stopFn := globalScannerMetrics.log(scannerMetricScanBucketDrive, s.diskPath, cache.Info.Name)
+	defer func() {
+		res := make(map[string]string)
+		if err != nil {
+			res["err"] = err.Error()
+		}
+		stopFn(res)
+	}()
 
 	// Updates must be closed before we return.
 	defer close(updates)
 	var lc *lifecycle.Lifecycle
-	var err error
 
 	// Check if the current bucket has a configured lifecycle policy
 	if globalLifecycleSys != nil {
@@ -514,7 +522,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 		}
 
 		done := globalScannerMetrics.time(scannerMetricApplyAll)
-		fivs.Versions, err = item.applyVersionActions(ctx, objAPI, fivs.Versions)
+		objInfos, err := item.applyVersionActions(ctx, objAPI, fivs.Versions)
 		done()
 
 		if err != nil {
@@ -524,8 +532,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 
 		versioned := vcfg != nil && vcfg.Versioned(item.objectPath())
 
-		for _, version := range fivs.Versions {
-			oi := version.ToObjectInfo(item.bucket, item.objectPath(), versioned)
+		for _, oi := range objInfos {
 			done = globalScannerMetrics.time(scannerMetricApplyVersion)
 			sz := item.applyActions(ctx, objAPI, oi, &sizeS)
 			done()
@@ -1672,35 +1679,10 @@ func (s *xlStorage) openFileDirect(path string, mode int) (f *os.File, err error
 }
 
 func (s *xlStorage) openFileSync(filePath string, mode int) (f *os.File, err error) {
-	// Create top level directories if they don't exist.
-	// with mode 0777 mkdir honors system umask.
-	if err = mkdirAll(pathutil.Dir(filePath), 0o777); err != nil {
-		return nil, osErrToFileErr(err)
-	}
-
-	w, err := OpenFile(filePath, mode|writeMode, 0o666)
-	if err != nil {
-		// File path cannot be verified since one of the parents is a file.
-		switch {
-		case isSysErrIsDir(err):
-			return nil, errIsNotRegular
-		case osIsPermission(err):
-			return nil, errFileAccessDenied
-		case isSysErrNotDir(err):
-			return nil, errFileAccessDenied
-		case isSysErrIO(err):
-			return nil, errFaultyDisk
-		case isSysErrTooManyFiles(err):
-			return nil, errTooManyOpenFiles
-		default:
-			return nil, err
-		}
-	}
-
-	return w, nil
+	return s.openFile(filePath, mode|writeMode)
 }
 
-func (s *xlStorage) openFileNoSync(filePath string, mode int) (f *os.File, err error) {
+func (s *xlStorage) openFile(filePath string, mode int) (f *os.File, err error) {
 	// Create top level directories if they don't exist.
 	// with mode 0777 mkdir honors system umask.
 	if err = mkdirAll(pathutil.Dir(filePath), 0o777); err != nil {
@@ -1715,6 +1697,8 @@ func (s *xlStorage) openFileNoSync(filePath string, mode int) (f *os.File, err e
 			return nil, errIsNotRegular
 		case osIsPermission(err):
 			return nil, errFileAccessDenied
+		case isSysErrNotDir(err):
+			return nil, errFileAccessDenied
 		case isSysErrIO(err):
 			return nil, errFaultyDisk
 		case isSysErrTooManyFiles(err):
@@ -1949,7 +1933,7 @@ func (s *xlStorage) writeAll(ctx context.Context, volume string, path string, b
 		}
 		w, err = s.openFileSync(filePath, flags)
 	} else {
-		w, err = s.openFileNoSync(filePath, flags)
+		w, err = s.openFile(filePath, flags)
 	}
 	if err != nil {
 		return err
@@ -2356,7 +2340,11 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 	// suspended or disabled on this bucket. RenameData will replace
 	// the 'null' version. We add a free-version to track its tiered
 	// content for asynchronous deletion.
-	if fi.VersionID == "" && !fi.IsRestoreObjReq() {
+	//
+	// Note: RestoreObject and HealObject requests don't end up replacing the
+	// null version and therefore don't require the free-version to track
+	// anything
+	if fi.VersionID == "" && !fi.IsRestoreObjReq() && !fi.Healing() {
 		// Note: Restore object request reuses PutObject/Multipart
 		// upload to copy back its data from the remote tier. This
 		// doesn't replace the existing version, so we don't need to add

docs/bucket/replication/setup_replication.sh

@@ -38,11 +38,11 @@ cat > repladmin-policy-source.json <<EOF
     ]
    }
 EOF
-mc admin policy add source repladmin-policy ./repladmin-policy-source.json
+mc admin policy create source repladmin-policy ./repladmin-policy-source.json
 cat ./repladmin-policy-source.json
 
 #assign this replication policy to repladmin
-mc admin policy set source repladmin-policy user=repladmin
+mc admin policy attach source repladmin-policy --user=repladmin
 
 ### on dest alias
 # Create a replication user : repluser on dest alias
@@ -90,11 +90,11 @@ cat > replpolicy.json <<EOF
  ]
 }
 EOF
-mc admin policy add dest replpolicy ./replpolicy.json
+mc admin policy create dest replpolicy ./replpolicy.json
 cat ./replpolicy.json
 
 # assign this replication policy to repluser
-mc admin policy set dest replpolicy user=repluser
+mc admin policy attach dest replpolicy --user=repluser
 
 # configure replication config to remote bucket at http://localhost:9000
 mc replicate add source/bucket --priority 1 --remote-bucket http://repluser:repluser123@localhost:9000/bucket \

docs/debugging/reorder-disks/go.mod

@@ -0,0 +1,5 @@
+module github.com/minio/minio/docs/debugging/reorder-disks
+
+go 1.19
+
+require github.com/minio/pkg v1.6.4

docs/debugging/reorder-disks/go.sum

@@ -0,0 +1,2 @@
+github.com/minio/pkg v1.6.4 h1:k6XlhyJ8zOn90PI4csuMeePx7BQrcX1jorKriR5J2fo=
+github.com/minio/pkg v1.6.4/go.mod h1:0iX1IuJGSCnMvIvrEJauk1GgQSX9JdU6Kh0P3EQRGkI=

docs/debugging/reorder-disks/main.go

@@ -0,0 +1,226 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/minio/pkg/ellipses"
+)
+
+type xl struct {
+	This string     `json:"this"`
+	Sets [][]string `json:"sets"`
+}
+
+type format struct {
+	ID string `json:"id"`
+	XL xl     `json:"xl"`
+}
+
+func getMountMap() (map[string]string, error) {
+	result := make(map[string]string)
+
+	mountInfo, err := os.Open("/proc/self/mountinfo")
+	if err != nil {
+		return nil, err
+	}
+	defer mountInfo.Close()
+
+	scanner := bufio.NewScanner(mountInfo)
+	for scanner.Scan() {
+		s := strings.Split(scanner.Text(), " ")
+		if len(s) != 11 {
+			return nil, errors.New("unsupport /proc/self/mountinfo format")
+		}
+		result[s[2]] = s[9]
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+func getDiskUUIDMap() (map[string]string, error) {
+	result := make(map[string]string)
+	err := filepath.Walk("/dev/disk/by-uuid/",
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			realPath, err := filepath.EvalSymlinks(path)
+			if err != nil {
+				return err
+			}
+			result[realPath] = strings.TrimPrefix(path, "/dev/disk/by-uuid/")
+			return nil
+		})
+	if err != nil {
+		return nil, err
+	}
+	return result, nil
+}
+
+type localDisk struct {
+	index int
+	path  string
+}
+
+func getMajorMinor(path string) (string, error) {
+	var stat syscall.Stat_t
+	if err := syscall.Stat(path, &stat); err != nil {
+		return "", err
+	}
+
+	major := (stat.Dev & 0x00000000000fff00) >> 8
+	major |= (stat.Dev & 0xfffff00000000000) >> 32
+	minor := (stat.Dev & 0x00000000000000ff) >> 0
+	minor |= (stat.Dev & 0x00000ffffff00000) >> 12
+
+	return fmt.Sprintf("%d:%d", major, minor), nil
+}
+
+func filterLocalDisks(node, args string) ([]localDisk, error) {
+	var result []localDisk
+
+	argP, err := ellipses.FindEllipsesPatterns(args)
+	if err != nil {
+		return nil, err
+	}
+	exp := argP.Expand()
+
+	if node == "" {
+		for index, e := range exp {
+			result = append(result, localDisk{index: index, path: strings.Join(e, "")})
+		}
+	} else {
+		for index, e := range exp {
+			u, err := url.Parse(strings.Join(e, ""))
+			if err != nil {
+				return nil, err
+			}
+			if strings.Contains(u.Host, node) {
+				result = append(result, localDisk{index: index, path: u.Path})
+			}
+		}
+	}
+
+	return result, nil
+}
+
+func getFormatJSON(path string) (format, error) {
+	formatJSON, err := ioutil.ReadFile(filepath.Join(path, ".minio.sys/format.json"))
+	if err != nil {
+		return format{}, err
+	}
+	var f format
+	err = json.Unmarshal(formatJSON, &f)
+	if err != nil {
+		return format{}, err
+	}
+
+	return f, nil
+}
+
+func getDiskLocation(f format) (string, error) {
+	for i, set := range f.XL.Sets {
+		for j, disk := range set {
+			if disk == f.XL.This {
+				return fmt.Sprintf("%d-%d", i, j), nil
+			}
+		}
+	}
+	return "", errors.New("disk not found")
+}
+
+func main() {
+
+	var node, args string
+
+	flag.StringVar(&node, "filter-node", "", "Filter disks which belong to the given node")
+	flag.StringVar(&args, "args", "", "arguments passed to MinIO server")
+
+	flag.Parse()
+
+	localDisks, err := filterLocalDisks(node, args)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	if len(localDisks) == 0 {
+		log.Fatal("Fix --filter-node or/and --args to select local disks.")
+	}
+
+	format, err := getFormatJSON(localDisks[0].path)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	setSize := len(format.XL.Sets[0])
+
+	expectedDisksName := make(map[string]string)
+	actualDisksName := make(map[string]string)
+
+	// Calculate the set/disk index
+	for _, disk := range localDisks {
+		expectedDisksName[fmt.Sprintf("%d-%d", disk.index/setSize, disk.index%setSize)] = disk.path
+		format, err := getFormatJSON(disk.path)
+		if err != nil {
+			log.Fatal(err)
+		}
+		foundDiskLoc, err := getDiskLocation(format)
+		if err != nil {
+			log.Fatal(err)
+		}
+		actualDisksName[foundDiskLoc] = disk.path
+	}
+
+	uuidMap, err := getDiskUUIDMap()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	mountMap, err := getMountMap()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for loc, expectedDiskName := range expectedDisksName {
+		diskName := actualDisksName[loc]
+		mami, err := getMajorMinor(diskName)
+		if err != nil {
+			log.Fatal(err)
+		}
+		devName := mountMap[mami]
+		uuid := uuidMap[devName]
+		fmt.Printf("UID=%s\t%s\txfs\tdefaults,noatime\t0\t2\n", uuid, expectedDiskName)
+	}
+}

docs/distributed/decom-compressed-sse-s3.sh

@@ -29,11 +29,11 @@ sleep 2
 ./mc admin user add myminio/ minio123 minio123
 ./mc admin user add myminio/ minio12345 minio12345
 
-./mc admin policy add myminio/ rw ./docs/distributed/rw.json
-./mc admin policy add myminio/ lake ./docs/distributed/rw.json
+./mc admin policy create myminio/ rw ./docs/distributed/rw.json
+./mc admin policy create myminio/ lake ./docs/distributed/rw.json
 
-./mc admin policy set myminio/ rw user=minio123
-./mc admin policy set myminio/ lake,rw user=minio12345
+./mc admin policy attach myminio/ rw --user=minio123
+./mc admin policy attach myminio/ lake,rw --user=minio12345
 
 ./mc mb -l myminio/versioned
 

docs/distributed/decom-encrypted-sse-s3.sh

@@ -24,11 +24,11 @@ sleep 2
 ./mc admin user add myminio/ minio123 minio123
 ./mc admin user add myminio/ minio12345 minio12345
 
-./mc admin policy add myminio/ rw ./docs/distributed/rw.json
-./mc admin policy add myminio/ lake ./docs/distributed/rw.json
+./mc admin policy create myminio/ rw ./docs/distributed/rw.json
+./mc admin policy create myminio/ lake ./docs/distributed/rw.json
 
-./mc admin policy set myminio/ rw user=minio123
-./mc admin policy set myminio/ lake,rw user=minio12345
+./mc admin policy attach myminio/ rw --user=minio123
+./mc admin policy attach myminio/ lake,rw --user=minio12345
 
 ./mc mb -l myminio/versioned
 

docs/distributed/decom-encrypted.sh

@@ -26,11 +26,11 @@ export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
 ./mc admin user add myminio/ minio123 minio123
 ./mc admin user add myminio/ minio12345 minio12345
 
-./mc admin policy add myminio/ rw ./docs/distributed/rw.json
-./mc admin policy add myminio/ lake ./docs/distributed/rw.json
+./mc admin policy create myminio/ rw ./docs/distributed/rw.json
+./mc admin policy create myminio/ lake ./docs/distributed/rw.json
 
-./mc admin policy set myminio/ rw user=minio123
-./mc admin policy set myminio/ lake,rw user=minio12345
+./mc admin policy attach myminio/ rw --user=minio123
+./mc admin policy attach myminio/ lake,rw --user=minio12345
 
 ./mc mb -l myminio/versioned
 

docs/distributed/decom.sh

@@ -6,6 +6,7 @@ fi
 
 pkill minio
 rm -rf /tmp/xl
+rm -rf /tmp/xltier
 
 if [ ! -f ./mc ]; then
     wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \
@@ -24,11 +25,11 @@ export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
 ./mc admin user add myminio/ minio123 minio123
 ./mc admin user add myminio/ minio12345 minio12345
 
-./mc admin policy add myminio/ rw ./docs/distributed/rw.json
-./mc admin policy add myminio/ lake ./docs/distributed/rw.json
+./mc admin policy create myminio/ rw ./docs/distributed/rw.json
+./mc admin policy create myminio/ lake ./docs/distributed/rw.json
 
-./mc admin policy set myminio/ rw user=minio123
-./mc admin policy set myminio/ lake,rw user=minio12345
+./mc admin policy attach myminio/ rw --user=minio123
+./mc admin policy attach myminio/ lake,rw --user=minio12345
 
 ./mc mb -l myminio/versioned
 
@@ -45,6 +46,23 @@ expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum)
 user_count=$(./mc admin user list myminio/ | wc -l)
 policy_count=$(./mc admin policy list myminio/ | wc -l)
 
+## create a warm tier instance
+(minio server /tmp/xltier/{1...4}/disk{0...1} --address :9001 2>&1 >/dev/null)&
+sleep 2
+export MC_HOST_mytier="http://minioadmin:minioadmin@localhost:9001/"
+
+./mc mb -l myminio/bucket2
+./mc mb -l mytier/tiered
+## create a tier and set up ilm policy to tier immediately
+./mc admin tier add minio myminio TIER1 --endpoint http://localhost:9001 --access-key minioadmin --secret-key minioadmin --bucket tiered --prefix prefix5/
+./mc ilm add myminio/bucket2 --transition-days 0 --transition-tier TIER1 --transition-days 0
+## mirror some content to bucket2 and capture versions tiered
+./mc mirror internal myminio/bucket2/ --quiet >/dev/null
+./mc ls -r myminio/bucket2/ > bucket2_ns.txt
+./mc ls -r --versions myminio/bucket2/ > bucket2_ns_versions.txt
+sleep 2
+./mc ls -r --versions mytier/tiered/ > tiered_ns_versions.txt
+
 kill $pid
 (minio server /tmp/xl/{1...10}/disk{0...1} /tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded.log) &
 pid=$!
@@ -134,4 +152,43 @@ if [ "${expected_checksum}" != "${got_checksum}" ]; then
     exit 1
 fi
 
+# after decommissioning, compare listings in bucket2 and tiered
+./mc version info myminio/bucket2 | grep -q "versioning is enabled"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected versioning enabled after decommission on bucket2"
+    exit 1
+fi
+
+./mc ls -r myminio/bucket2 > decommissioned_bucket2_ns.txt
+./mc ls -r --versions myminio/bucket2 > decommissioned_bucket2_ns_versions.txt
+./mc ls -r --versions mytier/tiered/ > tiered_ns_versions2.txt
+
+out=$(diff -qpruN bucket2_ns.txt decommissioned_bucket2_ns.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission in bucket2: $out"
+    exit 1
+fi
+
+out=$(diff -qpruN bucket2_ns_versions.txt decommissioned_bucket2_ns_versions.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission in bucket2x: $out"
+    exit 1
+fi
+
+out=$(diff -qpruN tiered_ns_versions.txt tiered_ns_versions2.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after decommission in warm tier: $out"
+    exit 1
+fi
+
+got_checksum=$(./mc cat myminio/bucket2/dsync/drwmutex.go | md5sum)
+if [ "${expected_checksum}" != "${got_checksum}" ]; then
+    echo "BUG: decommission failed on encrypted objects with tiering: expected ${expected_checksum} got ${got_checksum}"
+    exit 1
+fi
+
 kill $pid

docs/lambda/README.md

@@ -173,7 +173,7 @@ func main() {
 
 Use the Presigned URL via `curl` to receive the transformed object.
 ```
-~ curl -v $(go run presigned.go)
+curl -v $(go run presigned.go)
 ...
 ...
 > GET /functionbucket/testobject?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin%2F20230205%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230205T173023Z&X-Amz-Expires=1000&X-Amz-SignedHeaders=host&lambdaArn=arn%3Aminio%3As3-object-lambda%3A%3Atoupper%3Awebhook&X-Amz-Signature=d7e343f0da9d4fa2bc822c12ad2f54300ff16796a1edaa6d31f1313c8e94d5b2 HTTP/1.1

docs/multi-user/README.md

@@ -41,7 +41,7 @@ EOF
 Create new canned policy by name `getonly` using `getonly.json` policy file.
 
 ```
-mc admin policy add myminio getonly getonly.json
+mc admin policy create myminio getonly getonly.json
 ```
 
 Create a new user `newuser` on MinIO use `mc admin user`.
@@ -53,7 +53,7 @@ mc admin user add myminio newuser newuser123
 Once the user is successfully created you can now apply the `getonly` policy for this user.
 
 ```
-mc admin policy set myminio getonly user=newuser
+mc admin policy attach myminio getonly --user=newuser
 ```
 
 ### 3. Create a new group
@@ -65,7 +65,7 @@ mc admin group add myminio newgroup newuser
 Once the group is successfully created you can now apply the `getonly` policy for this group.
 
 ```
-mc admin policy set myminio getonly group=newgroup
+mc admin policy attach myminio getonly --group=newgroup
 ```
 
 ### 4. Disable user
@@ -107,13 +107,13 @@ mc admin group remove myminio newgroup
 Change the policy for user `newuser` to `putonly` canned policy.
 
 ```
-mc admin policy set myminio putonly user=newuser
+mc admin policy attach myminio putonly --user=newuser
 ```
 
 Change the policy for group `newgroup` to `putonly` canned policy.
 
 ```
-mc admin policy set myminio putonly group=newgroup
+mc admin policy attach myminio putonly --group=newgroup
 ```
 
 ### 7. List all users or groups

docs/multi-user/admin/README.md

@@ -50,7 +50,7 @@ EOF
 Create new canned policy by name `userManager` using `userManager.json` policy file.
 
 ```
-mc admin policy add myminio userManager adminManageUser.json
+mc admin policy attach myminio userManager adminManageUser.json
 ```
 
 Create a new admin user `admin1` on MinIO use `mc admin user`.
@@ -62,7 +62,7 @@ mc admin user add myminio admin1 admin123
 Once the user is successfully created you can now apply the `userManage` policy for this user.
 
 ```
-mc admin policy set myminio userManager user=admin1
+mc admin policy attach myminio userManager --user=admin1
 ```
 
 This admin user will then be allowed to perform create/delete user operations via `mc admin user`
@@ -73,8 +73,8 @@ This admin user will then be allowed to perform create/delete user operations vi
 mc alias set myminio-admin1 http://localhost:9000 admin1 admin123 --api s3v4
 
 mc admin user add myminio-admin1 user1 user123
-mc admin policy add myminio-admin1 user1policy ~/user1policy.json
-mc admin policy set myminio-admin1 user1policy user=user1
+mc admin policy attach myminio-admin1 user1policy ~/user1policy.json
+mc admin policy attach myminio-admin1 user1policy --user=user1
 ```
 
 ### 4. List of permissions defined for admin operations