Add minio_bucket_usage_version_total metric to Prometheus (#17023)

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>

.github/workflows/go-cross.yml

@@ -20,11 +20,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-      - uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/go-fips.yml

@@ -20,21 +20,28 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
       - name: Setup dockerfile for build test
         run: |
-          echo "FROM golang:1.19.4" >> Dockerfile.fips.test
-          echo "COPY . /minio" >> Dockerfile.fips.test
-          echo "WORKDIR /minio" >> Dockerfile.fips.test
-          echo "ENV GOEXPERIMENT=boringcrypto" >> Dockerfile.fips.test
-          echo "RUN make" >> Dockerfile.fips.test
+          GO_VERSION=$(go version | cut -d ' ' -f 3 | sed 's/go//')
+          echo Detected go version $GO_VERSION
+          cat > Dockerfile.fips.test <<EOF
+          FROM golang:${GO_VERSION}
+          COPY . /minio
+          WORKDIR /minio
+          ENV GOEXPERIMENT=boringcrypto
+          RUN make
+          EOF
 
       - name: Build
         uses: docker/build-push-action@v3
@@ -49,4 +56,4 @@ jobs:
       - name: Test binary
         run: |
           docker run --rm minio/fips-test:latest ./minio --version
-          docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio' | grep -q FIPS
+          docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio | grep FIPS | grep -q FIPS'

.github/workflows/go-healing.yml

@@ -20,10 +20,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/go-lint.yml

@@ -20,10 +20,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest, windows-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/go.yml

@@ -20,10 +20,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/iam-integrations.yaml

@@ -61,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]
@@ -75,7 +75,7 @@ jobs:
             openid: "http://127.0.0.1:5556/dex"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/markdown-lint.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Lint all docs
       run: |

.github/workflows/mint.yml

@@ -0,0 +1,57 @@
+name: Mint Tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  mint-test:
+    runs-on: mint
+    timeout-minutes: 120
+    steps:
+      - name: cleanup #https://github.com/actions/checkout/issues/273
+        run: |
+          sudo -S rm -rf ${GITHUB_WORKSPACE}
+          mkdir ${GITHUB_WORKSPACE}
+      - name: checkout-step
+        uses: actions/checkout@v3
+
+      - name: setup-go-step
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.20.x
+
+      - name: github sha short
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: build-minio
+        run: |
+          make install
+          docker build . -t "minio/minio:${{ steps.vars.outputs.sha_short }}"
+
+      - name: compress and encrypt
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "compress-encrypt" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+
+      - name: multiple pools
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "pools" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+
+      - name: standalone erasure
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+          docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
+
+
+

.github/workflows/mint/minio-compress-encrypt.yaml

@@ -0,0 +1,79 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_COMPRESS: "true"
+    MINIO_COMPRESS_MIMETYPES: "*"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - cdata1-1:/cdata1
+      - cdata1-2:/cdata2
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - cdata2-1:/cdata1
+      - cdata2-2:/cdata2
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - cdata3-1:/cdata1
+      - cdata3-2:/cdata2
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - cdata4-1:/cdata1
+      - cdata4-2:/cdata2
+
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  cdata1-1:
+  cdata1-2:
+  cdata2-1:
+  cdata2-2:
+  cdata3-1:
+  cdata3-2:
+  cdata4-1:
+  cdata4-2:

.github/workflows/mint/minio-erasure.yaml

@@ -0,0 +1,53 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" edata{1...4}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_COMPRESS: "true"
+    MINIO_COMPRESS_MIMETYPES: "*"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - edata1-1:/edata1
+      - edata1-2:/edata2
+      - edata1-3:/edata3
+      - edata1-4:/edata4
+
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-1-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  edata1-1:
+  edata1-2:
+  edata1-3:
+  edata1-4:

.github/workflows/mint/minio-pools.yaml

@@ -0,0 +1,117 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - pdata1-1:/pdata1
+      - pdata1-2:/pdata2
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - pdata2-1:/pdata1
+      - pdata2-2:/pdata2
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - pdata3-1:/pdata1
+      - pdata3-2:/pdata2
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - pdata4-1:/pdata1
+      - pdata4-2:/pdata2
+
+  minio5:
+    <<: *minio-common
+    hostname: minio5
+    volumes:
+      - pdata5-1:/pdata1
+      - pdata5-2:/pdata2
+
+  minio6:
+    <<: *minio-common
+    hostname: minio6
+    volumes:
+      - pdata6-1:/pdata1
+      - pdata6-2:/pdata2
+
+  minio7:
+    <<: *minio-common
+    hostname: minio7
+    volumes:
+      - pdata7-1:/pdata1
+      - pdata7-2:/pdata2
+
+  minio8:
+    <<: *minio-common
+    hostname: minio8
+    volumes:
+      - pdata8-1:/pdata1
+      - pdata8-2:/pdata2
+      
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-8-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+      - minio5
+      - minio6
+      - minio7
+      - minio8
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  pdata1-1:
+  pdata1-2:
+  pdata2-1:
+  pdata2-2:
+  pdata3-1:
+  pdata3-2:
+  pdata4-1:
+  pdata4-2:
+  pdata5-1:
+  pdata5-2:
+  pdata6-1:
+  pdata6-2:
+  pdata7-1:
+  pdata7-2:
+  pdata8-1:
+  pdata8-2:

.github/workflows/mint/nginx-1-node.conf

@@ -0,0 +1,100 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/mint/nginx-4-node.conf

@@ -0,0 +1,106 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/mint/nginx-8-node.conf

@@ -0,0 +1,114 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+        server minio5:9000;
+        server minio6:9000;
+        server minio7:9000;
+        server minio8:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+        server minio5:9001;
+        server minio6:9001;
+        server minio7:9001;
+        server minio8:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/mint/nginx.conf

@@ -0,0 +1,106 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/replication.yaml

@@ -21,10 +21,10 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/run-mint.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -ex
+
+export MODE="$1"
+export ACCESS_KEY="$2"
+export SECRET_KEY="$3"
+export JOB_NAME="$4"
+export MINT_MODE="full"
+
+docker system prune -f
+docker volume prune -f
+
+## change working directory
+cd .github/workflows/mint
+
+docker-compose -f minio-${MODE}.yaml up -d
+sleep 5m
+
+docker run --rm --net=host \
+       --name="mint-${MODE}-${JOB_NAME}" \
+       -e SERVER_ENDPOINT="127.0.0.1:9000" \
+       -e ACCESS_KEY="${ACCESS_KEY}" \
+       -e SECRET_KEY="${SECRET_KEY}" \
+       -e ENABLE_HTTPS=0 \
+       -e MINT_MODE="${MINT_MODE}" \
+       docker.io/minio/mint:edge \
+	 aws-sdk-go   \
+	 aws-sdk-java \
+	 aws-sdk-php  \
+	 aws-sdk-ruby \
+	 awscli       \
+	 healthcheck  \
+	 mc           \
+	 minio-go     \
+	 minio-java   \
+	 minio-js     \
+	 minio-py     \
+	 s3cmd        \
+	 s3select     \
+	 versioning
+
+docker-compose -f minio-${MODE}.yaml down || true
+sleep 10s
+
+docker system prune -f || true
+docker volume prune -f || true
+
+## change working directory
+cd ../../../
+

.github/workflows/upgrade-ci-cd.yaml

@@ -20,11 +20,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/vulncheck.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.x
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.gitignore

@@ -38,3 +38,4 @@ docs/debugging/s3-check-md5/s3-check-md5
 docs/debugging/hash-set/hash-set
 docs/debugging/healing-bin/healing-bin
 docs/debugging/inspect/inspect
+.bin/

.golangci.yml

@@ -1,39 +1,34 @@
 linters-settings:
   gofumpt:
-    lang-version: "1.18"
+    simplify: true
 
   misspell:
     locale: US
 
+  staticcheck:
+    checks: ['all', '-ST1005', '-ST1000', '-SA4000', '-SA9004', '-SA1019', '-SA1008', '-U1000', '-ST1016']
+
 linters:
   disable-all: true
   enable:
-    - typecheck
-    - goimports
-    - misspell
-    - govet
-    - revive
-    - ineffassign
-    - gomodguard
+    - durationcheck
+    - gocritic
     - gofmt
+    - gofumpt
+    - goimports
+    - gomodguard
+    - govet
+    - ineffassign
+    - misspell
+    - revive
+    - staticcheck
+    - tenv
+    - typecheck
     - unconvert
     - unused
-    - gocritic
-    - gofumpt
-    - tenv
-    - durationcheck
 
 issues:
   exclude-use-default: false
   exclude:
-      - should have a package comment
-      - error strings should not be capitalized or end with punctuation or a newline
-      # todo fix these when we get enough time.
-      - "singleCaseSwitch: should rewrite switch statement to if statement"
-      - "unlambda: replace"
-      - "captLocal:"
-      - "ifElseChain:"
-      - "elseif:"
-
-service:
-  golangci-lint-version: 1.43.0 # use the fixed version to not introduce new linters unexpectedly
+    - should have a package comment
+    - error strings should not be capitalized or end with punctuation or a newline

Makefile

@@ -8,6 +8,10 @@ GOOS := $(shell go env GOOS)
 VERSION ?= $(shell git describe --tags)
 TAG ?= "minio/minio:$(VERSION)"
 
+GOLANGCI_VERSION = v1.51.2
+GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
+GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
+
 all: build
 
 checks: ## check dependencies
@@ -19,24 +23,26 @@ help: ## print this help
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION)
 	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
-	@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
 
 crosscompile: ## cross compile minio
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps lint check-gen
+verifiers: lint check-gen
 
 check-gen: ## check for updated autogenerated files
 	@go generate ./... >/dev/null
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 
-lint: ## runs golangci-lint suite of linters
+lint: getdeps ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
-	@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
-	@${GOPATH}/bin/staticcheck --tests=false ./...
+	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
+
+lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
+	@echo "Running $@ check"
+	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
 
 check: test
 test: verifiers build ## builds minio, runs linters, tests

NOTICE

@@ -1,4 +1,4 @@
-MinIO Project, (C) 2015-2021 MinIO, Inc.
+MinIO Project, (C) 2015-2023 MinIO, Inc.
 
 This product includes software developed at MinIO, Inc.
 (https://min.io/).

README.md

@@ -86,8 +86,6 @@ chmod +x minio
 ./minio server /data
 ```
 
-Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
-
 The following table lists supported architectures. Replace the `wget` URL with the architecture for your Linux host.
 
 | Architecture                   | URL                                                        |

buildscripts/race.sh

@@ -5,7 +5,6 @@ set -e
 export GORACE="history_size=7"
 export MINIO_API_REQUESTS_MAX=10000
 
-## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
-for d in $(go list ./... | grep -v dsync); do
+for d in $(go list ./...); do
     CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
 done

buildscripts/verify-healing-with-root-disks.sh

@@ -45,16 +45,18 @@ function start_minio() {
 
 # Prepare fake disks with losetup
 function prepare_block_devices() {
-        mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
-        for i in 1 2 3 4; do
-                dd if=/dev/zero of=${WORK_DIR}/disks/img.$i bs=1M count=2048
-                mkfs.ext4 -F ${WORK_DIR}/disks/img.$i
-                sudo mknod /dev/minio-loopdisk$i b 7 $[256-$i]
-                sudo losetup /dev/minio-loopdisk$i ${WORK_DIR}/disks/img.$i
-                mkdir -p ${WORK_DIR}/mnt/disk$i/
-                sudo mount /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
-                sudo chown "$(id -u):$(id -g)" /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
-        done
+    set -e
+    mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
+    sudo modprobe loop
+    for i in 1 2 3 4; do
+        dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000
+        device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i})
+        sudo mkfs.ext4 -F ${device}
+        mkdir -p ${WORK_DIR}/mnt/disk${i}/
+        sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/
+        sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/
+    done
+    set +e
 }
 
 # Start a distributed MinIO setup, unmount one disk and check if it is formatted

cmd/acl-handlers.go

@@ -90,7 +90,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 	if aclHeader == "" {
 		acl := &accessControlPolicy{}
 		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
-			if err == io.EOF {
+			if terr, ok := err.(*xml.SyntaxError); ok && terr.Msg == io.EOF.Error() {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingSecurityHeader),
 					r.URL)
 				return

cmd/admin-bucket-handlers.go

@@ -31,7 +31,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -199,7 +199,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		ops = madmin.GetTargetUpdateOps(r.Form)
 	} else {
 		var exists bool // true if arn exists
-		target.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &target)
+		target.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &target, "")
 		if exists && target.Arn != "" { // return pre-existing ARN
 			data, err := json.Marshal(target.Arn)
 			if err != nil {
@@ -225,10 +225,12 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		for _, op := range ops {
 			switch op {
 			case madmin.CredentialsUpdateType:
-				if !globalSiteReplicationSys.isEnabled() {
-					tgt.Credentials = target.Credentials
-					tgt.TargetBucket = target.TargetBucket
+				if globalSiteReplicationSys.isEnabled() {
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetDenyEditError, err), r.URL)
+					return
 				}
+				tgt.Credentials = target.Credentials
+				tgt.TargetBucket = target.TargetBucket
 				tgt.Secure = target.Secure
 				tgt.Endpoint = target.Endpoint
 			case madmin.SyncUpdateType:
@@ -684,8 +686,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			continue
 		}
 		bucket, fileName := slc[0], slc[1]
-		switch fileName {
-		case objectLockConfig:
+		if fileName == objectLockConfig {
 			reader, err := file.Open()
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)
@@ -754,8 +755,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			continue
 		}
 		bucket, fileName := slc[0], slc[1]
-		switch fileName {
-		case bucketVersioningConfig:
+		if fileName == bucketVersioningConfig {
 			reader, err := file.Open()
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)

cmd/admin-handler-utils.go

@@ -23,7 +23,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"

cmd/admin-handlers-config-kv.go

@@ -35,6 +35,7 @@ import (
 	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -87,6 +88,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	// Check if subnet proxy being deleted and if so the value of proxy of subnet
+	// target of logger webhook configuration also should be deleted
+	loggerWebhookProxyDeleted := setLoggerWebhookSubnetProxy(subSys, cfg)
+
 	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -101,6 +106,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	dynamic := config.SubSystemsDynamic.Contains(subSys)
 	if dynamic {
 		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
+		if subSys == config.SubnetSubSys && loggerWebhookProxyDeleted {
+			// Logger webhook proxy deleted, apply the dynamic changes
+			applyDynamic(ctx, objectAPI, cfg, config.LoggerWebhookSubSys, r, w)
+		}
 	}
 }
 
@@ -132,9 +141,10 @@ func (bce badConfigErr) Unwrap() error {
 }
 
 type setConfigResult struct {
-	Cfg     config.Config
-	SubSys  string
-	Dynamic bool
+	Cfg                     config.Config
+	SubSys                  string
+	Dynamic                 bool
+	LoggerWebhookCfgUpdated bool
 }
 
 // SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
@@ -175,6 +185,11 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	if result.Dynamic {
 		applyDynamic(ctx, objectAPI, result.Cfg, result.SubSys, r, w)
+		// If logger webhook config updated (proxy due to callhome), explicitly dynamically
+		// apply the config
+		if result.LoggerWebhookCfgUpdated {
+			applyDynamic(ctx, objectAPI, result.Cfg, config.LoggerWebhookSubSys, r, w)
+		}
 	}
 
 	writeSuccessResponseHeadersOnly(w)
@@ -201,6 +216,10 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 		return
 	}
 
+	// Check if subnet proxy being set and if so set the same value to proxy of subnet
+	// target of logger webhook configuration
+	result.LoggerWebhookCfgUpdated = setLoggerWebhookSubnetProxy(result.SubSys, result.Cfg)
+
 	// Update the actual server config on disk.
 	if err = saveServerConfig(ctx, objectAPI, result.Cfg); err != nil {
 		return
@@ -501,7 +520,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 			case config.IdentityLDAPSubSys:
 				off = !xldap.Enabled(item.Config)
 			case config.IdentityTLSSubSys:
-				off = !globalSTSTLSConfig.Enabled
+				off = !globalIAMSys.STSTLSConfig.Enabled
 			case config.IdentityPluginSubSys:
 				off = !idplugin.Enabled(item.Config)
 			}
@@ -518,3 +537,18 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	writeSuccessResponseJSON(w, econfigData)
 }
+
+// setLoggerWebhookSubnetProxy - Sets the logger webhook's subnet proxy value to
+// one being set for subnet proxy
+func setLoggerWebhookSubnetProxy(subSys string, cfg config.Config) bool {
+	if subSys == config.SubnetSubSys {
+		subnetWebhookCfg := cfg[config.LoggerWebhookSubSys][subnet.LoggerWebhookName]
+		loggerWebhookSubnetProxy := subnetWebhookCfg.Get(logger.Proxy)
+		subnetProxy := cfg[config.SubnetSubSys][config.Default].Get(logger.Proxy)
+		if loggerWebhookSubnetProxy != subnetProxy {
+			subnetWebhookCfg.Set(logger.Proxy, subnetProxy)
+			return true
+		}
+	}
+	return false
+}

cmd/admin-handlers-idp-config.go

@@ -86,7 +86,7 @@ func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.Resp
 		if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default {
 			// LDAP does not support multiple configurations. So cfgName must be
 			// empty or `madmin.Default`.
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPNonDefaultConfigName), r.URL)
 			return
 		}
 	}
@@ -178,9 +178,9 @@ func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isU
 	var cfgInfos []madmin.IDPCfgInfo
 	switch subSys {
 	case madmin.IdentityOpenIDSubSys:
-		cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
+		cfgInfos, _ = globalIAMSys.OpenIDConfig.GetConfigInfo(s, cfgTarget)
 	case madmin.IdentityLDAPSubSys:
-		cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
+		cfgInfos, _ = globalIAMSys.LDAPConfig.GetConfigInfo(s, cfgTarget)
 	}
 
 	if len(cfgInfos) > 0 && !isUpdate {
@@ -240,10 +240,10 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
 	switch idpCfgType {
 	case madmin.OpenidIDPCfg:
 		cfg := globalServerConfig.Clone()
-		cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
+		cfgList, err = globalIAMSys.OpenIDConfig.GetConfigList(cfg)
 	case madmin.LDAPIDPCfg:
 		cfg := globalServerConfig.Clone()
-		cfgList, err = globalLDAPConfig.GetConfigList(cfg)
+		cfgList, err = globalIAMSys.LDAPConfig.GetConfigList(cfg)
 
 	default:
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -296,9 +296,9 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 	var err error
 	switch idpCfgType {
 	case madmin.OpenidIDPCfg:
-		cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+		cfgInfos, err = globalIAMSys.OpenIDConfig.GetConfigInfo(cfg, cfgName)
 	case madmin.LDAPIDPCfg:
-		cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
+		cfgInfos, err = globalIAMSys.LDAPConfig.GetConfigInfo(cfg, cfgName)
 	}
 	if err != nil {
 		if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
@@ -355,7 +355,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 	switch idpCfgType {
 	case madmin.OpenidIDPCfg:
 		subSys = config.IdentityOpenIDSubSys
-		cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
+		cfgInfos, err := globalIAMSys.OpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
 		if err != nil {
 			if errors.Is(err, openid.ErrProviderConfigNotFound) {
 				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
@@ -380,7 +380,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 		}
 	case madmin.LDAPIDPCfg:
 		subSys = config.IdentityLDAPSubSys
-		cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
+		cfgInfos, err := globalIAMSys.LDAPConfig.GetConfigInfo(cfgCopy, cfgName)
 		if err != nil {
 			if errors.Is(err, openid.ErrProviderConfigNotFound) {
 				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)

cmd/admin-handlers-users-race_test.go

@@ -27,12 +27,12 @@ import (
 	"context"
 	"fmt"
 	"runtime"
-	"sync"
 	"testing"
 	"time"
 
 	"github.com/minio/madmin-go/v2"
 	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio/internal/sync/errgroup"
 )
 
 func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
@@ -129,18 +129,21 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
 		secretKeys[i] = secretKey
 	}
 
-	wg := sync.WaitGroup{}
+	g := errgroup.Group{}
 	for i := 0; i < userCount; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
-			err := s.adm.RemoveUser(ctx, accessKeys[i])
-			if err != nil {
-				c.Fatalf("unable to remove user: %v", err)
+		g.Go(func(i int) func() error {
+			return func() error {
+				uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
+				err := s.adm.RemoveUser(ctx, accessKeys[i])
+				if err != nil {
+					return err
+				}
+				c.mustNotListObjects(ctx, uClient, bucket)
+				return nil
 			}
-			c.mustNotListObjects(ctx, uClient, bucket)
-		}(i)
+		}(i), i)
+	}
+	if errs := g.Wait(); len(errs) > 0 {
+		c.Fatalf("unable to remove users: %v", errs)
 	}
-	wg.Wait()
 }

cmd/admin-handlers-users.go

@@ -431,7 +431,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Not allowed to add a user with same access key as root credential
-	if owner && accessKey == cred.AccessKey {
+	if accessKey == globalActiveCred.AccessKey {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
 		return
 	}
@@ -582,6 +582,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		AccountStatus: stsAccount.Status,
 		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
+		Expiration:    &stsAccount.Expiration,
 	}
 
 	data, err := json.Marshal(infoResp)
@@ -650,10 +651,11 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	}
 
 	opts := newServiceAccountOpts{
-		accessKey: createReq.AccessKey,
-		secretKey: createReq.SecretKey,
-		comment:   createReq.Comment,
-		claims:    make(map[string]interface{}),
+		accessKey:  createReq.AccessKey,
+		secretKey:  createReq.SecretKey,
+		comment:    createReq.Comment,
+		expiration: createReq.Expiration,
+		claims:     make(map[string]interface{}),
 	}
 
 	// Find the user for the request sender (as it may be sent via a service
@@ -672,8 +674,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// It could be a regular user account or the root account.
 		_, isRegularUser := globalIAMSys.GetUser(ctx, targetUser)
 		if !isRegularUser && targetUser != globalActiveCred.AccessKey {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
-				fmt.Errorf("parent user %s does not exist. Cannot create service account", targetUser)), r.URL)
+			apiErr := toAdminAPIErr(ctx, errNoSuchUser)
+			apiErr.Description = fmt.Sprintf("Specified target user %s does not exist", targetUser)
+			writeErrorResponseJSON(ctx, w, apiErr, r.URL)
 			return
 		}
 	}
@@ -740,9 +743,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 
 		// In case of LDAP we need to resolve the targetUser to a DN and
 		// query their groups:
-		if globalLDAPConfig.Enabled() {
+		if globalIAMSys.LDAPConfig.Enabled() {
 			opts.claims[ldapUserN] = targetUser // simple username
-			targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
+			targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
@@ -774,8 +777,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 
 	createResp := madmin.AddServiceAccountResp{
 		Credentials: madmin.Credentials{
-			AccessKey: newCred.AccessKey,
-			SecretKey: newCred.SecretKey,
+			AccessKey:  newCred.AccessKey,
+			SecretKey:  newCred.SecretKey,
+			Expiration: newCred.Expiration,
 		},
 	}
 
@@ -808,6 +812,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 					Claims:        opts.claims,
 					SessionPolicy: createReq.Policy,
 					Status:        auth.AccountOn,
+					Expiration:    createReq.Expiration,
 				},
 			},
 			UpdatedAt: updatedAt,
@@ -890,6 +895,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		secretKey:     updateReq.NewSecretKey,
 		status:        updateReq.NewStatus,
 		comment:       updateReq.NewComment,
+		expiration:    updateReq.NewExpiration,
 		sessionPolicy: sp,
 	}
 	updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
@@ -909,6 +915,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 					Status:        opts.status,
 					Comment:       opts.comment,
 					SessionPolicy: updateReq.NewPolicy,
+					Expiration:    updateReq.NewExpiration,
 				},
 			},
 			UpdatedAt: updatedAt,
@@ -987,12 +994,18 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	var expiration *time.Time
+	if !svcAccount.Expiration.IsZero() && !svcAccount.Expiration.Equal(timeSentinel) {
+		expiration = &svcAccount.Expiration
+	}
+
 	infoResp := madmin.InfoServiceAccountResp{
 		ParentUser:    svcAccount.ParentUser,
 		Comment:       svcAccount.Comment,
 		AccountStatus: svcAccount.Status,
 		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
+		Expiration:    expiration,
 	}
 
 	data, err := json.Marshal(infoResp)
@@ -1320,11 +1333,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			// Fetch the data usage of the current bucket
 			var size uint64
 			var objectsCount uint64
-			var objectsHist map[string]uint64
+			var objectsHist, versionsHist map[string]uint64
 			if !dataUsageInfo.LastUpdate.IsZero() {
 				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
 				objectsCount = dataUsageInfo.BucketsUsage[bucket.Name].ObjectsCount
 				objectsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectSizesHistogram
+				versionsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectVersionsHistogram
 			}
 			// Fetch the prefix usage of the current bucket
 			var prefixUsage map[string]uint64
@@ -1338,12 +1352,13 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			tcfg, _, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
 
 			acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
-				Name:                 bucket.Name,
-				Created:              bucket.Created,
-				Size:                 size,
-				Objects:              objectsCount,
-				ObjectSizesHistogram: objectsHist,
-				PrefixUsage:          prefixUsage,
+				Name:                    bucket.Name,
+				Created:                 bucket.Created,
+				Size:                    size,
+				Objects:                 objectsCount,
+				ObjectSizesHistogram:    objectsHist,
+				ObjectVersionsHistogram: versionsHist,
+				PrefixUsage:             prefixUsage,
 				Details: &madmin.BucketDetails{
 					Versioning:          globalBucketVersioningSys.Enabled(bucket.Name),
 					VersioningSuspended: globalBucketVersioningSys.Suspended(bucket.Name),
@@ -1942,7 +1957,7 @@ func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 			UserOrGroup: userOrGroup,
 			UserType:    int(userType),
 			IsGroup:     isGroup,
-			Policy:      strings.Join(policiesToDetach, ","),
+			Policy:      newPolicies,
 		},
 		UpdatedAt: updatedAt,
 	}))
@@ -2034,9 +2049,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case allUsersFile:
 			userIdentities := make(map[string]UserIdentity)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadUsers(ctx, regUser, userIdentities)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2064,9 +2077,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case allGroupsFile:
 			groups := make(map[string]GroupInfo)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadGroups(ctx, groups)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2083,9 +2094,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case allSvcAcctsFile:
 			serviceAccounts := make(map[string]UserIdentity)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadUsers(ctx, svcUser, serviceAccounts)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2134,9 +2143,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case userPolicyMappingsFile:
 			userPolicyMap := make(map[string]MappedPolicy)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2153,9 +2160,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case groupPolicyMappingsFile:
 			groupPolicyMap := make(map[string]MappedPolicy)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2172,9 +2177,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case stsUserPolicyMappingsFile:
 			userPolicyMap := make(map[string]MappedPolicy)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2190,9 +2193,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		case stsGroupPolicyMappingsFile:
 			groupPolicyMap := make(map[string]MappedPolicy)
-			globalIAMSys.store.rlock()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
-			globalIAMSys.store.runlock()
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -2297,7 +2298,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 			for accessKey, ureq := range userAccts {
 				// Not allowed to add a user with same access key as root credential
-				if owner && accessKey == cred.AccessKey {
+				if accessKey == globalActiveCred.AccessKey {
 					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
 					return
 				}
@@ -2449,6 +2450,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 						secretKey:     svcAcctReq.SecretKey,
 						status:        svcAcctReq.Status,
 						comment:       svcAcctReq.Comment,
+						expiration:    svcAcctReq.Expiration,
 						sessionPolicy: sp,
 					}
 					_, err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
@@ -2464,13 +2466,14 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					sessionPolicy: sp,
 					claims:        svcAcctReq.Claims,
 					comment:       svcAcctReq.Comment,
+					expiration:    svcAcctReq.Expiration,
 				}
 
 				// In case of LDAP we need to resolve the targetUser to a DN and
 				// query their groups:
-				if globalLDAPConfig.Enabled() {
+				if globalIAMSys.LDAPConfig.Enabled() {
 					opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
-					targetUser, _, err := globalLDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
+					targetUser, _, err := globalIAMSys.LDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
 					if err != nil {
 						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
 						return

cmd/admin-handlers-users_test.go

@@ -825,7 +825,7 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
 	if set.CreateStringSet(groups...).Contains(group) {
 		c.Fatalf("created group still present!")
 	}
-	groupInfo, err = s.adm.GetGroupDescription(ctx, group)
+	_, err = s.adm.GetGroupDescription(ctx, group)
 	if err == nil {
 		c.Fatalf("group appears to exist")
 	}

cmd/admin-handlers.go

@@ -53,9 +53,9 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/logger/message/log"
 	xnet "github.com/minio/pkg/net"
 	"github.com/secure-io/sio-go"
 )
@@ -533,16 +533,19 @@ func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string)
 func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 	now := time.Now().UTC()
 	entryMap := make(map[string]*madmin.LockEntry)
+	toEntry := func(lri lockRequesterInfo) string {
+		return fmt.Sprintf("%s/%s", lri.Name, lri.UID)
+	}
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
 			continue
 		}
 		for k, v := range peerLock.Locks {
 			for _, lockReqInfo := range v {
-				if val, ok := entryMap[lockReqInfo.Name]; ok {
+				if val, ok := entryMap[toEntry(lockReqInfo)]; ok {
 					val.ServerList = append(val.ServerList, peerLock.Addr)
 				} else {
-					entryMap[lockReqInfo.Name] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr)
+					entryMap[toEntry(lockReqInfo)] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr)
 				}
 			}
 		}
@@ -803,6 +806,8 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 	for {
 		select {
 		case <-ctx.Done():
+			globalProfilerMu.Lock()
+			defer globalProfilerMu.Unlock()
 			for k, v := range globalProfiler {
 				v.Stop()
 				delete(globalProfiler, k)
@@ -1013,7 +1018,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					if hr.errBody == "" {
 						errorRespJSON = encodeResponseJSON(getAPIErrorResponse(ctx, hr.apiErr,
 							r.URL.Path, w.Header().Get(xhttp.AmzRequestID),
-							globalDeploymentID))
+							w.Header().Get(xhttp.AmzRequestHostID)))
 					} else {
 						errorRespJSON = encodeResponseJSON(APIErrorResponse{
 							Code:      hr.apiErr.Code,
@@ -1217,6 +1222,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 	storageClass := strings.TrimSpace(r.Form.Get(peerRESTStorageClass))
 	customBucket := strings.TrimSpace(r.Form.Get(peerRESTBucket))
 	autotune := r.Form.Get("autotune") == "true"
+	noClear := r.Form.Get("noclear") == "true"
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1258,14 +1264,16 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 			return
 		}
 
-		if !bucketExists {
+		if !noClear && !bucketExists {
 			defer deleteObjectPerfBucket(objectAPI)
 		}
 	}
 
-	defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{
-		DeletePrefix: true,
-	})
+	if !noClear {
+		defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{
+			DeletePrefix: true,
+		})
+	}
 
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
@@ -1521,6 +1529,11 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Publish bootstrap events that have already occurred before client could subscribe.
+	if traceOpts.TraceTypes().Contains(madmin.TraceBootstrap) {
+		go globalBootstrapTracer.Publish(ctx, globalTrace)
+	}
+
 	for _, peer := range peers {
 		if peer == nil {
 			continue
@@ -1776,12 +1789,48 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, resp)
 }
 
-func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
+func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]madmin.ErasureSetInfo, error) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+
+	z, _ := objectAPI.(*erasureServerPools)
+
+	poolsInfo := make(map[int]map[int]madmin.ErasureSetInfo)
+	for _, d := range allDisks {
+		poolInfo, ok := poolsInfo[d.PoolIndex]
+		if !ok {
+			poolInfo = make(map[int]madmin.ErasureSetInfo)
+		}
+		erasureSet, ok := poolInfo[d.SetIndex]
+		if !ok {
+			erasureSet.ID = d.SetIndex
+			cache := dataUsageCache{}
+			if err := cache.load(ctx, z.serverPools[d.PoolIndex].sets[d.SetIndex], dataUsageCacheName); err == nil {
+				dataUsageInfo := cache.dui(dataUsageRoot, nil)
+				erasureSet.ObjectsCount = dataUsageInfo.ObjectsTotalCount
+				erasureSet.VersionsCount = dataUsageInfo.VersionsTotalCount
+				erasureSet.Usage = dataUsageInfo.ObjectsTotalSize
+			}
+		}
+		erasureSet.RawCapacity += d.TotalSpace
+		erasureSet.RawUsage += d.UsedSpace
+		if d.Healing {
+			erasureSet.HealDisks = 1
+		}
+		poolInfo[d.SetIndex] = erasureSet
+		poolsInfo[d.PoolIndex] = poolInfo
+	}
+	return poolsInfo, nil
+}
+
+func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request) madmin.InfoMessage {
 	kmsStat := fetchKMSStatus()
 
 	ldap := madmin.LDAP{}
-	if globalLDAPConfig.Enabled() {
-		ldapConn, err := globalLDAPConfig.LDAP.Connect()
+	if globalIAMSys.LDAPConfig.Enabled() {
+		ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
 		//nolint:gocritic
 		if err != nil {
 			ldap.Status = string(madmin.ItemOffline)
@@ -1805,7 +1854,9 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 
 	assignPoolNumbers(servers)
 
+	var poolsInfo map[int]map[int]madmin.ErasureSetInfo
 	var backend interface{}
+
 	mode := madmin.ItemInitializing
 
 	buckets := madmin.Buckets{}
@@ -1832,25 +1883,23 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 
 		// Fetching the backend information
 		backendInfo := objectAPI.BackendInfo()
-		if backendInfo.Type == madmin.Erasure {
-			// Calculate the number of online/offline disks of all nodes
-			var allDisks []madmin.Disk
-			for _, s := range servers {
-				allDisks = append(allDisks, s.Disks...)
-			}
-			onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
+		// Calculate the number of online/offline disks of all nodes
+		var allDisks []madmin.Disk
+		for _, s := range servers {
+			allDisks = append(allDisks, s.Disks...)
+		}
+		onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
 
-			backend = madmin.ErasureBackend{
-				Type:             madmin.ErasureType,
-				OnlineDisks:      onlineDisks.Sum(),
-				OfflineDisks:     offlineDisks.Sum(),
-				StandardSCParity: backendInfo.StandardSCParity,
-				RRSCParity:       backendInfo.RRSCParity,
-			}
-		} else {
-			backend = madmin.FSBackend{
-				Type: madmin.FsType,
-			}
+		backend = madmin.ErasureBackend{
+			Type:             madmin.ErasureType,
+			OnlineDisks:      onlineDisks.Sum(),
+			OfflineDisks:     offlineDisks.Sum(),
+			StandardSCParity: backendInfo.StandardSCParity,
+			RRSCParity:       backendInfo.RRSCParity,
+		}
+
+		if poolsInfoEnabled {
+			poolsInfo, _ = getPoolsInfo(ctx, allDisks)
 		}
 	}
 
@@ -1876,6 +1925,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 		Services:     services,
 		Backend:      backend,
 		Servers:      servers,
+		Pools:        poolsInfo,
 	}
 }
 
@@ -2182,7 +2232,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		getAndWriteSysConfig()
 
 		if query.Get("minioinfo") == "true" {
-			infoMessage := getServerInfo(healthCtx, nil)
+			infoMessage := getServerInfo(healthCtx, false, nil)
 			servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
 			for _, server := range infoMessage.Servers {
 				anonEndpoint := anonAddr(server.Endpoint)
@@ -2262,7 +2312,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	errResp := func(err error) {
 		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
-			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+			w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 		encodedErrorResponse := encodeResponse(errorResponse)
 		healthInfo.Error = string(encodedErrorResponse)
 		logger.LogIf(ctx, enc.Encode(healthInfo))
@@ -2357,7 +2407,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	// Marshal API response
-	jsonBytes, err := json.Marshal(getServerInfo(ctx, r))
+	jsonBytes, err := json.Marshal(getServerInfo(ctx, true, r))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -2387,7 +2437,7 @@ func assignPoolNumbers(servers []madmin.ServerProperties) {
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
-	for _, tgt := range globalConfigTargetList.Targets() {
+	for _, tgt := range globalNotifyTargetList.Targets() {
 		targetIDStatus := make(map[string]madmin.Status)
 		active, _ := tgt.IsActive()
 		targetID := tgt.ID()

cmd/admin-handlers_test.go

@@ -21,10 +21,12 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"sort"
 	"sync"
 	"testing"
 	"time"
@@ -380,3 +382,146 @@ func TestExtractHealInitParams(t *testing.T) {
 		}
 	}
 }
+
+type byResourceUID struct{ madmin.LockEntries }
+
+func (b byResourceUID) Less(i, j int) bool {
+	toUniqLock := func(entry madmin.LockEntry) string {
+		return fmt.Sprintf("%s/%s", entry.Resource, entry.ID)
+	}
+	return toUniqLock(b.LockEntries[i]) < toUniqLock(b.LockEntries[j])
+}
+
+func TestTopLockEntries(t *testing.T) {
+	locksHeld := make(map[string][]lockRequesterInfo)
+	var owners []string
+	for i := 0; i < 4; i++ {
+		owners = append(owners, fmt.Sprintf("node-%d", i))
+	}
+
+	// Simulate DeleteObjects of 10 objects in a single request. i.e same lock
+	// request UID, but 10 different resource names associated with it.
+	var lris []lockRequesterInfo
+	uuid := mustGetUUID()
+	for i := 0; i < 10; i++ {
+		resource := fmt.Sprintf("bucket/delete-object-%d", i)
+		lri := lockRequesterInfo{
+			Name:   resource,
+			Writer: true,
+			UID:    uuid,
+			Owner:  owners[i%len(owners)],
+			Group:  true,
+			Quorum: 3,
+		}
+		lris = append(lris, lri)
+		locksHeld[resource] = []lockRequesterInfo{lri}
+	}
+
+	// Add a few concurrent read locks to the mix
+	for i := 0; i < 50; i++ {
+		resource := fmt.Sprintf("bucket/get-object-%d", i)
+		lri := lockRequesterInfo{
+			Name:   resource,
+			UID:    mustGetUUID(),
+			Owner:  owners[i%len(owners)],
+			Quorum: 2,
+		}
+		lris = append(lris, lri)
+		locksHeld[resource] = append(locksHeld[resource], lri)
+		// concurrent read lock, same resource different uid
+		lri.UID = mustGetUUID()
+		lris = append(lris, lri)
+		locksHeld[resource] = append(locksHeld[resource], lri)
+	}
+
+	var peerLocks []*PeerLocks
+	for _, owner := range owners {
+		peerLocks = append(peerLocks, &PeerLocks{
+			Addr:  owner,
+			Locks: locksHeld,
+		})
+	}
+	var exp madmin.LockEntries
+	for _, lri := range lris {
+		lockType := func(lri lockRequesterInfo) string {
+			if lri.Writer {
+				return "WRITE"
+			}
+			return "READ"
+		}
+		exp = append(exp, madmin.LockEntry{
+			Resource:   lri.Name,
+			Type:       lockType(lri),
+			ServerList: owners,
+			Owner:      lri.Owner,
+			ID:         lri.UID,
+			Quorum:     lri.Quorum,
+		})
+	}
+
+	testCases := []struct {
+		peerLocks []*PeerLocks
+		expected  madmin.LockEntries
+	}{
+		{
+			peerLocks: peerLocks,
+			expected:  exp,
+		},
+	}
+
+	// printEntries := func(entries madmin.LockEntries) {
+	// 	for i, entry := range entries {
+	// 		fmt.Printf("%d: %s %s %s %s %v %d\n", i, entry.Resource, entry.ID, entry.Owner, entry.Type, entry.ServerList, entry.Elapsed)
+	// 	}
+	// }
+
+	check := func(exp, got madmin.LockEntries) (int, bool) {
+		if len(exp) != len(got) {
+			return 0, false
+		}
+		sort.Sort(byResourceUID{exp})
+		sort.Sort(byResourceUID{got})
+		// printEntries(exp)
+		// printEntries(got)
+		for i, e := range exp {
+			if !e.Timestamp.Equal(got[i].Timestamp) {
+				return i, false
+			}
+			// Skip checking elapsed since it's time sensitive.
+			// if e.Elapsed != got[i].Elapsed {
+			// 	return false
+			// }
+			if e.Resource != got[i].Resource {
+				return i, false
+			}
+			if e.Type != got[i].Type {
+				return i, false
+			}
+			if e.Source != got[i].Source {
+				return i, false
+			}
+			if e.Owner != got[i].Owner {
+				return i, false
+			}
+			if e.ID != got[i].ID {
+				return i, false
+			}
+			if len(e.ServerList) != len(got[i].ServerList) {
+				return i, false
+			}
+			for j := range e.ServerList {
+				if e.ServerList[j] != got[i].ServerList[j] {
+					return i, false
+				}
+			}
+		}
+		return 0, true
+	}
+
+	for i, tc := range testCases {
+		got := topLockEntries(tc.peerLocks, false)
+		if idx, ok := check(tc.expected, got); !ok {
+			t.Fatalf("%d: mismatch at %d \n expected %#v but got %#v", i, idx, tc.expected[idx], got[idx])
+		}
+	}
+}

cmd/admin-heal-ops.go

@@ -407,9 +407,6 @@ type healSequence struct {
 	// bucket, and object on which heal seq. was initiated
 	bucket, object string
 
-	// A channel of entities with heal result
-	respCh chan healResult
-
 	// Report healing progress
 	reportProgress bool
 
@@ -472,7 +469,6 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 	clientToken := mustGetUUID()
 
 	return &healSequence{
-		respCh:         make(chan healResult),
 		bucket:         bucket,
 		object:         objPrefix,
 		reportProgress: true,
@@ -719,28 +715,30 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 			if serverDebugLog {
 				logger.Info("Task in the queue: %#v", task)
 			}
-		case <-h.ctx.Done():
-			return nil
 		default:
 			// task queue is full, no more workers, we shall move on and heal later.
 			return nil
 		}
-	} else {
-		// respCh must be set for guaranteed result
-		task.respCh = h.respCh
-		select {
-		case globalBackgroundHealRoutine.tasks <- task:
-			if serverDebugLog {
-				logger.Info("Task in the queue: %#v", task)
-			}
-		case <-h.ctx.Done():
-			return nil
+		// Don't wait for result
+		return nil
+	}
+
+	// respCh must be set to wait for result.
+	// We make it size 1, so a result can always be written
+	// even if we aren't listening.
+	task.respCh = make(chan healResult, 1)
+	select {
+	case globalBackgroundHealRoutine.tasks <- task:
+		if serverDebugLog {
+			logger.Info("Task in the queue: %#v", task)
 		}
+	case <-h.ctx.Done():
+		return nil
 	}
 
 	// task queued, now wait for the response.
 	select {
-	case res := <-h.respCh:
+	case res := <-task.respCh:
 		if !h.reportProgress {
 			if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal
 				return nil

cmd/admin-router.go

@@ -237,6 +237,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
 			gz(httpTraceHdrs(adminAPI.DescribeBatchJob)))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/cancel-job").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.CancelBatchJob)))
 
 		// Bucket migration operations
 		// ExportBucketMetaHandler

cmd/admin-server-info.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -118,7 +119,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		config.EnvRootUser:          {},
 		config.EnvRootPassword:      {},
 		config.EnvMinIOSubnetAPIKey: {},
-		config.EnvKMSSecretKey:      {},
+		kms.EnvKMSSecretKey:         {},
 	}
 	for _, v := range os.Environ() {
 		if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") {

cmd/api-errors.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -38,10 +38,12 @@ import (
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/versioning"
+	levent "github.com/minio/minio/internal/config/lambda/event"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/pkg/bucket/policy"
@@ -132,6 +134,7 @@ const (
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
 	ErrReplicationDenyEditError
+	ErrRemoteTargetDenyEditError
 	ErrReplicationNoExistingObjects
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
@@ -164,7 +167,6 @@ const (
 	ErrMalformedDate
 	ErrMalformedPresignedDate
 	ErrMalformedCredentialDate
-	ErrMalformedCredentialRegion
 	ErrMalformedExpires
 	ErrNegativeExpires
 	ErrAuthHeaderEmpty
@@ -219,6 +221,7 @@ const (
 	ErrIncompatibleEncryptionMethod
 	ErrKMSNotConfigured
 	ErrKMSKeyNotFoundException
+	ErrKMSDefaultKeyAlreadyConfigured
 
 	ErrNoAccessKey
 	ErrInvalidToken
@@ -242,8 +245,6 @@ const (
 	// Add new extended error codes here.
 
 	// MinIO extended errors.
-	ErrReadQuorum
-	ErrWriteQuorum
 	ErrStorageFull
 	ErrRequestBodyParse
 	ErrObjectExistsAsDirectory
@@ -267,6 +268,7 @@ const (
 	ErrAdminNoSuchUser
 	ErrAdminNoSuchGroup
 	ErrAdminGroupNotEmpty
+	ErrAdminGroupDisabled
 	ErrAdminNoSuchJob
 	ErrAdminNoSuchPolicy
 	ErrAdminPolicyChangeAlreadyApplied
@@ -280,6 +282,7 @@ const (
 	ErrAdminConfigEnvOverridden
 	ErrAdminConfigDuplicateKeys
 	ErrAdminConfigInvalidIDPType
+	ErrAdminConfigLDAPNonDefaultConfigName
 	ErrAdminConfigLDAPValidation
 	ErrAdminConfigIDPCfgNameAlreadyExists
 	ErrAdminConfigIDPCfgNameDoesNotExist
@@ -410,6 +413,12 @@ const (
 	ErrPostPolicyConditionInvalidFormat
 
 	ErrInvalidChecksum
+
+	// Lambda functions
+	ErrLambdaARNInvalid
+	ErrLambdaARNNotFound
+
+	apiErrCodeEnd // This is used only for the testing code
 )
 
 type errorCodeMap map[APIErrorCode]APIError
@@ -423,8 +432,7 @@ func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError
 		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
 	}
 	if globalSite.Region != "" {
-		switch errCode {
-		case ErrAuthorizationHeaderMalformed:
+		if errCode == ErrAuthorizationHeaderMalformed {
 			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
 			return apiErr
 		}
@@ -919,6 +927,11 @@ var errorCodes = errorCodeMap{
 		Description:    "No matching ExistingsObjects rule enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrRemoteTargetDenyEditError: {
+		Code:           "XMinioAdminRemoteTargetDenyEdit",
+		Description:    "Cannot alter remote target endpoint since this server is in a cluster replication setup. use `mc admin replicate update`",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrReplicationDenyEditError: {
 		Code:           "XMinioReplicationDenyEdit",
 		Description:    "Cannot alter local replication config since this server is in a cluster replication setup",
@@ -1166,6 +1179,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid keyId",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrKMSDefaultKeyAlreadyConfigured: {
+		Code:           "KMS.DefaultKeyAlreadyConfiguredException",
+		Description:    "A default encryption already exists and cannot be changed on KMS",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrNoAccessKey: {
 		Code:           "AccessDenied",
 		Description:    "No AWSAccessKey was presented",
@@ -1250,6 +1268,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The specified group is not empty - cannot remove it.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminGroupDisabled: {
+		Code:           "XMinioAdminGroupDisabled",
+		Description:    "The specified group is disabled.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminNoSuchPolicy: {
 		Code:           "XMinioAdminNoSuchPolicy",
 		Description:    "The canned policy does not exist.",
@@ -1312,6 +1335,11 @@ var errorCodes = errorCodeMap{
 		Description:    fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigLDAPNonDefaultConfigName: {
+		Code:           "XMinioAdminConfigLDAPNonDefaultConfigName",
+		Description:    "Only a single LDAP configuration is supported - config name must be empty or `_`",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigLDAPValidation: {
 		Code:           "XMinioAdminConfigLDAPValidation",
 		Description:    "LDAP Configuration validation failed",
@@ -1509,7 +1537,7 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBusy: {
-		Code:           "Busy",
+		Code:           "ServerBusy",
 		Description:    "The service is unavailable. Please retry.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
@@ -1948,6 +1976,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid checksum provided.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrLambdaARNInvalid: {
+		Code:           "LambdaARNInvalid",
+		Description:    "The specified lambda ARN is invalid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrLambdaARNNotFound: {
+		Code:           "LambdaARNNotFound",
+		Description:    "The specified lambda ARN does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrPolicyAlreadyAttached: {
 		Code:           "XMinioPolicyAlreadyAttached",
 		Description:    "The specified policy is already attached.",
@@ -1971,15 +2009,15 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 
 	// Only return ErrClientDisconnected if the provided context is actually canceled.
 	// This way downstream context.Canceled will still report ErrOperationTimedOut
-	if contextCanceled(ctx) {
-		if ctx.Err() == context.Canceled {
-			return ErrClientDisconnected
-		}
+	if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
+		return ErrClientDisconnected
 	}
 
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
+	case errNoSuchPolicy:
+		apiErr = ErrAdminNoSuchPolicy
 	case errNoSuchUser:
 		apiErr = ErrAdminNoSuchUser
 	case errNoSuchServiceAccount:
@@ -1990,8 +2028,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminGroupNotEmpty
 	case errNoSuchJob:
 		apiErr = ErrAdminNoSuchJob
-	case errNoSuchPolicy:
-		apiErr = ErrAdminNoSuchPolicy
 	case errNoPolicyToAttachOrDetach:
 		apiErr = ErrAdminPolicyChangeAlreadyApplied
 	case errSignatureMismatch:
@@ -2010,6 +2046,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidSecretKey
 	case errInvalidStorageClass:
 		apiErr = ErrInvalidStorageClass
+	case errErasureReadQuorum:
+		apiErr = ErrSlowDown
+	case errErasureWriteQuorum:
+		apiErr = ErrSlowDown
 	// SSE errors
 	case errInvalidEncryptionParameters:
 		apiErr = ErrInvalidEncryptionParameters
@@ -2041,7 +2081,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case errKMSKeyNotFound:
 		apiErr = ErrKMSKeyNotFoundException
-
+	case errKMSDefaultKeyAlreadyConfigured:
+		apiErr = ErrKMSDefaultKeyAlreadyConfigured
 	case context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
@@ -2059,8 +2100,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	}
 
 	// Compression errors
-	switch err {
-	case errInvalidDecompressedSize:
+	if err == errInvalidDecompressedSize {
 		apiErr = ErrInvalidDecompressedSize
 	}
 
@@ -2071,7 +2111,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 
 	// etcd specific errors, a key is always a bucket for us return
 	// ErrNoSuchBucket in such a case.
-	if err == dns.ErrNoEntriesFound {
+	if errors.Is(err, dns.ErrNoEntriesFound) {
 		return ErrNoSuchBucket
 	}
 
@@ -2186,7 +2226,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrTransitionStorageClassNotFoundError
 	case InvalidObjectState:
 		apiErr = ErrInvalidObjectState
-
+	case PreConditionFailed:
+		apiErr = ErrPreconditionFailed
 	case BucketQuotaExceeded:
 		apiErr = ErrAdminBucketQuotaExceeded
 	case *event.ErrInvalidEventName:
@@ -2195,6 +2236,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrARNNotification
 	case *event.ErrARNNotFound:
 		apiErr = ErrARNNotification
+	case *levent.ErrInvalidARN:
+		apiErr = ErrLambdaARNInvalid
+	case *levent.ErrARNNotFound:
+		apiErr = ErrLambdaARNNotFound
 	case *event.ErrUnknownRegion:
 		apiErr = ErrRegionNotification
 	case *event.ErrInvalidFilterName:
@@ -2258,38 +2303,27 @@ func toAPIError(ctx context.Context, err error) APIError {
 	}
 
 	apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
-	e, ok := err.(dns.ErrInvalidBucketName)
-	if ok {
-		code := toAPIErrorCode(ctx, e)
-		apiErr = errorCodes.ToAPIErrWithErr(code, e)
-	}
-
-	if apiErr.Code == "NotImplemented" {
-		switch e := err.(type) {
-		case NotImplemented:
-			desc := e.Error()
-			if desc == "" {
-				desc = apiErr.Description
-			}
-			apiErr = APIError{
-				Code:           apiErr.Code,
-				Description:    desc,
-				HTTPStatusCode: apiErr.HTTPStatusCode,
-			}
-			return apiErr
+	switch apiErr.Code {
+	case "NotImplemented":
+		desc := fmt.Sprintf("%s (%v)", apiErr.Description, err)
+		apiErr = APIError{
+			Code:           apiErr.Code,
+			Description:    desc,
+			HTTPStatusCode: apiErr.HTTPStatusCode,
 		}
-	}
-
-	if apiErr.Code == "XMinioBackendDown" {
+	case "XMinioBackendDown":
 		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
-		return apiErr
-	}
-
-	if apiErr.Code == "InternalError" {
+	case "InternalError":
 		// If we see an internal error try to interpret
 		// any underlying errors if possible depending on
 		// their internal error types.
 		switch e := err.(type) {
+		case kms.Error:
+			apiErr = APIError{
+				Description:    e.Err.Error(),
+				Code:           e.APICode,
+				HTTPStatusCode: e.HTTPStatusCode,
+			}
 		case batchReplicationJobError:
 			apiErr = APIError(e)
 		case InvalidArgument:
@@ -2300,22 +2334,20 @@ func toAPIError(ctx context.Context, err error) APIError {
 			}
 		case *xml.SyntaxError:
 			apiErr = APIError{
-				Code: "MalformedXML",
-				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description,
-					e.Error()),
+				Code:           "MalformedXML",
+				Description:    fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, e),
 				HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
 			}
 		case url.EscapeError:
 			apiErr = APIError{
-				Code: "XMinioInvalidObjectName",
-				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description,
-					e.Error()),
+				Code:           "XMinioInvalidObjectName",
+				Description:    fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description, e),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
 		case versioning.Error:
 			apiErr = APIError{
 				Code:           "IllegalVersioningConfigurationException",
-				Description:    fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e.Error()),
+				Description:    fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
 		case lifecycle.Error:
@@ -2381,19 +2413,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 			// Add more other SDK related errors here if any in future.
 		default:
 			//nolint:gocritic
-			if errors.Is(err, errMalformedEncoding) {
-				apiErr = APIError{
-					Code:           "BadRequest",
-					Description:    err.Error(),
-					HTTPStatusCode: http.StatusBadRequest,
-				}
-			} else if errors.Is(err, errChunkTooBig) {
-				apiErr = APIError{
-					Code:           "BadRequest",
-					Description:    err.Error(),
-					HTTPStatusCode: http.StatusBadRequest,
-				}
-			} else if errors.Is(err, strconv.ErrRange) {
+			if errors.Is(err, errMalformedEncoding) || errors.Is(err, errChunkTooBig) || errors.Is(err, strconv.ErrRange) {
 				apiErr = APIError{
 					Code:           "BadRequest",
 					Description:    err.Error(),

cmd/api-errors_test.go

@@ -80,3 +80,19 @@ func TestAPIErrCode(t *testing.T) {
 		}
 	}
 }
+
+// Check if an API error is properly defined
+func TestAPIErrCodeDefinition(t *testing.T) {
+	for errAPI := ErrNone + 1; errAPI < apiErrCodeEnd; errAPI++ {
+		errCode, ok := errorCodes[errAPI]
+		if !ok {
+			t.Fatal(errAPI, "error code is not defined in the API error code table")
+		}
+		if errCode.Code == "" {
+			t.Fatal(errAPI, "error code has an empty XML code")
+		}
+		if errCode.HTTPStatusCode == 0 {
+			t.Fatal(errAPI, "error code has a zero HTTP status code")
+		}
+	}
+}

cmd/api-response.go

@@ -35,6 +35,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 	xxml "github.com/minio/xxml"
 )
 
@@ -359,6 +360,7 @@ type Object struct {
 
 	// UserMetadata user-defined metadata
 	UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
+	UserTags     string    `xml:"UserTags,omitempty"`
 }
 
 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -492,7 +494,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 }
 
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
-func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
+func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
 
 	owner := Owner{
@@ -500,11 +502,19 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		DisplayName: "minio",
 	}
 	data := ListVersionsResponse{}
+	var lastObjMetaName string
+	var tagErr, metaErr APIErrorCode = -1, -1
 
 	for _, object := range resp.Objects {
 		if object.Name == "" {
 			continue
 		}
+		// Cache checks for the same object
+		if metadata != nil && lastObjMetaName != object.Name {
+			tagErr = metadata(object.Name, policy.GetObjectTaggingAction)
+			metaErr = metadata(object.Name, policy.GetObjectAction)
+			lastObjMetaName = object.Name
+		}
 		content := ObjectVersion{}
 		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
@@ -517,6 +527,32 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		} else {
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
+		if tagErr == ErrNone {
+			content.UserTags = object.UserTags
+		}
+		if metaErr == ErrNone {
+			content.UserMetadata = &Metadata{}
+			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
+			case crypto.S3:
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+			case crypto.S3KMS:
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
+			case crypto.SSEC:
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
+			}
+			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+					// Do not need to send any internal metadata
+					// values to client.
+					continue
+				}
+				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+					continue
+				}
+				content.UserMetadata.Set(k, v)
+			}
+		}
 		content.Owner = owner
 		content.VersionID = object.VersionID
 		if content.VersionID == "" {
@@ -600,7 +636,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata metaCheckFn) ListObjectsV2Response {
 	contents := make([]Object, 0, len(objects))
 	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
@@ -625,27 +661,32 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
 		content.Owner = owner
-		if metadata {
-			content.UserMetadata = &Metadata{}
-			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
-			case crypto.S3:
-				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
-			case crypto.S3KMS:
-				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
-			case crypto.SSEC:
-				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
+		if metadata != nil {
+			if metadata(object.Name, policy.GetObjectTaggingAction) == ErrNone {
+				content.UserTags = object.UserTags
 			}
-			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
-				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
-					// Do not need to send any internal metadata
-					// values to client.
-					continue
+			if metadata(object.Name, policy.GetObjectAction) == ErrNone {
+				content.UserMetadata = &Metadata{}
+				switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
+				case crypto.S3:
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+				case crypto.S3KMS:
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
+				case crypto.SSEC:
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 				}
-				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
-					continue
+				for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+					if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+						// Do not need to send any internal metadata
+						// values to client.
+						continue
+					}
+					// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+					if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+						continue
+					}
+					content.UserMetadata.Set(k, v)
 				}
-				content.UserMetadata.Set(k, v)
 			}
 		}
 		contents = append(contents, content)
@@ -673,6 +714,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	return data
 }
 
+type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
+
 // generates CopyObjectResponse from etag and lastModified time.
 func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
 	return CopyObjectResponse{
@@ -875,7 +918,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 
 	// Generate error response.
 	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
-		w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+		w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 	encodedErrorResponse := encodeResponse(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
@@ -893,7 +936,7 @@ func writeErrorResponseString(ctx context.Context, w http.ResponseWriter, err AP
 // useful for admin APIs.
 func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }

cmd/api-router.go

@@ -285,7 +285,10 @@ func registerAPIRouter(router *mux.Router) {
 		// GetObjectLegalHold
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
-		// GetObject - note gzip compression is *not* added due to Range requests.
+		// GetObject with lambda ARNs
+		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}")
+		// GetObject
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
 		// CopyObject
@@ -389,6 +392,9 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
 		// ListObjectVersions
+		router.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true")
+		// ListObjectVersions
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
 		// GetBucketPolicyStatus
@@ -431,8 +437,9 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodHead).HandlerFunc(
 			collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler)))))
 		// PostPolicy
-		router.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
-			collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
+		router.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
+			return isRequestPostPolicySignatureV4(r)
+		}).HandlerFunc(collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
 		// DeleteMultipleObjects
 		router.Methods(http.MethodPost).HandlerFunc(
 			collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "")
@@ -516,7 +523,11 @@ func corsHandler(handler http.Handler) http.Handler {
 
 	return cors.New(cors.Options{
 		AllowOriginFunc: func(origin string) bool {
-			for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
+			allowedOrigins := globalAPIConfig.getCorsAllowOrigins()
+			if len(allowedOrigins) == 0 {
+				allowedOrigins = []string{"*"}
+			}
+			for _, allowedOrigin := range allowedOrigins {
 				if wildcard.MatchSimple(allowedOrigin, origin) {
 					return true
 				}

cmd/api-utils.go

@@ -94,14 +94,8 @@ func s3URLEncode(s string) string {
 }
 
 // s3EncodeName encodes string in response when encodingType is specified in AWS S3 requests.
-func s3EncodeName(name string, encodingType string) (result string) {
-	// Quick path to exit
-	if encodingType == "" {
-		return name
-	}
-	encodingType = strings.ToLower(encodingType)
-	switch encodingType {
-	case "url":
+func s3EncodeName(name, encodingType string) string {
+	if strings.ToLower(encodingType) == "url" {
 		return s3URLEncode(name)
 	}
 	return name

cmd/auth-handler.go

@@ -25,6 +25,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"io"
+	"mime"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -74,8 +75,11 @@ func isRequestPresignedSignatureV2(r *http.Request) bool {
 
 // Verify if request has AWS Post policy Signature Version '4'.
 func isRequestPostPolicySignatureV4(r *http.Request) bool {
-	return strings.Contains(r.Header.Get(xhttp.ContentType), "multipart/form-data") &&
-		r.Method == http.MethodPost
+	mediaType, _, err := mime.ParseMediaType(r.Header.Get(xhttp.ContentType))
+	if err != nil {
+		return false
+	}
+	return mediaType == "multipart/form-data" && r.Method == http.MethodPost
 }
 
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
@@ -154,6 +158,10 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 		return cred, owner, s3Err
 	}
 
+	logger.GetReqInfo(ctx).Cred = cred
+	logger.GetReqInfo(ctx).Owner = owner
+	logger.GetReqInfo(ctx).Region = globalSite.Region
+
 	return cred, owner, ErrNone
 }
 
@@ -203,7 +211,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 	// that clients cannot decode the token using the temp
 	// secret keys and generate an entirely new claim by essentially
 	// hijacking the policies. We need to make sure that this is
-	// based an admin credential such that token cannot be decoded
+	// based on admin credential such that token cannot be decoded
 	// on the client side and is treated like an opaque value.
 	claims, err := auth.ExtractClaims(token, secret)
 	if err != nil {
@@ -253,7 +261,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		return nil, ErrNoAccessKey
 	}
 
-	if token == "" && cred.IsTemp() {
+	if token == "" && cred.IsTemp() && !cred.IsServiceAccount() {
 		// Temporary credentials should always have x-amz-security-token
 		return nil, ErrInvalidToken
 	}
@@ -263,7 +271,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		return nil, ErrInvalidToken
 	}
 
-	if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+	if !cred.IsServiceAccount() && cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
 		// validate token for temporary credentials only.
 		return nil, ErrInvalidToken
 	}
@@ -344,6 +352,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
 
 	logger.GetReqInfo(ctx).Cred = cred
 	logger.GetReqInfo(ctx).Owner = owner
+	logger.GetReqInfo(ctx).Region = globalSite.Region
 
 	// region is valid only for CreateBucketAction.
 	var region string
@@ -585,6 +594,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 				// All our internal APIs are sensitive towards Date
 				// header, for all requests where Date header is not
 				// present we will reject such clients.
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -598,6 +608,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = true
 				}
 
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -613,6 +624,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 			tc.ResponseRecorder.LogErrBody = true
 		}
 
+		defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
 		atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1)
 	})

cmd/background-heal-ops.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"runtime"
 	"strconv"
+	"time"
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/logger"
@@ -60,13 +61,43 @@ func activeListeners() int {
 	return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers())
 }
 
-func waitForLowHTTPReq() {
-	var currentIO func() int
-	if httpServer := newHTTPServerFn(); httpServer != nil {
-		currentIO = httpServer.GetRequestCount
+func waitForLowIO(maxIO int, maxWait time.Duration, currentIO func() int) {
+	// No need to wait run at full speed.
+	if maxIO <= 0 {
+		return
 	}
 
-	globalHealConfig.Wait(currentIO, activeListeners)
+	const waitTick = 100 * time.Millisecond
+
+	tmpMaxWait := maxWait
+
+	for currentIO() >= maxIO {
+		if tmpMaxWait > 0 {
+			if tmpMaxWait < waitTick {
+				time.Sleep(tmpMaxWait)
+			} else {
+				time.Sleep(waitTick)
+			}
+			tmpMaxWait -= waitTick
+		}
+		if tmpMaxWait <= 0 {
+			return
+		}
+	}
+}
+
+func currentHTTPIO() int {
+	httpServer := newHTTPServerFn()
+	if httpServer == nil {
+		return 0
+	}
+
+	return httpServer.GetRequestCount() - activeListeners()
+}
+
+func waitForLowHTTPReq() {
+	maxIO, maxWait, _ := globalHealConfig.Clone()
+	waitForLowIO(maxIO, maxWait, currentHTTPIO)
 }
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {

cmd/background-newdisks-heal-ops.go

@@ -271,6 +271,9 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
+	globalLocalDrivesMu.RLock()
+	globalLocalDrives := globalLocalDrives
+	globalLocalDrivesMu.RUnlock()
 	for _, disk := range globalLocalDrives {
 		_, err := disk.GetDiskID()
 		if errors.Is(err, errUnformattedDisk) {
@@ -299,7 +302,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	defer disk.Close()
 	poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
 	if poolIdx < 0 {
-		return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
+		return fmt.Errorf("unexpected pool index (%d) found for %s", poolIdx, disk.Endpoint())
 	}
 
 	// Calculate the set index where the current endpoint belongs
@@ -310,14 +313,15 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return err
 	}
 	if setIdx < 0 {
-		return fmt.Errorf("unexpected set index (%d) found in %s", setIdx, disk.Endpoint())
+		return fmt.Errorf("unexpected set index (%d) found for  %s", setIdx, disk.Endpoint())
 	}
 
 	// Prevent parallel erasure set healing
 	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
 	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
 	if err != nil {
-		return err
+		return fmt.Errorf("Healing of drive '%v' on %s pool, belonging to %s erasure set already in progress: %w",
+			disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1), err)
 	}
 	ctx = lkctx.Context()
 	defer locker.Unlock(lkctx)
@@ -325,19 +329,20 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	// Load healing tracker in this disk
 	tracker, err := loadHealingTracker(ctx, disk)
 	if err != nil {
-		// A healing track can be not found when another disk in the same
-		// erasure set and same healing-id successfully finished healing.
-		if err == errFileNotFound {
+		// A healing tracker may be deleted if another disk in the
+		// same erasure set with same healing-id successfully finished
+		// healing.
+		if errors.Is(err, errFileNotFound) {
 			return nil
 		}
-		logger.LogIf(ctx, fmt.Errorf("Unable to load a healing tracker on '%s': %w", disk, err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
 		tracker = newHealingTracker(disk, mustGetUUID())
 	}
 
-	logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
+	logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
 
 	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
-	// Buckets data are dispersed in multiple zones/sets, make
+	// Buckets data are dispersed in multiple pools/sets, make
 	// sure to heal all bucket metadata configuration.
 	buckets = append(buckets, BucketInfo{
 		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
@@ -355,7 +360,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	})
 
 	if serverDebugLog {
-		logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
+		logger.Info("Healing drive '%v' on %s pool, belonging to %s erasure set", disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1))
 	}
 
 	// Load bucket totals
@@ -378,9 +383,13 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	if tracker.ItemsFailed > 0 {
-		logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing of drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
 	} else {
-		logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing of drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	}
+
+	if len(tracker.QueuedBuckets) > 0 {
+		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
 	}
 
 	if serverDebugLog {
@@ -388,7 +397,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		logger.Info("\n")
 	}
 
-	if tracker.HealID == "" { // HealID is empty only before Feb 2023
+	if tracker.HealID == "" { // HealID was empty only before Feb 2023
 		logger.LogIf(ctx, tracker.delete(ctx))
 		return nil
 	}
@@ -397,7 +406,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	for _, disk := range z.serverPools[poolIdx].sets[setIdx].getDisks() {
 		t, err := loadHealingTracker(ctx, disk)
 		if err != nil {
-			if err != errFileNotFound {
+			if !errors.Is(err, errFileNotFound) {
 				logger.LogIf(ctx, err)
 			}
 			continue
@@ -442,10 +451,12 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 			for _, disk := range healDisks {
 				go func(disk Endpoint) {
 					globalBackgroundHealState.setDiskHealingStatus(disk, true)
-					err := healFreshDisk(ctx, z, disk)
-					if err != nil {
+					if err := healFreshDisk(ctx, z, disk); err != nil {
 						globalBackgroundHealState.setDiskHealingStatus(disk, false)
-						printEndpointError(disk, err, false)
+						timedout := OperationTimedOut{}
+						if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) {
+							printEndpointError(disk, err, false)
+						}
 						return
 					}
 					// Only upon success pop the healed disk.

cmd/batch-handlers_gen.go

@@ -1538,6 +1538,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "KeyRotate":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "KeyRotate")
+					return
+				}
+				z.KeyRotate = nil
+			} else {
+				if z.KeyRotate == nil {
+					z.KeyRotate = new(BatchJobKeyRotateV1)
+				}
+				err = z.KeyRotate.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "KeyRotate")
+					return
+				}
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -1551,9 +1569,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 5
+	// map header, size 6
 	// write "ID"
-	err = en.Append(0x85, 0xa2, 0x49, 0x44)
+	err = en.Append(0x86, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -1609,15 +1627,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "KeyRotate"
+	err = en.Append(0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65)
+	if err != nil {
+		return
+	}
+	if z.KeyRotate == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.KeyRotate.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "KeyRotate")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
+	// map header, size 6
 	// string "ID"
-	o = append(o, 0x85, 0xa2, 0x49, 0x44)
+	o = append(o, 0x86, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -1639,6 +1674,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 			return
 		}
 	}
+	// string "KeyRotate"
+	o = append(o, 0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65)
+	if z.KeyRotate == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.KeyRotate.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "KeyRotate")
+			return
+		}
+	}
 	return
 }
 
@@ -1701,6 +1747,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "KeyRotate":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.KeyRotate = nil
+			} else {
+				if z.KeyRotate == nil {
+					z.KeyRotate = new(BatchJobKeyRotateV1)
+				}
+				bts, err = z.KeyRotate.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "KeyRotate")
+					return
+				}
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -1721,6 +1784,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
 	} else {
 		s += z.Replicate.Msgsize()
 	}
+	s += 10
+	if z.KeyRotate == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.KeyRotate.Msgsize()
+	}
 	return
 }
 

cmd/batch-rotate.go

@@ -0,0 +1,564 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/workers"
+	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/wildcard"
+)
+
+// keyrotate:
+//   apiVersion: v1
+//   bucket: BUCKET
+//   prefix: PREFIX
+//   encryption:
+//     type: sse-s3 # valid values are sse-s3 and sse-kms
+//     key: <new-kms-key> # valid only for sse-kms
+//     context: <new-kms-key-context> # valid only for sse-kms
+// # optional flags based filtering criteria
+// # for all objects
+// flags:
+//   filter:
+//     newerThan: "7d" # match objects newer than this value (e.g. 7d10h31s)
+//     olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+//     createdAfter: "date" # match objects created after "date"
+//     createdBefore: "date" # match objects created before "date"
+//     tags:
+//       - key: "name"
+//         value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
+//     metadata:
+//       - key: "content-type"
+//         value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
+//     kmskey: "key-id" # match objects with KMS key-id (applicable only for sse-kms)
+//   notify:
+//     endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
+//     token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+
+//   retry:
+//     attempts: 10 # number of retries for the job before giving up
+//     delay: "500ms" # least amount of delay between each retry
+
+//go:generate msgp -file $GOFILE -unexported
+
+// BatchKeyRotateKV is a datatype that holds key and values for filtering of objects
+// used by metadata filter as well as tags based filtering.
+type BatchKeyRotateKV struct {
+	Key   string `yaml:"key" json:"key"`
+	Value string `yaml:"value" json:"value"`
+}
+
+// Validate returns an error if key is empty
+func (kv BatchKeyRotateKV) Validate() error {
+	if kv.Key == "" {
+		return errInvalidArgument
+	}
+	return nil
+}
+
+// Empty indicates if kv is not set
+func (kv BatchKeyRotateKV) Empty() bool {
+	return kv.Key == "" && kv.Value == ""
+}
+
+// Match matches input kv with kv, value will be wildcard matched depending on the user input
+func (kv BatchKeyRotateKV) Match(ikv BatchKeyRotateKV) bool {
+	if kv.Empty() {
+		return true
+	}
+	if strings.EqualFold(kv.Key, ikv.Key) {
+		return wildcard.Match(kv.Value, ikv.Value)
+	}
+	return false
+}
+
+// BatchKeyRotateRetry datatype represents total retry attempts and delay between each retries.
+type BatchKeyRotateRetry struct {
+	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
+	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
+}
+
+// Validate validates input replicate retries.
+func (r BatchKeyRotateRetry) Validate() error {
+	if r.Attempts < 0 {
+		return errInvalidArgument
+	}
+
+	if r.Delay < 0 {
+		return errInvalidArgument
+	}
+
+	return nil
+}
+
+// BatchKeyRotationType defines key rotation type
+type BatchKeyRotationType string
+
+const (
+	sses3  BatchKeyRotationType = "sse-s3"
+	ssekms BatchKeyRotationType = "sse-kms"
+)
+
+// BatchJobKeyRotateEncryption defines key rotation encryption options passed
+type BatchJobKeyRotateEncryption struct {
+	Type       BatchKeyRotationType `yaml:"type" json:"type"`
+	Key        string               `yaml:"key" json:"key"`
+	Context    string               `yaml:"context" json:"context"`
+	kmsContext kms.Context          `msg:"-"`
+}
+
+// Validate validates input key rotation encryption options.
+func (e BatchJobKeyRotateEncryption) Validate() error {
+	if e.Type != sses3 && e.Type != ssekms {
+		return errInvalidArgument
+	}
+	spaces := strings.HasPrefix(e.Key, " ") || strings.HasSuffix(e.Key, " ")
+	if e.Type == ssekms && spaces {
+		return crypto.ErrInvalidEncryptionKeyID
+	}
+	if e.Type == ssekms && GlobalKMS != nil {
+		ctx := kms.Context{}
+		if e.Context != "" {
+			b, err := base64.StdEncoding.DecodeString(e.Context)
+			if err != nil {
+				return err
+			}
+
+			json := jsoniter.ConfigCompatibleWithStandardLibrary
+			if err := json.Unmarshal(b, &ctx); err != nil {
+				return err
+			}
+		}
+		e.kmsContext = kms.Context{}
+		for k, v := range ctx {
+			e.kmsContext[k] = v
+		}
+		ctx["MinIO batch API"] = "batchrotate" // Context for a test key operation
+		if _, err := GlobalKMS.GenerateKey(GlobalContext, e.Key, ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// BatchKeyRotateFilter holds all the filters currently supported for batch replication
+type BatchKeyRotateFilter struct {
+	NewerThan     time.Duration      `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     time.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time          `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchKeyRotateKV `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchKeyRotateKV `yaml:"metadata,omitempty" json:"metadata"`
+	KMSKeyID      string             `yaml:"kmskeyid" json:"kmskey"`
+}
+
+// BatchKeyRotateNotification success or failure notification endpoint for each job attempts
+type BatchKeyRotateNotification struct {
+	Endpoint string `yaml:"endpoint" json:"endpoint"`
+	Token    string `yaml:"token" json:"token"`
+}
+
+// BatchJobKeyRotateFlags various configurations for replication job definition currently includes
+// - filter
+// - notify
+// - retry
+type BatchJobKeyRotateFlags struct {
+	Filter BatchKeyRotateFilter       `yaml:"filter" json:"filter"`
+	Notify BatchKeyRotateNotification `yaml:"notify" json:"notify"`
+	Retry  BatchKeyRotateRetry        `yaml:"retry" json:"retry"`
+}
+
+// BatchJobKeyRotateV1 v1 of batch key rotation job
+type BatchJobKeyRotateV1 struct {
+	APIVersion string                      `yaml:"apiVersion" json:"apiVersion"`
+	Flags      BatchJobKeyRotateFlags      `yaml:"flags" json:"flags"`
+	Bucket     string                      `yaml:"bucket" json:"bucket"`
+	Prefix     string                      `yaml:"prefix" json:"prefix"`
+	Endpoint   string                      `yaml:"endpoint" json:"endpoint"`
+	Encryption BatchJobKeyRotateEncryption `yaml:"encryption" json:"encryption"`
+}
+
+// Notify notifies notification endpoint if configured regarding job failure or success.
+func (r BatchJobKeyRotateV1) Notify(ctx context.Context, body io.Reader) error {
+	if r.Flags.Notify.Endpoint == "" {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.Flags.Notify.Endpoint, body)
+	if err != nil {
+		return err
+	}
+
+	if r.Flags.Notify.Token != "" {
+		req.Header.Set("Authorization", r.Flags.Notify.Token)
+	}
+
+	clnt := http.Client{Transport: getRemoteInstanceTransport}
+	resp, err := clnt.Do(req)
+	if err != nil {
+		return err
+	}
+
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(resp.Status)
+	}
+
+	return nil
+}
+
+// KeyRotate rotates encryption key of an object
+func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error {
+	srcBucket := r.Bucket
+	srcObject := objInfo.Name
+
+	if objInfo.DeleteMarker || !objInfo.VersionPurgeStatus.Empty() {
+		return nil
+	}
+	sseKMS := crypto.S3KMS.IsEncrypted(objInfo.UserDefined)
+	sseS3 := crypto.S3.IsEncrypted(objInfo.UserDefined)
+	if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
+		return errInvalidEncryptionParameters
+	}
+	if sseKMS && r.Encryption.Type == sses3 { // previously encrypted with sse-kms, now sse-s3 disallowed
+		return errInvalidEncryptionParameters
+	}
+	versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
+	versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
+
+	lock := api.NewNSLock(r.Bucket, objInfo.Name)
+	lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return err
+	}
+	ctx = lkctx.Context()
+	defer lock.Unlock(lkctx)
+
+	opts := ObjectOptions{
+		VersionID:        objInfo.VersionID,
+		Versioned:        versioned,
+		VersionSuspended: versionSuspended,
+		NoLock:           true,
+	}
+	obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts)
+	if err != nil {
+		return err
+	}
+	oi := obj.Clone()
+	var (
+		newKeyID      string
+		newKeyContext kms.Context
+	)
+	encMetadata := make(map[string]string)
+	for k, v := range oi.UserDefined {
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			encMetadata[k] = v
+		}
+	}
+
+	if (sseKMS || sseS3) && r.Encryption.Type == ssekms {
+		if err = r.Encryption.Validate(); err != nil {
+			return err
+		}
+		newKeyID = strings.TrimPrefix(r.Encryption.Key, crypto.ARNPrefix)
+		newKeyContext = r.Encryption.kmsContext
+	}
+	if err = rotateKey(ctx, []byte{}, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext); err != nil {
+		return err
+	}
+
+	// Since we are rotating the keys, make sure to update the metadata.
+	oi.metadataOnly = true
+	oi.keyRotation = true
+	for k, v := range encMetadata {
+		oi.UserDefined[k] = v
+	}
+	if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{
+		VersionID: oi.VersionID,
+	}, ObjectOptions{
+		VersionID: oi.VersionID,
+		NoLock:    true,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+const (
+	batchKeyRotationName               = "batch-rotate.bin"
+	batchKeyRotationFormat             = 1
+	batchKeyRotateVersionV1            = 1
+	batchKeyRotateVersion              = batchKeyRotateVersionV1
+	batchKeyRotateAPIVersion           = "v1"
+	batchKeyRotateJobDefaultRetries    = 3
+	batchKeyRotateJobDefaultRetryDelay = 250 * time.Millisecond
+)
+
+// Start the batch key rottion job, resumes if there was a pending job via "job.ID"
+func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	ri := &batchJobInfo{
+		JobID:     job.ID,
+		JobType:   string(job.Type()),
+		StartTime: job.Started,
+	}
+	if err := ri.load(ctx, api, job); err != nil {
+		return err
+	}
+
+	globalBatchJobsMetrics.save(job.ID, ri)
+	lastObject := ri.Object
+
+	delay := job.KeyRotate.Flags.Retry.Delay
+	if delay == 0 {
+		delay = batchKeyRotateJobDefaultRetryDelay
+	}
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	skip := func(info FileInfo) (ok bool) {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
+			// skip all objects that are newer than specified older duration
+			return false
+		}
+
+		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan {
+			// skip all objects that are older than specified newer duration
+			return false
+		}
+
+		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(info.ModTime) {
+			// skip all objects that are created before the specified time.
+			return false
+		}
+
+		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(info.ModTime) {
+			// skip all objects that are created after the specified time.
+			return false
+		}
+
+		if len(r.Flags.Filter.Tags) > 0 {
+			// Only parse object tags if tags filter is specified.
+			tagMap := map[string]string{}
+			tagStr := info.Metadata[xhttp.AmzObjectTagging]
+			if len(tagStr) != 0 {
+				t, err := tags.ParseObjectTags(tagStr)
+				if err != nil {
+					return false
+				}
+				tagMap = t.ToMap()
+			}
+
+			for _, kv := range r.Flags.Filter.Tags {
+				for t, v := range tagMap {
+					if kv.Match(BatchKeyRotateKV{Key: t, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided tags filter match skip the object
+			return false
+		}
+
+		if len(r.Flags.Filter.Metadata) > 0 {
+			for _, kv := range r.Flags.Filter.Metadata {
+				for k, v := range info.Metadata {
+					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
+						continue
+					}
+					// We only need to match x-amz-meta or standardHeaders
+					if kv.Match(BatchKeyRotateKV{Key: k, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided metadata filters match skip the object.
+			return false
+		}
+		if r.Flags.Filter.KMSKeyID != "" {
+			if v, ok := info.Metadata[xhttp.AmzServerSideEncryptionKmsID]; ok && strings.TrimPrefix(v, crypto.ARNPrefix) != r.Flags.Filter.KMSKeyID {
+				return false
+			}
+		}
+		return true
+	}
+
+	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_KEYROTATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+	if err != nil {
+		return err
+	}
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		// invalid worker size.
+		return err
+	}
+
+	retryAttempts := ri.RetryAttempts
+	ctx, cancel := context.WithCancel(ctx)
+
+	results := make(chan ObjectInfo, 100)
+	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, ObjectOptions{
+		WalkMarker: lastObject,
+		WalkFilter: skip,
+	}); err != nil {
+		cancel()
+		// Do not need to retry if we can't list objects on source.
+		return err
+	}
+
+	for result := range results {
+		result := result
+		sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined)
+		sseS3 := crypto.S3.IsEncrypted(result.UserDefined)
+		if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
+			continue
+		}
+		wk.Take()
+		go func() {
+			defer wk.Give()
+			for attempts := 1; attempts <= retryAttempts; attempts++ {
+				attempts := attempts
+				stopFn := globalBatchJobsMetrics.trace(batchKeyRotationMetricObject, job.ID, attempts, result)
+				success := true
+				if err := r.KeyRotate(ctx, api, result); err != nil {
+					stopFn(err)
+					logger.LogIf(ctx, err)
+					success = false
+				} else {
+					stopFn(nil)
+				}
+				ri.trackCurrentBucketObject(r.Bucket, result, success)
+				ri.RetryAttempts = attempts
+				globalBatchJobsMetrics.save(job.ID, ri)
+				// persist in-memory state to disk after every 10secs.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				if success {
+					break
+				}
+			}
+		}()
+	}
+	wk.Wait()
+
+	ri.Complete = ri.ObjectsFailed == 0
+	ri.Failed = ri.ObjectsFailed > 0
+	globalBatchJobsMetrics.save(job.ID, ri)
+	// persist in-memory state to disk.
+	logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+
+	buf, _ := json.Marshal(ri)
+	if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+	}
+
+	cancel()
+	if ri.Failed {
+		ri.ObjectsFailed = 0
+		ri.Bucket = ""
+		ri.Object = ""
+		ri.Objects = 0
+		time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+	}
+
+	return nil
+}
+
+//msgp:ignore batchKeyRotationJobError
+type batchKeyRotationJobError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+func (e batchKeyRotationJobError) Error() string {
+	return e.Description
+}
+
+// Validate validates the job definition input
+func (r *BatchJobKeyRotateV1) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
+	if r == nil {
+		return nil
+	}
+
+	if r.APIVersion != batchKeyRotateAPIVersion {
+		return errInvalidArgument
+	}
+
+	if r.Bucket == "" {
+		return errInvalidArgument
+	}
+
+	if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil {
+		if isErrBucketNotFound(err) {
+			return batchKeyRotationJobError{
+				Code:           "NoSuchSourceBucket",
+				Description:    "The specified source bucket does not exist",
+				HTTPStatusCode: http.StatusNotFound,
+			}
+		}
+		return err
+	}
+	if GlobalKMS == nil {
+		return errKMSNotConfigured
+	}
+	if err := r.Encryption.Validate(); err != nil {
+		return err
+	}
+
+	for _, tag := range r.Flags.Filter.Tags {
+		if err := tag.Validate(); err != nil {
+			return err
+		}
+	}
+
+	for _, meta := range r.Flags.Filter.Metadata {
+		if err := meta.Validate(); err != nil {
+			return err
+		}
+	}
+
+	if err := r.Flags.Retry.Validate(); err != nil {
+		return err
+	}
+	return nil
+}

cmd/batch-rotate_gen_test.go

@@ -0,0 +1,801 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBatchJobKeyRotateEncryption(t *testing.T) {
+	v := BatchJobKeyRotateEncryption{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKeyRotateEncryption(t *testing.T) {
+	v := BatchJobKeyRotateEncryption{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateEncryption Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKeyRotateEncryption{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobKeyRotateFlags(t *testing.T) {
+	v := BatchJobKeyRotateFlags{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKeyRotateFlags(t *testing.T) {
+	v := BatchJobKeyRotateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateFlags Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKeyRotateFlags{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobKeyRotateV1(t *testing.T) {
+	v := BatchJobKeyRotateV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKeyRotateV1(t *testing.T) {
+	v := BatchJobKeyRotateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateV1 Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKeyRotateV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateFilter(t *testing.T) {
+	v := BatchKeyRotateFilter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateFilter(t *testing.T) {
+	v := BatchKeyRotateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateFilter Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateFilter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateKV(t *testing.T) {
+	v := BatchKeyRotateKV{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateKV(t *testing.T) {
+	v := BatchKeyRotateKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateKV Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateKV{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateNotification(t *testing.T) {
+	v := BatchKeyRotateNotification{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateNotification(t *testing.T) {
+	v := BatchKeyRotateNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateNotification Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateNotification{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateRetry(t *testing.T) {
+	v := BatchKeyRotateRetry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateRetry(t *testing.T) {
+	v := BatchKeyRotateRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateRetry Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateRetry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batchjobmetric_string.go

@@ -0,0 +1,24 @@
+// Code generated by "stringer -type=batchJobMetric -trimprefix=batchJobMetric batch-handlers.go"; DO NOT EDIT.
+
+package cmd
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[batchReplicationMetricObject-0]
+	_ = x[batchKeyRotationMetricObject-1]
+}
+
+const _batchJobMetric_name = "batchReplicationMetricObjectbatchKeyRotationMetricObject"
+
+var _batchJobMetric_index = [...]uint8{0, 28, 56}
+
+func (i batchJobMetric) String() string {
+	if i >= batchJobMetric(len(_batchJobMetric_index)-1) {
+		return "batchJobMetric(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _batchJobMetric_name[_batchJobMetric_index[i]:_batchJobMetric_index[i+1]]
+}

cmd/batchreplicationmetric_string.go

@@ -1,23 +0,0 @@
-// Code generated by "stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric batch-handlers.go"; DO NOT EDIT.
-
-package cmd
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[batchReplicationMetricObject-0]
-}
-
-const _batchReplicationMetric_name = "Object"
-
-var _batchReplicationMetric_index = [...]uint8{0, 6}
-
-func (i batchReplicationMetric) String() string {
-	if i >= batchReplicationMetric(len(_batchReplicationMetric_index)-1) {
-		return "batchReplicationMetric(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-	return _batchReplicationMetric_name[_batchReplicationMetric_index[i]:_batchReplicationMetric_index[i+1]]
-}

cmd/bootstrap-messages.go

@@ -0,0 +1,116 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio/internal/pubsub"
+)
+
+const bootstrapMsgsLimit = 4 << 10
+
+type bootstrapInfo struct {
+	msg    string
+	ts     time.Time
+	source string
+}
+type bootstrapTracer struct {
+	mu         sync.RWMutex
+	idx        int
+	info       [bootstrapMsgsLimit]bootstrapInfo
+	lastUpdate time.Time
+}
+
+var globalBootstrapTracer = &bootstrapTracer{}
+
+func (bs *bootstrapTracer) DropEvents() {
+	bs.mu.Lock()
+	defer bs.mu.Unlock()
+
+	if time.Now().UTC().Sub(bs.lastUpdate) > 24*time.Hour {
+		bs.info = [4096]bootstrapInfo{}
+		bs.idx = 0
+	}
+}
+
+func (bs *bootstrapTracer) Empty() bool {
+	var empty bool
+	bs.mu.RLock()
+	empty = bs.info[0].msg == ""
+	bs.mu.RUnlock()
+
+	return empty
+}
+
+func (bs *bootstrapTracer) Record(msg string, skip int) {
+	source := getSource(skip + 1)
+	bs.mu.Lock()
+	now := time.Now().UTC()
+	bs.info[bs.idx] = bootstrapInfo{
+		msg:    msg,
+		ts:     now,
+		source: source,
+	}
+	bs.lastUpdate = now
+	bs.idx = (bs.idx + 1) % bootstrapMsgsLimit
+	bs.mu.Unlock()
+}
+
+func (bs *bootstrapTracer) Events() []madmin.TraceInfo {
+	traceInfo := make([]madmin.TraceInfo, 0, bootstrapMsgsLimit)
+
+	// Add all messages in order
+	addAll := func(info []bootstrapInfo) {
+		for _, msg := range info {
+			if msg.ts.IsZero() {
+				continue // skip empty events
+			}
+			traceInfo = append(traceInfo, madmin.TraceInfo{
+				TraceType: madmin.TraceBootstrap,
+				Time:      msg.ts,
+				NodeName:  globalLocalNodeName,
+				FuncName:  "BOOTSTRAP",
+				Message:   fmt.Sprintf("%s %s", msg.source, msg.msg),
+			})
+		}
+	}
+
+	bs.mu.RLock()
+	addAll(bs.info[bs.idx:])
+	addAll(bs.info[:bs.idx])
+	bs.mu.RUnlock()
+	return traceInfo
+}
+
+func (bs *bootstrapTracer) Publish(ctx context.Context, trace *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) {
+	if bs.Empty() {
+		return
+	}
+	for _, bsEvent := range bs.Events() {
+		select {
+		case <-ctx.Done():
+		default:
+			trace.Publish(bsEvent)
+		}
+	}
+}

cmd/bootstrap-messages_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestBootstrap(t *testing.T) {
+	// Bootstrap events exceed bootstrap messages limit
+	bsTracer := &bootstrapTracer{}
+	for i := 0; i < bootstrapMsgsLimit+10; i++ {
+		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
+	}
+
+	traceInfos := bsTracer.Events()
+	if len(traceInfos) != bootstrapMsgsLimit {
+		t.Fatalf("Expected length of events %d but got %d", bootstrapMsgsLimit, len(traceInfos))
+	}
+
+	// Simulate the case where bootstrap events were updated a day ago
+	bsTracer.lastUpdate = time.Now().UTC().Add(-25 * time.Hour)
+	bsTracer.DropEvents()
+	if !bsTracer.Empty() {
+		t.Fatalf("Expected all bootstrap events to have been dropped, but found %d events", len(bsTracer.Events()))
+	}
+
+	// Fewer than 4K bootstrap events
+	for i := 0; i < 10; i++ {
+		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
+	}
+	events := bsTracer.Events()
+	if len(events) != 10 {
+		t.Fatalf("Expected length of events %d but got %d", 10, len(events))
+	}
+	for i, traceInfo := range bsTracer.Events() {
+		msg := fmt.Sprintf("msg-%d", i)
+		if !strings.HasSuffix(traceInfo.Message, msg) {
+			t.Fatalf("Expected %s but got %s", msg, traceInfo.Message)
+		}
+	}
+}

cmd/bootstrap-peer-server.go

@@ -101,10 +101,14 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 }
 
 var skipEnvs = map[string]struct{}{
-	"MINIO_OPTS":         {},
-	"MINIO_CERT_PASSWD":  {},
-	"MINIO_SERVER_DEBUG": {},
-	"MINIO_DSYNC_TRACE":  {},
+	"MINIO_OPTS":          {},
+	"MINIO_CERT_PASSWD":   {},
+	"MINIO_SERVER_DEBUG":  {},
+	"MINIO_DSYNC_TRACE":   {},
+	"MINIO_ROOT_USER":     {},
+	"MINIO_ROOT_PASSWORD": {},
+	"MINIO_ACCESS_KEY":    {},
+	"MINIO_SECRET_KEY":    {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {
@@ -118,7 +122,7 @@ func getServerSystemCfg() ServerSystemConfig {
 		if _, ok := skipEnvs[envK]; ok {
 			continue
 		}
-		envValues[envK] = env.Get(envK, "")
+		envValues[envK] = logger.HashString(env.Get(envK, ""))
 	}
 	return ServerSystemConfig{
 		MinioEndpoints: globalEndpoints,
@@ -126,11 +130,22 @@ func getServerSystemCfg() ServerSystemConfig {
 	}
 }
 
+func (b *bootstrapRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusForbidden)
+	w.Write([]byte(err.Error()))
+}
+
 // HealthHandler returns success if request is valid
 func (b *bootstrapRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {}
 
 func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "VerifyHandler")
+
+	if err := storageServerRequestValidate(r); err != nil {
+		b.writeErrorResponse(w, err)
+		return
+	}
+
 	cfg := getServerSystemCfg()
 	logger.LogIf(ctx, json.NewEncoder(w).Encode(&cfg))
 }

cmd/bucket-encryption-handlers.go

@@ -25,7 +25,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"

cmd/bucket-handlers.go

@@ -755,19 +755,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Parse incoming location constraint.
-	location, s3Error := parseLocationConstraint(r)
+	_, s3Error = parseLocationConstraint(r)
 	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
 
-	// Validate if location sent by the client is valid, reject
-	// requests which do not follow valid region requirements.
-	if !isValidLocation(location) {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRegion), r.URL)
-		return
-	}
-
 	// check if client is attempting to create more buckets, complain about it.
 	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
 		logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
@@ -803,8 +796,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 				// Make sure to add Location information here only for bucket
-				w.Header().Set(xhttp.Location,
-					getObjectLocation(r, globalDomainNames, bucket, ""))
+				w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket))
 
 				writeSuccessResponseHeadersOnly(w)
 
@@ -855,9 +847,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	logger.LogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts))
 
 	// Make sure to add Location information here only for bucket
-	if cp := pathClean(r.URL.Path); cp != "" {
-		w.Header().Set(xhttp.Location, cp) // Clean any trailing slashes.
-	}
+	w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket))
 
 	writeSuccessResponseHeadersOnly(w)
 
@@ -1130,7 +1120,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
 
-	w.Header().Set(xhttp.Location, getObjectLocation(r, globalDomainNames, bucket, object))
+	if obj := getObjectLocation(r, globalDomainNames, bucket, object); obj != "" {
+		w.Header().Set(xhttp.Location, obj)
+	}
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{
@@ -1142,6 +1134,17 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
 	})
+	if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
+		defer sendEvent(eventArgs{
+			EventName:    event.ObjectManyVersions,
+			BucketName:   objInfo.Bucket,
+			Object:       objInfo,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         handlers.GetSourceIP(r),
+		})
+	}
 
 	if redirectURL != nil { // success_action_redirect is valid and set.
 		v := redirectURL.Query()

cmd/bucket-lifecycle.go

@@ -24,12 +24,15 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
@@ -38,6 +41,8 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
+	"github.com/minio/minio/internal/workers"
+	"github.com/minio/pkg/env"
 )
 
 const (
@@ -66,6 +71,30 @@ func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
 
+func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string) madmin.TraceInfo {
+	return madmin.TraceInfo{
+		TraceType: madmin.TraceILM,
+		Time:      startTime,
+		NodeName:  globalLocalNodeName,
+		FuncName:  event,
+		Duration:  duration,
+		Path:      pathJoin(oi.Bucket, oi.Name),
+		Error:     "",
+		Message:   getSource(4),
+		Custom:    map[string]string{"version-id": oi.VersionID},
+	}
+}
+
+func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string) {
+	startTime := time.Now()
+	return func(event string) {
+		duration := time.Since(startTime)
+		if globalTrace.NumSubscribers(madmin.TraceILM) > 0 {
+			globalTrace.Publish(ilmTrace(startTime, duration, oi, event))
+		}
+	}
+}
+
 type expiryTask struct {
 	objInfo        ObjectInfo
 	versionExpiry  bool
@@ -116,26 +145,49 @@ var globalExpiryState *expiryState
 
 func newExpiryState() *expiryState {
 	return &expiryState{
-		byDaysCh:            make(chan expiryTask, 10000),
-		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 10000),
+		byDaysCh:            make(chan expiryTask, 100000),
+		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 100000),
 	}
 }
 
 func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 	globalExpiryState = newExpiryState()
+
+	workerSize, _ := strconv.Atoi(env.Get("_MINIO_ILM_EXPIRY_WORKERS", strconv.Itoa((runtime.GOMAXPROCS(0)+1)/2)))
+
+	ewk, err := workers.New(workerSize)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
+	nwk, err := workers.New(workerSize)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
 	go func() {
 		for t := range globalExpiryState.byDaysCh {
-			if t.objInfo.TransitionedObject.Status != "" {
-				applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
-			} else {
-				applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
-			}
+			ewk.Take()
+			go func(t expiryTask) {
+				defer ewk.Give()
+				if t.objInfo.TransitionedObject.Status != "" {
+					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
+				} else {
+					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
+				}
+			}(t)
 		}
+		ewk.Wait()
 	}()
 	go func() {
 		for t := range globalExpiryState.byNewerNoncurrentCh {
-			deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+			nwk.Take()
+			go func(t newerNoncurrentTask) {
+				defer nwk.Give()
+				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+			}(t)
 		}
+		nwk.Wait()
 	}()
 }
 
@@ -152,7 +204,6 @@ type transitionTask struct {
 }
 
 type transitionState struct {
-	once         sync.Once
 	transitionCh chan transitionTask
 
 	ctx        context.Context
@@ -169,10 +220,7 @@ type transitionState struct {
 
 func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
 	select {
-	case <-GlobalContext.Done():
-		t.once.Do(func() {
-			close(t.transitionCh)
-		})
+	case <-t.ctx.Done():
 	case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
 	default:
 	}
@@ -214,20 +262,20 @@ func (t *transitionState) ActiveTasks() int {
 }
 
 // worker waits for transition tasks
-func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
+func (t *transitionState) worker(objectAPI ObjectLayer) {
 	for {
 		select {
 		case <-t.killCh:
 			return
-		case <-ctx.Done():
+		case <-t.ctx.Done():
 			return
 		case task, ok := <-t.transitionCh:
 			if !ok {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			if err := transitionObject(ctx, objectAPI, task.objInfo, task.tier); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
+			if err := transitionObject(t.ctx, objectAPI, task.objInfo, task.tier); err != nil {
+				logger.LogIf(t.ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
 					task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 			} else {
 				ts := tierStats{
@@ -278,7 +326,7 @@ func (t *transitionState) UpdateWorkers(n int) {
 
 func (t *transitionState) updateWorkers(n int) {
 	for t.numWorkers < n {
-		go t.worker(t.ctx, t.objAPI)
+		go t.worker(t.objAPI)
 		t.numWorkers++
 	}
 
@@ -337,6 +385,7 @@ const (
 // 1. when a restored (via PostRestoreObject API) object expires.
 // 2. when a transitioned object expires (based on an ILM rule).
 func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error {
+	traceFn := globalLifecycleSys.trace(*oi)
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name)
 	opts.VersionID = lcOpts.VersionID
@@ -362,7 +411,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		}
 
 		// Send audit for the lifecycle delete operation
-		auditLogLifecycle(ctx, *oi, ILMExpiry)
+		defer auditLogLifecycle(ctx, *oi, ILMExpiry, traceFn)
 
 		eventName := event.ObjectRemovedDelete
 		if lcOpts.DeleteMarker {
@@ -378,7 +427,8 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 			EventName:  eventName,
 			BucketName: oi.Bucket,
 			Object:     objInfo,
-			Host:       "Internal: [ILM-EXPIRY]",
+			UserAgent:  "Internal: [ILM-Expiry]",
+			Host:       globalLocalNodeName,
 		})
 
 	case expireRestoredObj:

cmd/bucket-listobjects-handlers.go

@@ -59,10 +59,18 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max
 	return ErrNone
 }
 
-// ListObjectVersions - GET Bucket Object versions
+func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+	api.listObjectVersionsHandler(w, r, false)
+}
+
+func (api objectAPIHandlers) ListObjectVersionsMHandler(w http.ResponseWriter, r *http.Request) {
+	api.listObjectVersionsHandler(w, r, true)
+}
+
+// ListObjectVersionsHandler - GET Bucket Object versions
 // You can use the versions subresource to list metadata about all
 // of the versions of objects in a bucket.
-func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) listObjectVersionsHandler(w http.ResponseWriter, r *http.Request, metadata bool) {
 	ctx := newContext(r, w, "ListObjectVersions")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -80,7 +88,12 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
-
+	var checkObjMeta metaCheckFn
+	if metadata {
+		checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
+			return checkRequestAuthType(ctx, r, action, bucket, name)
+		}
+	}
 	urlValues := r.Form
 
 	// Extract all the listBucketVersions query params to their native values.
@@ -111,8 +124,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
-	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
+	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo, checkObjMeta)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponseList(response))
@@ -128,64 +140,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 // MinIO continues to support ListObjectsV1 and V2 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2M")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-
-	urlValues := r.Form
-
-	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
-	if errCode != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
-		return
-	}
-
-	// Validate the query params before beginning to serve the request.
-	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-
-	listObjectsV2 := objectAPI.ListObjectsV2
-
-	// Inititate a list objects operation based on the input params.
-	// On success would return back ListObjectsInfo object to be
-	// marshaled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	// The next continuation token has id@node_index format to optimize paginated listing
-	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-
-	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
-		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
-		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true)
-
-	// Write success response.
-	writeSuccessResponseXML(w, encodeResponseList(response))
+	api.listObjectsV2Handler(ctx, w, r, true)
 }
 
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
@@ -198,7 +153,11 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 // MinIO continues to support ListObjectsV1 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
+	api.listObjectsV2Handler(ctx, w, r, false)
+}
 
+// listObjectsV2Handler performs listing either with or without extra metadata.
+func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.ResponseWriter, r *http.Request, metadata bool) {
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
@@ -215,6 +174,12 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
+	var checkObjMeta metaCheckFn
+	if metadata {
+		checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
+			return checkRequestAuthType(ctx, r, action, bucket, name)
+		}
+	}
 	urlValues := r.Form
 
 	// Extract all the listObjectsV2 query params to their native values.
@@ -257,7 +222,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
-		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
+		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, checkObjMeta)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponseList(response))

cmd/bucket-metadata-sys.go

@@ -21,10 +21,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"runtime"
 	"sync"
 	"time"
 
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	bucketsse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -40,6 +42,8 @@ import (
 
 // BucketMetadataSys captures all bucket metadata for a given cluster.
 type BucketMetadataSys struct {
+	objAPI ObjectLayer
+
 	sync.RWMutex
 	metadataMap map[string]BucketMetadata
 }
@@ -53,13 +57,29 @@ func (sys *BucketMetadataSys) Count() int {
 }
 
 // Remove bucket metadata from memory.
-func (sys *BucketMetadataSys) Remove(bucket string) {
+func (sys *BucketMetadataSys) Remove(buckets ...string) {
 	sys.Lock()
-	delete(sys.metadataMap, bucket)
-	globalBucketMonitor.DeleteBucket(bucket)
+	for _, bucket := range buckets {
+		delete(sys.metadataMap, bucket)
+		globalBucketMonitor.DeleteBucket(bucket)
+	}
 	sys.Unlock()
 }
 
+// RemoveStaleBuckets removes all stale buckets in memory that are not on disk.
+func (sys *BucketMetadataSys) RemoveStaleBuckets(diskBuckets set.StringSet) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	for bucket := range sys.metadataMap {
+		if diskBuckets.Contains(bucket) {
+			continue
+		} // doesn't exist on disk remove from memory.
+		delete(sys.metadataMap, bucket)
+		globalBucketMonitor.DeleteBucket(bucket)
+	}
+}
+
 // Set - sets a new metadata in-memory.
 // Only a shallow copy is saved and fields with references
 // cannot be modified without causing a race condition,
@@ -182,7 +202,7 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 // GetVersioningConfig returns configured versioning config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, meta.Created, nil
@@ -195,7 +215,7 @@ func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Ve
 // GetTaggingConfig returns configured tagging config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketTaggingNotFound{Bucket: bucket}
@@ -211,7 +231,7 @@ func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.
 // GetObjectLockConfig returns configured object lock config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketObjectLockConfigNotFound{Bucket: bucket}
@@ -227,7 +247,7 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co
 // GetLifecycleConfig returns configured lifecycle config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketLifecycleNotFound{Bucket: bucket}
@@ -243,7 +263,7 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 // GetNotificationConfig returns configured notification config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return nil, err
 	}
@@ -253,7 +273,7 @@ func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Confi
 // GetSSEConfig returns configured SSE config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketSSEConfigNotFound{Bucket: bucket}
@@ -268,7 +288,7 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
 
 // CreatedAt returns the time of creation of bucket
 func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return time.Time{}, err
 	}
@@ -278,7 +298,7 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
 // GetPolicyConfig returns configured bucket policy
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
@@ -294,7 +314,7 @@ func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, ti
 // GetQuotaConfig returns configured bucket quota
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, time.Time, error) {
-	meta, err := sys.GetConfig(ctx, bucket)
+	meta, _, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketQuotaConfigNotFound{Bucket: bucket}
@@ -307,7 +327,7 @@ func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string)
 // GetReplicationConfig returns configured bucket replication config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, time.Time, error) {
-	meta, err := sys.GetConfig(ctx, bucket)
+	meta, reloaded, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
@@ -318,13 +338,18 @@ func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket s
 	if meta.replicationConfig == nil {
 		return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
 	}
+	if reloaded {
+		globalBucketTargetSys.set(BucketInfo{
+			Name: bucket,
+		}, meta)
+	}
 	return meta.replicationConfig, meta.ReplicationConfigUpdatedAt, nil
 }
 
 // GetBucketTargetsConfig returns configured bucket targets for this bucket
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, reloaded, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketRemoteTargetNotFound{Bucket: bucket}
@@ -334,6 +359,11 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 	if meta.bucketTargetConfig == nil {
 		return nil, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
+	if reloaded {
+		globalBucketTargetSys.set(BucketInfo{
+			Name: bucket,
+		}, meta)
+	}
 	return meta.bucketTargetConfig, nil
 }
 
@@ -353,31 +383,32 @@ func (sys *BucketMetadataSys) GetConfigFromDisk(ctx context.Context, bucket stri
 
 // GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (BucketMetadata, error) {
+// reloaded will be true if metadata refreshed from disk
+func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (meta BucketMetadata, reloaded bool, err error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return newBucketMetadata(bucket), errServerNotInitialized
+		return newBucketMetadata(bucket), reloaded, errServerNotInitialized
 	}
 
 	if isMinioMetaBucketName(bucket) {
-		return newBucketMetadata(bucket), errInvalidArgument
+		return newBucketMetadata(bucket), reloaded, errInvalidArgument
 	}
 
 	sys.RLock()
 	meta, ok := sys.metadataMap[bucket]
 	sys.RUnlock()
 	if ok {
-		return meta, nil
+		return meta, reloaded, nil
 	}
-	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
+	meta, err = loadBucketMetadata(ctx, objAPI, bucket)
 	if err != nil {
-		return meta, err
+		return meta, reloaded, err
 	}
 	sys.Lock()
 	sys.metadataMap[bucket] = meta
 	sys.Unlock()
 
-	return meta, nil
+	return meta, true, nil
 }
 
 // Init - initializes bucket metadata system for all buckets.
@@ -386,59 +417,129 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 		return errServerNotInitialized
 	}
 
+	sys.objAPI = objAPI
+
 	// Load bucket metadata sys in background
-	go sys.load(ctx, buckets, objAPI)
+	go sys.init(ctx, buckets)
+	return nil
+}
+
+func (sys *BucketMetadataSys) loadBucketMetadata(ctx context.Context, bucket BucketInfo) error {
+	meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket.Name)
+	if err != nil {
+		return err
+	}
+
+	sys.Lock()
+	sys.metadataMap[bucket.Name] = meta
+	sys.Unlock()
+
 	return nil
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo) {
 	g := errgroup.WithNErrs(len(buckets))
+	bucketMetas := make([]BucketMetadata, len(buckets))
 	for index := range buckets {
 		index := index
 		g.Go(func() error {
-			_, _ = objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
+			_, _ = sys.objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
 				// Ensure heal opts for bucket metadata be deep healed all the time.
 				ScanMode: madmin.HealDeepScan,
 				Recreate: true,
 			})
-			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
+			meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[index].Name)
 			if err != nil {
-				if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
-					meta = newBucketMetadata(buckets[index].Name)
-				} else {
-					return err
-				}
+				return err
 			}
-			sys.Lock()
-			sys.metadataMap[buckets[index].Name] = meta
-			sys.Unlock()
-
-			globalEventNotifier.set(buckets[index], meta) // set notification targets
-
-			globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
-
+			bucketMetas[index] = meta
 			return nil
 		}, index)
 	}
-	for _, err := range g.Wait() {
+
+	errs := g.Wait()
+	for _, err := range errs {
 		if err != nil {
 			logger.LogIf(ctx, err)
 		}
 	}
+
+	// Hold lock here to update in-memory map at once,
+	// instead of serializing the Go routines.
+	sys.Lock()
+	for i, meta := range bucketMetas {
+		if errs[i] != nil {
+			continue
+		}
+		sys.metadataMap[buckets[i].Name] = meta
+	}
+	sys.Unlock()
+
+	for i, meta := range bucketMetas {
+		if errs[i] != nil {
+			continue
+		}
+		globalEventNotifier.set(buckets[i], meta)   // set notification targets
+		globalBucketTargetSys.set(buckets[i], meta) // set remote replication targets
+	}
+}
+
+func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
+	const bucketMetadataRefresh = 15 * time.Minute
+
+	t := time.NewTimer(bucketMetadataRefresh)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{})
+			if err != nil {
+				logger.LogIf(ctx, err)
+				continue
+			}
+
+			// Handle if we have some buckets in-memory those are stale.
+			// first delete them and then replace the newer state()
+			// from disk.
+			diskBuckets := set.CreateStringSet()
+			for _, bucket := range buckets {
+				diskBuckets.Add(bucket.Name)
+			}
+			sys.RemoveStaleBuckets(diskBuckets)
+
+			for _, bucket := range buckets {
+				err := sys.loadBucketMetadata(ctx, bucket)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+				// Check if there is a spare procs, wait 100ms instead
+				waitForLowIO(runtime.GOMAXPROCS(0), 100*time.Millisecond, currentHTTPIO)
+			}
+
+			t.Reset(bucketMetadataRefresh)
+		}
+	}
 }
 
 // Loads bucket metadata for all buckets into BucketMetadataSys.
-func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) {
 	count := 100 // load 100 bucket metadata at a time.
 	for {
 		if len(buckets) < count {
-			sys.concurrentLoad(ctx, buckets, objAPI)
-			return
+			sys.concurrentLoad(ctx, buckets)
+			break
 		}
-		sys.concurrentLoad(ctx, buckets[:count], objAPI)
+		sys.concurrentLoad(ctx, buckets[:count])
 		buckets = buckets[count:]
 	}
+
+	if globalIsDistErasure {
+		go sys.refreshBucketsMetadataLoop(ctx)
+	}
 }
 
 // Reset the state of the BucketMetadataSys.

cmd/bucket-metadata.go

@@ -323,8 +323,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
 
 		configData, err := readConfig(ctx, objectAPI, configFile)
 		if err != nil {
-			switch err.(type) {
-			case ObjectExistsAsDirectory:
+			if _, ok := err.(ObjectExistsAsDirectory); ok {
 				// in FS mode it possible that we have actual
 				// files in this folder with `.minio.sys/buckets/bucket/configFile`
 				continue

cmd/bucket-object-lock.go

@@ -90,8 +90,8 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 
 	opts.VersionID = object.VersionID
 	if gerr != nil { // error from GetObjectInfo
-		switch gerr.(type) {
-		case MethodNotAllowed: // This happens usually for a delete marker
+		if _, ok := gerr.(MethodNotAllowed); ok {
+			// This happens usually for a delete marker
 			if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 				// Delete marker should be present and valid.
 				return ErrNone
@@ -156,11 +156,8 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 				return ErrNone
 			}
 			// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
-			// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention
-			// or s3:GetBucketObjectLockConfiguration permissions, the operation will succeed.
-			govBypassPerms1 := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName)
-			govBypassPerms2 := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, object.ObjectName)
-			if govBypassPerms1 != ErrNone && govBypassPerms2 != ErrNone {
+			// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention, the operation will succeed.
+			if checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName) != ErrNone {
 				return ErrAccessDenied
 			}
 		}
@@ -195,8 +192,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 				days, objRetention.RetainUntilDate.Time,
 				objRetention.Mode, byPassSet, r, cred,
 				owner)
-			switch apiErr {
-			case ErrAccessDenied:
+			if apiErr == ErrAccessDenied {
 				return errAuthentication
 			}
 			return nil
@@ -213,8 +209,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 					return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
 				}
 			}
-			switch govPerm {
-			case ErrAccessDenied:
+			if govPerm == ErrAccessDenied {
 				return errAuthentication
 			}
 			return nil
@@ -227,8 +222,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 			apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
 				days, objRetention.RetainUntilDate.Time, objRetention.Mode,
 				false, r, cred, owner)
-			switch apiErr {
-			case ErrAccessDenied:
+			if apiErr == ErrAccessDenied {
 				return errAuthentication
 			}
 			return nil
@@ -239,8 +233,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 	apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
 		days, objRetention.RetainUntilDate.Time,
 		objRetention.Mode, byPassSet, r, cred, owner)
-	switch apiErr {
-	case ErrAccessDenied:
+	if apiErr == ErrAccessDenied {
 		return errAuthentication
 	}
 	return nil

cmd/bucket-policy-handlers.go

@@ -86,7 +86,11 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "ErrMalformedXML",
+			HTTPStatusCode: http.StatusBadRequest,
+			Description:    err.Error(),
+		}, r.URL)
 		return
 	}
 

cmd/bucket-policy.go

@@ -75,6 +75,11 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
 		groups   = cred.Groups
 	)
 
+	if cred.IsTemp() || cred.IsServiceAccount() {
+		// For derived credentials, check the parent user's permissions.
+		username = cred.ParentUser
+	}
+
 	principalType := "Anonymous"
 	if username != "" {
 		principalType = "User"

cmd/bucket-replication-handlers.go

@@ -277,6 +277,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 				Bucket: bucket,
 				Err:    fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
 			}), r.URL)
+			return
 		}
 	}
 	resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24))
@@ -347,6 +348,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		}
+		return
 	}
 	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	if err != nil {

cmd/bucket-replication-utils.go

@@ -496,8 +496,7 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 			oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus)
 		}
 		for k, v := range oi.UserDefined {
-			switch {
-			case strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset):
+			if strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset) {
 				delete(oi.UserDefined, k)
 				oi.UserDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
 			}

cmd/bucket-replication.go

@@ -387,7 +387,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -403,7 +404,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -415,7 +417,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
-		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, rcfg.RoleArn))
+		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, dobj.TargetArn))
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -424,7 +426,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -449,7 +452,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 					VersionID:    versionID,
 					DeleteMarker: dobj.DeleteMarker,
 				},
-				Host:      "Internal: [Replication]",
+				UserAgent: "Internal: [Replication]",
+				Host:      globalLocalNodeName,
 				EventName: event.ObjectReplicationNotTracked,
 			})
 			continue
@@ -508,7 +512,6 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
 	if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on.
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %s", bucket, dobj.ObjectName, versionID, err))
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -517,14 +520,16 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: eventName,
 		})
 	} else {
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object:     dobjInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 			EventName:  eventName,
 		})
 	}
@@ -561,7 +566,8 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 				VersionID:    dobj.VersionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		if dobj.VersionID == "" {
@@ -586,11 +592,13 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 				return
 			}
 		}
-		// mark delete marker replication as failed if target cluster not ready to receive
-		// this request yet (object version not replicated yet)
-		if err != nil && !toi.ReplicationReady {
-			rinfo.ReplicationStatus = replication.Failed
-			return
+		if !isErrObjectNotFound(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
+			// mark delete marker replication as failed if target cluster not ready to receive
+			// this request yet (object version not replicated yet)
+			if err != nil && !toi.ReplicationReady {
+				rinfo.ReplicationStatus = replication.Failed
+				return
+			}
 		}
 	}
 
@@ -802,10 +810,12 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 		oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID {
 		return replicateNone
 	}
+	sz, _ := oi1.GetActualSize()
+
 	// needs full replication
 	if oi1.ETag != oi2.ETag ||
 		oi1.VersionID != oi2.VersionID ||
-		oi1.Size != oi2.Size ||
+		sz != oi2.Size ||
 		oi1.DeleteMarker != oi2.IsDeleteMarker ||
 		oi1.ModTime.Unix() != oi2.LastModified.Unix() {
 		return replicateAll
@@ -917,7 +927,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -935,10 +946,11 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
-		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, cfg.RoleArn))
+		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, ri.TargetArn))
 		return
 	}
 	ctx = lkctx.Context()
@@ -955,7 +967,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			continue
 		}
@@ -1001,10 +1014,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			},
 		}
 
-		if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w",
-				bucket, objInfo.Name, objInfo.VersionID, err))
-		}
+		_, _ = objectAPI.PutObjectMetadata(ctx, bucket, object, popts)
 		opType := replication.MetadataReplicationType
 		if rinfos.Action() == replicateAll {
 			opType = replication.ObjectReplicationType
@@ -1020,7 +1030,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		EventName:  eventName,
 		BucketName: bucket,
 		Object:     objInfo,
-		Host:       "Internal: [Replication]",
+		UserAgent:  "Internal: [Replication]",
+		Host:       globalLocalNodeName,
 	})
 
 	// re-queue failures once more - keep a retry count to avoid flooding the queue if
@@ -1065,7 +1076,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1084,7 +1096,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
@@ -1101,7 +1114,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1112,7 +1126,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return rinfo
 	}
@@ -1137,7 +1152,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1202,19 +1218,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		ReplicationAction:     rAction,
 	}
 
-	if ri.ObjectInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
-		rinfo.ReplicationStatus = replication.Completed
-		rinfo.ReplicationResynced = true
-		return
-	}
-
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1233,7 +1244,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
@@ -1243,6 +1255,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 
 	objInfo = gr.ObjInfo
 
+	// use latest ObjectInfo to check if previous replication attempt succeeded
+	if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
+		rinfo.ReplicationStatus = replication.Completed
+		rinfo.ReplicationResynced = true
+		return
+	}
+
 	size, err := objInfo.GetActualSize()
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -1250,18 +1269,20 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
 
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("Unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return rinfo
 	}
@@ -1286,12 +1307,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		if rAction == replicateNone {
 			if ri.OpType == replication.ExistingObjectReplicationType &&
 				objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID {
-				logger.LogIf(ctx, fmt.Errorf("Unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
 				sendEvent(eventArgs{
 					EventName:  event.ObjectReplicationNotTracked,
 					BucketName: bucket,
 					Object:     objInfo,
-					Host:       "Internal: [Replication]",
+					UserAgent:  "Internal: [Replication]",
+					Host:       globalLocalNodeName,
 				})
 			}
 			// object with same VersionID already exists, replication kicked off by
@@ -1328,7 +1350,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 		if _, err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); err != nil {
 			rinfo.ReplicationStatus = replication.Failed
-			logger.LogIf(ctx, fmt.Errorf("Unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 		}
 	} else {
 		var putOpts minio.PutObjectOptions
@@ -1339,7 +1361,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			return
 		}
@@ -1365,7 +1388,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				r, objInfo, putOpts); err != nil {
 				if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1374,7 +1397,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
 				if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1419,7 +1442,12 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		if err != nil {
 			return err
 		}
-		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, "", "", opts.ServerSideEncryption)
+
+		popts := minio.PutObjectPartOptions{
+			SSE: opts.ServerSideEncryption,
+		}
+
+		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
 		if err != nil {
 			return err
 		}
@@ -1535,7 +1563,6 @@ type ReplicationPool struct {
 	// workers:
 	workers         []chan ReplicationWorkerOperation
 	existingWorkers chan ReplicationWorkerOperation
-	workerWg        sync.WaitGroup
 
 	// mrf:
 	mrfWorkerKillCh chan struct{}
@@ -1544,7 +1571,6 @@ type ReplicationPool struct {
 	mrfStopCh       chan struct{}
 	mrfWorkerSize   int
 	saveStateCh     chan struct{}
-	mrfWorkerWg     sync.WaitGroup
 }
 
 // ReplicationWorkerOperation is a shared interface of replication operations.
@@ -1607,7 +1633,6 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 
 	pool.ResizeWorkers(workers, 0)
 	pool.ResizeFailedWorkers(failedWorkers)
-	pool.workerWg.Add(1)
 	go pool.AddWorker(pool.existingWorkers, nil)
 	go pool.resyncer.PersistToDisk(ctx, o)
 	go pool.processMRF()
@@ -1619,7 +1644,6 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 // AddMRFWorker adds a pending/failed replication worker to handle requests that could not be queued
 // to the other workers
 func (p *ReplicationPool) AddMRFWorker() {
-	defer p.mrfWorkerWg.Done()
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -1646,7 +1670,6 @@ func (p *ReplicationPool) AddMRFWorker() {
 // An optional pointer to a tracker that will be atomically
 // incremented when operations are running can be provided.
 func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opTracker *int32) {
-	defer p.workerWg.Done()
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -1704,7 +1727,6 @@ func (p *ReplicationPool) ResizeWorkers(n, checkOld int) {
 		input := make(chan ReplicationWorkerOperation, 10000)
 		p.workers = append(p.workers, input)
 
-		p.workerWg.Add(1)
 		go p.AddWorker(input, &p.activeWorkers)
 	}
 	for len(p.workers) > n {
@@ -1748,7 +1770,6 @@ func (p *ReplicationPool) ResizeFailedWorkers(n int) {
 
 	for p.mrfWorkerSize < n {
 		p.mrfWorkerSize++
-		p.mrfWorkerWg.Add(1)
 		go p.AddMRFWorker()
 	}
 	for p.mrfWorkerSize > n {

cmd/bucket-targets.go

@@ -208,7 +208,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	}
 	clnt, err := sys.getRemoteTargetClient(tgt)
 	if err != nil {
-		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket, Err: err}
 	}
 	// validate if target credentials are ok
 	exists, err := clnt.BucketExists(ctx, tgt.TargetBucket)
@@ -453,6 +453,8 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	if err != nil {
 		return nil, err
 	}
+	api.SetAppInfo("minio-replication-target", ReleaseTag+" "+tcfg.Arn)
+
 	hcDuration := defaultHealthCheckDuration
 	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
 		hcDuration = tcfg.HealthCheckDuration
@@ -473,20 +475,23 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 }
 
 // getRemoteARN gets existing ARN for an endpoint or generates a new one.
-func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget) (arn string, exists bool) {
+func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget, deplID string) (arn string, exists bool) {
 	if target == nil {
 		return
 	}
 	tgts := sys.targetsMap[bucket]
 	for _, tgt := range tgts {
-		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL().String() == tgt.URL().String() && tgt.Credentials.AccessKey == target.Credentials.AccessKey {
+		if tgt.Type == target.Type &&
+			tgt.TargetBucket == target.TargetBucket &&
+			target.URL().String() == tgt.URL().String() &&
+			tgt.Credentials.AccessKey == target.Credentials.AccessKey {
 			return tgt.Arn, true
 		}
 	}
 	if !target.Type.IsValid() {
 		return
 	}
-	return generateARN(target), false
+	return generateARN(target, deplID), false
 }
 
 // getRemoteARNForPeer returns the remote target for a peer site in site replication
@@ -506,10 +511,14 @@ func (sys *BucketTargetSys) getRemoteARNForPeer(bucket string, peer madmin.PeerI
 }
 
 // generate ARN that is unique to this target type
-func generateARN(t *madmin.BucketTarget) string {
+func generateARN(t *madmin.BucketTarget, deplID string) string {
+	uuid := deplID
+	if uuid == "" {
+		uuid = mustGetUUID()
+	}
 	arn := madmin.ARN{
 		Type:   t.Type,
-		ID:     mustGetUUID(),
+		ID:     uuid,
 		Region: t.Region,
 		Bucket: t.TargetBucket,
 	}

cmd/common-main.go

@@ -51,7 +51,7 @@ import (
 	consoleCerts "github.com/minio/console/pkg/certs"
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -205,7 +205,7 @@ func minioConfigToConsoleFeatures() {
 		}
 	}
 	// Enable if LDAP is enabled.
-	if globalLDAPConfig.Enabled() {
+	if globalIAMSys.LDAPConfig.Enabled() {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
@@ -215,8 +215,9 @@ func minioConfigToConsoleFeatures() {
 }
 
 func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
-	m := make(map[string]consoleoauth2.ProviderConfig, len(globalOpenIDConfig.ProviderCfgs))
-	for name, cfg := range globalOpenIDConfig.ProviderCfgs {
+	pcfgs := globalIAMSys.OpenIDConfig.ProviderCfgs
+	m := make(map[string]consoleoauth2.ProviderConfig, len(pcfgs))
+	for name, cfg := range pcfgs {
 		callback := getConsoleEndpoints()[0] + "/oauth_callback"
 		if cfg.RedirectURI != "" {
 			callback = cfg.RedirectURI
@@ -256,6 +257,9 @@ func initConsoleServer() (*restapi.Server, error) {
 		Path: globalCertsCADir.Get(),
 	}
 
+	// set certs before other console initialization
+	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
+
 	swaggerSpec, err := loads.Embedded(restapi.SwaggerJSON, restapi.FlatSwaggerJSON)
 	if err != nil {
 		return nil, err
@@ -282,8 +286,6 @@ func initConsoleServer() (*restapi.Server, error) {
 	// register all APIs
 	server.ConfigureAPI()
 
-	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
-
 	consolePort, _ := strconv.Atoi(globalMinioConsolePort)
 
 	server.Host = globalMinioConsoleHost
@@ -600,13 +602,13 @@ func loadEnvVarsFromFiles() {
 		}
 	}
 
-	if env.IsSet(config.EnvKMSSecretKeyFile) {
-		kmsSecret, err := readFromSecret(env.Get(config.EnvKMSSecretKeyFile, ""))
+	if env.IsSet(kms.EnvKMSSecretKeyFile) {
+		kmsSecret, err := readFromSecret(env.Get(kms.EnvKMSSecretKeyFile, ""))
 		if err != nil {
 			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
 		}
 		if kmsSecret != "" {
-			os.Setenv(config.EnvKMSSecretKey, kmsSecret)
+			os.Setenv(kms.EnvKMSSecretKey, kmsSecret)
 		}
 	}
 
@@ -639,7 +641,7 @@ func handleCommonEnvVars() {
 			if !((u.Scheme == "http" || u.Scheme == "https") &&
 				u.Opaque == "" &&
 				!u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
-				err := fmt.Errorf("URL contains unexpected resources, expected URL to be of http(s)://minio.example.com format: %v", u)
+				err := fmt.Errorf("URL contains unexpected resources, expected URL to be one of http(s)://console.example.com or as a subpath via API endpoint http(s)://minio.example.com/minio format: %v", u)
 				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value is environment variable")
 			}
 			globalBrowserRedirectURL = u
@@ -660,6 +662,7 @@ func handleCommonEnvVars() {
 		}
 		u.Path = "" // remove any path component such as `/`
 		globalMinioEndpoint = u.String()
+		globalMinioEndpointURL = u
 	}
 
 	globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff))
@@ -782,21 +785,29 @@ func handleCommonEnvVars() {
 // Initialize KMS global variable after valiadating and loading the configuration.
 // It depends on KMS env variables and global cli flags.
 func handleKMSConfig() {
-	switch {
-	case env.IsSet(config.EnvKMSSecretKey) && env.IsSet(config.EnvKESEndpoint):
-		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", config.EnvKMSSecretKey, config.EnvKESEndpoint))
+	if env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint) {
+		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKMSSecretKey, kms.EnvKESEndpoint))
 	}
 
-	if env.IsSet(config.EnvKMSSecretKey) {
-		KMS, err := kms.Parse(env.Get(config.EnvKMSSecretKey, ""))
+	if env.IsSet(kms.EnvKMSSecretKey) {
+		KMS, err := kms.Parse(env.Get(kms.EnvKMSSecretKey, ""))
 		if err != nil {
 			logger.Fatal(err, "Unable to parse the KMS secret key inherited from the shell environment")
 		}
 		GlobalKMS = KMS
 	}
-	if env.IsSet(config.EnvKESEndpoint) {
+	if env.IsSet(kms.EnvKESEndpoint) {
+		if env.IsSet(kms.EnvKESAPIKey) {
+			if env.IsSet(kms.EnvKESClientKey) {
+				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientKey))
+			}
+			if env.IsSet(kms.EnvKESClientCert) {
+				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientCert))
+			}
+		}
+
 		var endpoints []string
-		for _, endpoint := range strings.Split(env.Get(config.EnvKESEndpoint, ""), ",") {
+		for _, endpoint := range strings.Split(env.Get(kms.EnvKESEndpoint, ""), ",") {
 			if strings.TrimSpace(endpoint) == "" {
 				continue
 			}
@@ -812,67 +823,82 @@ func handleKMSConfig() {
 				endpoints = append(endpoints, strings.Join(lbls, ""))
 			}
 		}
-		rootCAs, err := certs.GetRootCAs(env.Get(config.EnvKESServerCA, globalCertsCADir.Get()))
+		rootCAs, err := certs.GetRootCAs(env.Get(kms.EnvKESServerCA, globalCertsCADir.Get()))
 		if err != nil {
-			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
+			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(kms.EnvKESServerCA, globalCertsCADir.Get())))
 		}
 
-		loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
-			// Manually load the certificate and private key into memory.
-			// We need to check whether the private key is encrypted, and
-			// if so, decrypt it using the user-provided password.
-			certBytes, err := os.ReadFile(certFile)
+		var kmsConf kms.Config
+		if env.IsSet(kms.EnvKESAPIKey) {
+			key, err := kes.ParseAPIKey(env.Get(kms.EnvKESAPIKey, ""))
 			if err != nil {
-				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+				logger.Fatal(err, fmt.Sprintf("Failed to parse KES API key from %q", env.Get(kms.EnvKESAPIKey, "")))
 			}
-			keyBytes, err := os.ReadFile(keyFile)
-			if err != nil {
-				return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
+			kmsConf = kms.Config{
+				Endpoints:    endpoints,
+				Enclave:      env.Get(kms.EnvKESEnclave, ""),
+				DefaultKeyID: env.Get(kms.EnvKESKeyName, ""),
+				APIKey:       key,
+				RootCAs:      rootCAs,
 			}
-			privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
-			if len(rest) != 0 {
-				return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
-			}
-			if x509.IsEncryptedPEMBlock(privateKeyPEM) {
-				keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
+		} else {
+			loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
+				// Manually load the certificate and private key into memory.
+				// We need to check whether the private key is encrypted, and
+				// if so, decrypt it using the user-provided password.
+				certBytes, err := os.ReadFile(certFile)
 				if err != nil {
-					return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
+					return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
 				}
-				keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+				keyBytes, err := os.ReadFile(keyFile)
+				if err != nil {
+					return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
+				}
+				privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
+				if len(rest) != 0 {
+					return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
+				}
+				if x509.IsEncryptedPEMBlock(privateKeyPEM) {
+					keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(kms.EnvKESClientPassword, "")))
+					if err != nil {
+						return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
+					}
+					keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+				}
+				certificate, err := tls.X509KeyPair(certBytes, keyBytes)
+				if err != nil {
+					return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+				}
+				return certificate, nil
 			}
-			certificate, err := tls.X509KeyPair(certBytes, keyBytes)
+
+			reloadCertEvents := make(chan tls.Certificate, 1)
+			certificate, err := certs.NewCertificate(env.Get(kms.EnvKESClientCert, ""), env.Get(kms.EnvKESClientKey, ""), loadX509KeyPair)
 			if err != nil {
-				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+				logger.Fatal(err, "Failed to load KES client certificate")
+			}
+			certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
+			certificate.Notify(reloadCertEvents)
+
+			kmsConf = kms.Config{
+				Endpoints:        endpoints,
+				Enclave:          env.Get(kms.EnvKESEnclave, ""),
+				DefaultKeyID:     env.Get(kms.EnvKESKeyName, ""),
+				Certificate:      certificate,
+				ReloadCertEvents: reloadCertEvents,
+				RootCAs:          rootCAs,
 			}
-			return certificate, nil
 		}
 
-		reloadCertEvents := make(chan tls.Certificate, 1)
-		certificate, err := certs.NewCertificate(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""), loadX509KeyPair)
-		if err != nil {
-			logger.Fatal(err, "Failed to load KES client certificate")
-		}
-		certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
-		certificate.Notify(reloadCertEvents)
-
-		defaultKeyID := env.Get(config.EnvKESKeyName, "")
-		KMS, err := kms.NewWithConfig(kms.Config{
-			Endpoints:        endpoints,
-			Enclave:          env.Get(config.EnvKESEnclave, ""),
-			DefaultKeyID:     defaultKeyID,
-			Certificate:      certificate,
-			ReloadCertEvents: reloadCertEvents,
-			RootCAs:          rootCAs,
-		})
+		KMS, err := kms.NewWithConfig(kmsConf)
 		if err != nil {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
-
 		// We check that the default key ID exists or try to create it otherwise.
 		// This implicitly checks that we can communicate to KES. We don't treat
 		// a policy error as failure condition since MinIO may not have the permission
 		// to create keys - just to generate/decrypt data encryption keys.
-		if err = KMS.CreateKey(context.Background(), defaultKeyID); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
+		if err = KMS.CreateKey(context.Background(), env.Get(kms.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
 		GlobalKMS = KMS

cmd/config-current.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/config/identity/openid"
 	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	xtls "github.com/minio/minio/internal/config/identity/tls"
+	"github.com/minio/minio/internal/config/lambda"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
@@ -74,6 +75,9 @@ func initHelp() {
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
 	}
+	for k, v := range lambda.DefaultLambdaKVS {
+		kvs[k] = v
+	}
 	if globalIsErasure {
 		kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
 		kvs[config.HealSubSys] = heal.DefaultKVS
@@ -196,6 +200,11 @@ func initHelp() {
 			Description:     "publish bucket notifications to Redis datastores",
 			MultipleTargets: true,
 		},
+		config.HelpKV{
+			Key:             config.LambdaWebhookSubSys,
+			Description:     "manage remote lambda functions",
+			MultipleTargets: true,
+		},
 		config.HelpKV{
 			Key:         config.EtcdSubSys,
 			Description: "persist IAM assets externally to etcd",
@@ -246,6 +255,7 @@ func initHelp() {
 		config.NotifyRedisSubSys:    notify.HelpRedis,
 		config.NotifyWebhookSubSys:  notify.HelpWebhook,
 		config.NotifyESSubSys:       notify.HelpES,
+		config.LambdaWebhookSubSys:  lambda.HelpWebhook,
 		config.SubnetSubSys:         subnet.HelpSubnet,
 		config.CallhomeSubSys:       callhome.HelpCallhome,
 	}
@@ -387,6 +397,13 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			return err
 		}
 	}
+
+	if config.LambdaSubSystems.Contains(subSys) {
+		if err := lambda.TestSubSysLambdaTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -423,6 +440,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 	}
 	if err == nil && dnsURL != "" {
+		bootstrapTrace("initialize remote bucket DNS store")
 		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
 			dns.Authentication(dnsUser, dnsPass),
 			dns.RootCAs(globalRootCAs))
@@ -437,6 +455,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	if etcdCfg.Enabled {
+		bootstrapTrace("initialize etcd store")
 		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
@@ -493,23 +512,20 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")
 	}
 
-	globalSTSTLSConfig, err = xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
-	}
-
-	if globalSTSTLSConfig.InsecureSkipVerify {
-		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
-	}
-
 	transport := NewHTTPTransport()
 
-	bootstrapTrace("lookup the event notification targets")
-	globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
+	bootstrapTrace("initialize the event notification targets")
+	globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
+	bootstrapTrace("initialize the lambda targets")
+	globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
+	}
+
 	bootstrapTrace("applying the dynamic configuration")
 	// Apply dynamic config values
 	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {

cmd/consolelogger.go

@@ -25,10 +25,10 @@ import (
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/minio/internal/logger/target/console"
 	"github.com/minio/minio/internal/logger/target/types"
 	"github.com/minio/minio/internal/pubsub"
+	"github.com/minio/pkg/logger/message/log"
 	xnet "github.com/minio/pkg/net"
 )
 

cmd/crossdomain-xml-handler.go

@@ -33,8 +33,7 @@ const crossDomainXMLEntity = "/crossdomain.xml"
 func setCrossDomainPolicy(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Look for 'crossdomain.xml' in the incoming request.
-		switch r.URL.Path {
-		case crossDomainXMLEntity:
+		if r.URL.Path == crossDomainXMLEntity {
 			// Write the standard cross domain policy xml.
 			w.Write([]byte(crossDomainXML))
 			// Request completed, no need to serve to other handlers.

cmd/data-scanner-metric.go

@@ -69,9 +69,9 @@ const (
 
 // log scanner action.
 // Use for s > scannerMetricStartTrace
-func (p *scannerMetrics) log(s scannerMetric, paths ...string) func() {
+func (p *scannerMetrics) log(s scannerMetric, paths ...string) func(custom map[string]string) {
 	startTime := time.Now()
-	return func() {
+	return func(custom map[string]string) {
 		duration := time.Since(startTime)
 
 		atomic.AddUint64(&p.operations[s], 1)
@@ -80,7 +80,7 @@ func (p *scannerMetrics) log(s scannerMetric, paths ...string) func() {
 		}
 
 		if s > scannerMetricStartTrace && globalTrace.NumSubscribers(madmin.TraceScanner) > 0 {
-			globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " ")))
+			globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " "), custom))
 		}
 	}
 }

cmd/data-scanner.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"encoding/binary"
 	"encoding/json"
@@ -29,11 +28,11 @@ import (
 	"math/rand"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/bits-and-blooms/bloom/v3"
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -59,6 +58,9 @@ const (
 	healDeleteDangling    = true
 	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
 	healObjectSelectProb  = 512 // Overall probability of a file being scanned; one in n.
+
+	dataScannerExcessiveVersionsThreshold = 1000  // Issue a warning when a single object has more versions than this
+	dataScannerExcessiveFoldersThreshold  = 50000 // Issue a warning when a folder has more subfolders than this in a *set*
 )
 
 var (
@@ -158,7 +160,6 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 
 	// Load current bloom cycle
 	var cycleInfo currentScannerCycle
-	cycleInfo.next = intDataUpdateTracker.current() + 1
 
 	buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath)
 	if len(buf) == 8 {
@@ -203,11 +204,13 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
-			bf, err := globalNotificationSys.updateBloomFilter(ctx, cycleInfo.current)
+			err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results, uint32(cycleInfo.current), scanMode)
-			logger.LogIf(ctx, err)
-			stopFn()
+			res := map[string]string{"cycle": fmt.Sprint(cycleInfo.current)}
+			if err != nil {
+				res["error"] = err.Error()
+			}
+			stopFn(res)
 			if err == nil {
 				// Store new cycle...
 				cycleInfo.next++
@@ -240,10 +243,8 @@ type folderScanner struct {
 	oldCache    dataUsageCache
 	newCache    dataUsageCache
 	updateCache dataUsageCache
-	withFilter  *bloomFilter
 
 	dataUsageScannerDebug bool
-	healFolderInclude     uint32 // Include a clean folder one in n cycles.
 	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 	scanMode              madmin.HealScanMode
 
@@ -302,8 +303,6 @@ type folderScanner struct {
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
 func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
-	logPrefix := color.Green("data-usage: ")
-
 	switch cache.Info.Name {
 	case "", dataUsageRoot:
 		return cache, errors.New("internal error: root scan attempted")
@@ -317,8 +316,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		oldCache:              cache,
 		newCache:              dataUsageCache{Info: cache.Info},
 		updateCache:           dataUsageCache{Info: cache.Info},
-		dataUsageScannerDebug: intDataUpdateTracker.debug,
-		healFolderInclude:     0,
+		dataUsageScannerDebug: false,
 		healObjectSelect:      0,
 		scanMode:              scanMode,
 		updates:               cache.Info.updates,
@@ -341,19 +339,9 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 
 	// Enable healing in XL mode.
 	if globalIsErasure && !cache.Info.SkipHealing {
-		// Include a clean folder one in n cycles.
-		s.healFolderInclude = healFolderIncludeProb
 		// Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 		s.healObjectSelect = healObjectSelectProb
 	}
-	if len(cache.Info.BloomFilter) > 0 {
-		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-		_, err := s.withFilter.ReadFrom(bytes.NewReader(cache.Info.BloomFilter))
-		if err != nil {
-			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
-			s.withFilter = nil
-		}
-	}
 
 	done := ctx.Done()
 
@@ -410,14 +398,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			return ctx.Err()
 		default:
 		}
-		existing, ok := f.oldCache.Cache[thisHash.Key()]
 		var abandonedChildren dataUsageHashMap
 		if !into.Compacted {
 			abandonedChildren = f.oldCache.findChildrenCopy(thisHash)
 		}
 
 		// If there are lifecycle rules for the prefix, remove the filter.
-		filter := f.withFilter
 		_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 		var activeLifeCycle *lifecycle.Lifecycle
 		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) {
@@ -425,33 +411,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
 			activeLifeCycle = f.oldCache.Info.lifeCycle
-			filter = nil
 		}
 		// If there are replication rules for the prefix, remove the filter.
 		var replicationCfg replicationConfig
 		if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) {
 			replicationCfg = f.oldCache.Info.replication
-			filter = nil
 		}
 		// Check if we can skip it due to bloom filter...
-		if filter != nil && ok && existing.Compacted {
-			// If folder isn't in filter and we have data, skip it completely.
-			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
-				if f.healObjectSelect == 0 || !thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
-					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-					f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-					if f.dataUsageScannerDebug {
-						console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
-					}
-					return nil
-				}
-				if f.dataUsageScannerDebug {
-					console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
-				}
-				// If probability was already scannerHealFolderInclude, keep it.
-				folder.objectHealProbDiv = f.healFolderInclude
-			}
-		}
 		scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder)
 
 		var existingFolders, newFolders []cachedFolder
@@ -566,6 +532,19 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
 			len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
 
+		if len(existingFolders)+len(newFolders) > dataScannerExcessiveFoldersThreshold {
+			// Notify object accessed via a GET request.
+			sendEvent(eventArgs{
+				EventName:  event.PrefixManyFolders,
+				BucketName: f.root,
+				Object: ObjectInfo{
+					Name: strings.TrimSuffix(folder.name, "/") + "/",
+					Size: int64(len(existingFolders) + len(newFolders)),
+				},
+				UserAgent: "scanner",
+				Host:      globalMinioHost,
+			})
+		}
 		if !into.Compacted && shouldCompact {
 			into.Compacted = true
 			newFolders = append(newFolders, existingFolders...)
@@ -632,7 +611,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			f.updateCurrentPath(folder.name)
 			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
 			scanFolder(folder)
-			stopFn()
+			stopFn(map[string]string{"type": "new"})
+
 			// Add new folders if this is new and we don't have existing.
 			if !into.Compacted {
 				parent := f.updateCache.find(thisHash.Key())
@@ -651,19 +631,16 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// and the entry itself is compacted.
 			if !into.Compacted && f.oldCache.isCompacted(h) {
 				if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-					if f.healObjectSelect == 0 || !h.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
-						// Transfer and add as child...
-						f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
-						into.addChild(h)
-						continue
-					}
-					folder.objectHealProbDiv = f.healFolderInclude
+					// Transfer and add as child...
+					f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
+					into.addChild(h)
+					continue
 				}
 			}
 			f.updateCurrentPath(folder.name)
-			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name, "EXISTING")
+			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
 			scanFolder(folder)
-			stopFn()
+			stopFn(map[string]string{"type": "existing"})
 		}
 
 		// Scan for healing
@@ -803,7 +780,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: 1}
 				stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, this.name, "HEALED")
 				scanFolder(this)
-				stopFn()
+				stopFn(map[string]string{"type": "healed"})
 			}
 		}
 		break
@@ -817,9 +794,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		flat.Compacted = true
 		var compact bool
 		if flat.Objects < dataScannerCompactLeastObject {
-			if flat.Objects > 1 {
-				globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
-			}
 			compact = true
 		} else {
 			// Compact if we only have objects as children...
@@ -832,13 +806,20 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					}
 				}
 			}
-			if compact {
-				globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
-			}
+
 		}
 		if compact {
+			stop := globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
 			f.newCache.deleteRecursive(thisHash)
 			f.newCache.replaceHashed(thisHash, folder.parent, *flat)
+			total := map[string]string{
+				"objects": fmt.Sprint(flat.Objects),
+				"size":    fmt.Sprint(flat.Size),
+			}
+			if flat.Versions > 0 {
+				total["versions"] = fmt.Sprint(flat.Versions)
+			}
+			stop(total)
 		}
 
 	}
@@ -976,6 +957,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 // applyTierObjSweep removes remote object pending deletion and the free-version
 // tracking this information.
 func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
+	traceFn := globalLifecycleSys.trace(oi)
 	if !oi.TransitionedObject.FreeVersion {
 		// nothing to be done
 		return
@@ -1001,7 +983,7 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 		InclFreeVersions: true,
 	})
 	if err == nil {
-		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete)
+		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, traceFn)
 	}
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
@@ -1010,27 +992,35 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 
 // applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
 // Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
-func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
-	if i.lifeCycle == nil {
-		return fivs, nil
-	}
+func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
 	done := globalScannerMetrics.time(scannerMetricApplyNonCurrent)
 	defer done()
 
-	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
-	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
-		return fivs, nil
-	}
-
-	overflowVersions := fivs[lim+1:]
-	// current version + most recent lim noncurrent versions
-	fivs = fivs[:lim+1]
-
 	rcfg, _ := globalBucketObjectLockSys.Get(i.bucket)
 	vcfg, _ := globalBucketVersioningSys.Get(i.bucket)
 
 	versioned := vcfg != nil && vcfg.Versioned(i.objectPath())
 
+	// current version + most recent lim noncurrent versions
+	objectInfos := make([]ObjectInfo, 0, len(fivs))
+
+	if i.lifeCycle == nil {
+		for _, fi := range fivs {
+			objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
+		}
+		return objectInfos, nil
+	}
+
+	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
+		for _, fi := range fivs {
+			objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
+		}
+		return objectInfos, nil
+	}
+
+	overflowVersions := fivs[lim+1:]
+
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
 		obj := fi.ToObjectInfo(i.bucket, i.objectPath(), versioned)
@@ -1045,7 +1035,7 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 			}
 			// add this version back to remaining versions for
 			// subsequent lifecycle policy applications
-			fivs = append(fivs, fi)
+			objectInfos = append(objectInfos, obj)
 			continue
 		}
 
@@ -1059,19 +1049,19 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 
 		toDel = append(toDel, ObjectToDelete{
 			ObjectV: ObjectV{
-				ObjectName: fi.Name,
-				VersionID:  fi.VersionID,
+				ObjectName: obj.Name,
+				VersionID:  obj.VersionID,
 			},
 		})
 	}
 
 	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel)
-	return fivs, nil
+	return objectInfos, nil
 }
 
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
-func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
+func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
 	if i.heal.enabled {
 		if healDeleteDangling {
 			done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
@@ -1083,7 +1073,27 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 		}
 	}
 
-	return i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+	objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if we have many versions after applyNewerNoncurrentVersionLimit.
+	if len(objInfos) > dataScannerExcessiveVersionsThreshold {
+		// Notify object accessed via a GET request.
+		sendEvent(eventArgs{
+			EventName:  event.ObjectManyVersions,
+			BucketName: i.bucket,
+			Object: ObjectInfo{
+				Name: i.objectPath(),
+			},
+			UserAgent:    "Internal: [Scanner]",
+			Host:         globalLocalNodeName,
+			RespElements: map[string]string{"x-minio-versions": strconv.Itoa(len(fivs))},
+		})
+	}
+
+	return objInfos, nil
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -1168,6 +1178,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 }
 
 func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
+	traceFn := globalLifecycleSys.trace(obj)
 	opts := ObjectOptions{
 		Expiration: ExpirationOptions{Expire: true},
 	}
@@ -1191,7 +1202,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	}
 
 	// Send audit for the lifecycle delete operation
-	auditLogLifecycle(ctx, obj, ILMExpiry)
+	auditLogLifecycle(ctx, obj, ILMExpiry, traceFn)
 
 	eventName := event.ObjectRemovedDelete
 	if obj.DeleteMarker {
@@ -1203,7 +1214,8 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		EventName:  eventName,
 		BucketName: obj.Bucket,
 		Object:     obj,
-		Host:       "Internal: [ILM-EXPIRY]",
+		UserAgent:  "Internal: [ILM-Expiry]",
+		Host:       globalLocalNodeName,
 	})
 
 	return true
@@ -1273,8 +1285,7 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 		sizeS.replTargetStats[arn] = tgtSizeS
 	}
 
-	switch oi.ReplicationStatus {
-	case replication.Replica:
+	if oi.ReplicationStatus == replication.Replica {
 		sizeS.replicaSize += oi.Size
 	}
 }
@@ -1434,7 +1445,7 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, traceFn func(event string)) {
 	var apiName string
 	switch event {
 	case ILMExpiry:
@@ -1451,4 +1462,5 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
 		Object:    oi.Name,
 		VersionID: oi.VersionID,
 	})
+	traceFn(event)
 }

cmd/data-update-tracker.go

@@ -1,682 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"encoding/binary"
-	"errors"
-	"io"
-	"os"
-	"path"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/bits-and-blooms/bloom/v3"
-	"github.com/minio/minio/internal/color"
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/console"
-)
-
-const (
-	// Estimate bloom filter size. With this many items
-	dataUpdateTrackerEstItems = 200000
-	// ... we want this false positive rate:
-	dataUpdateTrackerFP        = 0.1
-	dataUpdateTrackerQueueSize = 0
-
-	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 7
-	dataUpdateTrackerSaveInterval = 5 * time.Minute
-)
-
-var intDataUpdateTracker *dataUpdateTracker
-
-func init() {
-	intDataUpdateTracker = newDataUpdateTracker()
-}
-
-type dataUpdateTracker struct {
-	mu         sync.Mutex
-	input      chan string
-	save       chan struct{}
-	debug      bool
-	saveExited chan struct{}
-	dirty      bool
-
-	Current dataUpdateFilter
-	History dataUpdateTrackerHistory
-	Saved   time.Time
-}
-
-// newDataUpdateTracker returns a dataUpdateTracker with default settings.
-func newDataUpdateTracker() *dataUpdateTracker {
-	d := &dataUpdateTracker{
-		Current: dataUpdateFilter{
-			idx: 1,
-		},
-		debug:      serverDebugLog,
-		input:      make(chan string, dataUpdateTrackerQueueSize),
-		save:       make(chan struct{}, 1),
-		saveExited: make(chan struct{}),
-	}
-	d.Current.bf = d.newBloomFilter()
-	d.dirty = true
-	return d
-}
-
-type dataUpdateTrackerHistory []dataUpdateFilter
-
-type dataUpdateFilter struct {
-	idx uint64
-	bf  bloomFilter
-}
-
-type bloomFilter struct {
-	*bloom.BloomFilter
-}
-
-// emptyBloomFilter returns an empty bloom filter.
-func emptyBloomFilter() bloomFilter {
-	return bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-}
-
-// containsDir returns whether the bloom filter contains a directory.
-// Note that objects in XL mode are also considered directories.
-func (b bloomFilter) containsDir(in string) bool {
-	split := splitPathDeterministic(path.Clean(in))
-
-	if len(split) == 0 {
-		return false
-	}
-	return b.TestString(hashPath(path.Join(split...)).String())
-}
-
-// bytes returns the bloom filter serialized as a byte slice.
-func (b *bloomFilter) bytes() []byte {
-	if b == nil || b.BloomFilter == nil {
-		return nil
-	}
-	var buf bytes.Buffer
-	_, err := b.WriteTo(&buf)
-	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return nil
-	}
-	return buf.Bytes()
-}
-
-// sort the dataUpdateTrackerHistory, newest first.
-// Returns whether the history is complete.
-func (d dataUpdateTrackerHistory) sort() bool {
-	if len(d) == 0 {
-		return true
-	}
-	sort.Slice(d, func(i, j int) bool {
-		return d[i].idx > d[j].idx
-	})
-	return d[0].idx-d[len(d)-1].idx == uint64(len(d))
-}
-
-// removeOlderThan will remove entries older than index 'n'.
-func (d *dataUpdateTrackerHistory) removeOlderThan(n uint64) {
-	d.sort()
-	dd := *d
-	end := len(dd)
-	for i := end - 1; i >= 0; i-- {
-		if dd[i].idx < n {
-			end = i
-		}
-	}
-	dd = dd[:end]
-	*d = dd
-}
-
-// newBloomFilter returns a new bloom filter with default settings.
-func (d *dataUpdateTracker) newBloomFilter() bloomFilter {
-	return bloomFilter{bloom.NewWithEstimates(dataUpdateTrackerEstItems, dataUpdateTrackerFP)}
-}
-
-// current returns the current index.
-func (d *dataUpdateTracker) current() uint64 {
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	return d.Current.idx
-}
-
-// latestWithDir returns the highest index that contains the directory.
-// This means that any cycle higher than this does NOT contain the entry.
-func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
-	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
-	bucket, _ := path2BucketObjectWithBasePath("", dir)
-	if bucket == "" {
-		if d.debug && len(dir) > 0 {
-			console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", dir)
-		}
-		return d.current()
-	}
-	if isReservedOrInvalidBucket(bucket, false) {
-		return d.current()
-	}
-
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	if d.Current.bf.containsDir(dir) || d.Current.idx == 0 {
-		return d.Current.idx
-	}
-	if d.debug {
-		console.Debugf(dateUpdateTrackerLogPrefix+" current bloom does NOT contains dir %s\n", dir)
-	}
-
-	idx := d.Current.idx - 1
-	for {
-		f := d.History.find(idx)
-		if f == nil || f.bf.containsDir(dir) || idx == 0 {
-			break
-		}
-		idx--
-	}
-	return idx
-}
-
-// start will load the current data from the drives start collecting information and
-// start a saver goroutine.
-// All of these will exit when the context is canceled.
-func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
-	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No local drives specified"))
-		return
-	}
-	d.load(ctx, drives...)
-	go d.startCollector(ctx)
-	// startSaver will unlock.
-	d.mu.Lock()
-	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives...)
-}
-
-// load will attempt to load data tracking information from the supplied drives.
-// The data will only be loaded if d.Saved is older than the one found on disk.
-// The newest working cache will be kept in d.
-// If no valid data usage tracker can be found d will remain unchanged.
-// If object is shared the caller should lock it.
-func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
-	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No local drives specified"))
-		return
-	}
-	for _, drive := range drives {
-
-		cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
-		f, err := OpenFile(cacheFormatPath, readMode, 0o666)
-		if err != nil {
-			if osIsNotExist(err) {
-				continue
-			}
-			logger.LogIf(ctx, err)
-			continue
-		}
-		err = d.deserialize(f, d.Saved)
-		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
-			logger.LogIf(ctx, err)
-		}
-		f.Close()
-	}
-}
-
-// startSaver will start a saver that will write d to all supplied drives at specific intervals.
-// 'd' must be write locked when started and will be unlocked.
-// The saver will save and exit when supplied context is closed.
-func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives ...string) {
-	if len(drives) == 0 {
-		return
-	}
-
-	saveNow := d.save
-	exited := make(chan struct{})
-	d.saveExited = exited
-	d.mu.Unlock()
-	t := time.NewTicker(interval)
-	defer t.Stop()
-	defer close(exited)
-	var buf bytes.Buffer
-	for {
-		var exit bool
-		select {
-		case <-ctx.Done():
-			exit = true
-		case <-t.C:
-		case <-saveNow:
-		}
-		buf.Reset()
-		d.mu.Lock()
-		if !d.dirty {
-			d.mu.Unlock()
-			if exit {
-				return
-			}
-			continue
-		}
-		d.Saved = UTCNow()
-		err := d.serialize(&buf)
-		if d.debug {
-			console.Debugf(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v\n", buf.Len(), d.Current.idx)
-		}
-		d.dirty = false
-		d.mu.Unlock()
-		if err != nil {
-			logger.LogIf(ctx, err, "Error serializing usage tracker data")
-			if exit {
-				return
-			}
-			continue
-		}
-		if buf.Len() == 0 {
-			logger.LogIf(ctx, errors.New("zero sized output, skipping save"))
-			continue
-		}
-		for _, drive := range drives {
-			cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
-			err := os.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
-			if err != nil {
-				if osIsNotExist(err) {
-					continue
-				}
-				logger.LogIf(ctx, err)
-				continue
-			}
-		}
-		if exit {
-			return
-		}
-	}
-}
-
-// serialize all data in d to dst.
-// Caller should hold lock if d is expected to be shared.
-// If an error is returned, there will likely be partial data written to dst.
-func (d *dataUpdateTracker) serialize(dst io.Writer) (err error) {
-	ctx := GlobalContext
-	var tmp [8]byte
-	o := bufio.NewWriter(dst)
-	defer func() {
-		if err == nil {
-			err = o.Flush()
-		}
-	}()
-
-	// Version
-	if err := o.WriteByte(dataUpdateTrackerVersion); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	// Timestamp.
-	binary.LittleEndian.PutUint64(tmp[:], uint64(d.Saved.Unix()))
-	if _, err := o.Write(tmp[:]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	// Current
-	binary.LittleEndian.PutUint64(tmp[:], d.Current.idx)
-	if _, err := o.Write(tmp[:]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	if _, err := d.Current.bf.WriteTo(o); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	// History
-	binary.LittleEndian.PutUint64(tmp[:], uint64(len(d.History)))
-	if _, err := o.Write(tmp[:]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	for _, bf := range d.History {
-		// Current
-		binary.LittleEndian.PutUint64(tmp[:], bf.idx)
-		if _, err := o.Write(tmp[:]); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-
-		if _, err := bf.bf.WriteTo(o); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-	}
-	return nil
-}
-
-// deserialize will deserialize the supplied input if the input is newer than the supplied time.
-func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) error {
-	ctx := GlobalContext
-	var dst dataUpdateTracker
-	var tmp [8]byte
-
-	// Version
-	if _, err := io.ReadFull(src, tmp[:1]); err != nil {
-		if d.debug {
-			if err != io.EOF {
-				logger.LogIf(ctx, err)
-			}
-		}
-		return err
-	}
-	switch tmp[0] {
-	case 1, 2, 3, 4, 5, 6:
-		if intDataUpdateTracker.debug {
-			console.Debugln(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
-		}
-		return nil
-	case dataUpdateTrackerVersion:
-	default:
-		return errors.New("dataUpdateTracker: Unknown data version")
-	}
-	// Timestamp.
-	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	t := time.Unix(int64(binary.LittleEndian.Uint64(tmp[:])), 0)
-	if !t.After(newerThan) {
-		return nil
-	}
-
-	// Current
-	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	dst.Current.idx = binary.LittleEndian.Uint64(tmp[:])
-	dst.Current.bf = emptyBloomFilter()
-	if _, err := dst.Current.bf.ReadFrom(src); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	// History
-	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	n := binary.LittleEndian.Uint64(tmp[:])
-	dst.History = make(dataUpdateTrackerHistory, int(n))
-	for i, e := range dst.History {
-		if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-		e.idx = binary.LittleEndian.Uint64(tmp[:])
-		e.bf = emptyBloomFilter()
-		if _, err := e.bf.ReadFrom(src); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-		dst.History[i] = e
-	}
-	// Ignore what remains on the stream.
-	// Update d:
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	d.Current = dst.Current
-	d.History = dst.History
-	d.Saved = dst.Saved
-	return nil
-}
-
-// start a collector that picks up entries from objectUpdatedCh
-// and adds them  to the current bloom filter.
-func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case in := <-d.input:
-			bucket, _ := path2BucketObjectWithBasePath("", in)
-			if bucket == "" {
-				if d.debug && len(in) > 0 {
-					console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
-				}
-				continue
-			}
-
-			if isReservedOrInvalidBucket(bucket, false) {
-				continue
-			}
-			split := splitPathDeterministic(in)
-
-			// Add all paths until done.
-			d.mu.Lock()
-			for i := range split {
-				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-			}
-			d.dirty = d.dirty || len(split) > 0
-			d.mu.Unlock()
-		}
-	}
-}
-
-// markDirty adds the supplied path to the current bloom filter.
-func (d *dataUpdateTracker) markDirty(bucket, prefix string) {
-	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
-	if bucket == "" && d.debug {
-		console.Debugf(dateUpdateTrackerLogPrefix + " no bucket specified\n")
-		return
-	}
-
-	if isReservedOrInvalidBucket(bucket, false) && d.debug {
-		return
-	}
-	split := splitPathDeterministic(pathJoin(bucket, prefix))
-
-	// Add all paths until done.
-	d.mu.Lock()
-	for i := range split {
-		d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-	}
-	d.dirty = d.dirty || len(split) > 0
-	d.mu.Unlock()
-}
-
-// find entry with specified index.
-// Returns nil if not found.
-func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
-	for _, f := range d {
-		if f.idx == idx {
-			return &f
-		}
-	}
-	return nil
-}
-
-// filterFrom will return a combined bloom filter.
-func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint64) *bloomFilterResponse {
-	bf := d.newBloomFilter()
-	bfr := bloomFilterResponse{
-		OldestIdx:  oldest,
-		CurrentIdx: d.Current.idx,
-		Complete:   true,
-	}
-	// Loop through each index requested.
-	for idx := oldest; idx <= newest; idx++ {
-		v := d.History.find(idx)
-		if v == nil {
-			if d.Current.idx == idx {
-				// Merge current.
-				err := bf.Merge(d.Current.bf.BloomFilter)
-				logger.LogIf(ctx, err)
-				if err != nil {
-					bfr.Complete = false
-				}
-				continue
-			}
-			bfr.Complete = false
-			bfr.OldestIdx = idx + 1
-			continue
-		}
-
-		err := bf.Merge(v.bf.BloomFilter)
-		if err != nil {
-			bfr.Complete = false
-			logger.LogIf(ctx, err)
-			continue
-		}
-		bfr.NewestIdx = idx
-	}
-	var dst bytes.Buffer
-	_, err := bf.WriteTo(&dst)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return nil
-	}
-	bfr.Filter = dst.Bytes()
-
-	return &bfr
-}
-
-// cycleFilter will cycle the bloom filter to start recording to index y if not already.
-// The response will contain a bloom filter starting at index x up to, but not including index y.
-// If y is 0, the response will not update y, but return the currently recorded information
-// from the oldest (unless 0, then it will be all) until and including current y.
-func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
-	if req.OldestClean != "" {
-		return &bloomFilterResponse{OldestIdx: d.latestWithDir(req.OldestClean)}, nil
-	}
-	current := req.Current
-	oldest := req.Oldest
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	if current == 0 {
-		if len(d.History) == 0 {
-			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
-		}
-		d.History.sort()
-		if oldest == 0 {
-			oldest = d.History[len(d.History)-1].idx
-		}
-		return d.filterFrom(ctx, oldest, d.Current.idx), nil
-	}
-
-	// Move current to history if new one requested
-	if d.Current.idx != current {
-		d.dirty = true
-		if d.debug {
-			console.Debugf(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v\n", d.Current.idx, current)
-		}
-
-		d.History = append(d.History, d.Current)
-		d.Current.idx = current
-		d.Current.bf = d.newBloomFilter()
-		select {
-		case d.save <- struct{}{}:
-		default:
-		}
-	}
-	d.History.removeOlderThan(oldest)
-	return d.filterFrom(ctx, oldest, current), nil
-}
-
-// splitPathDeterministic will split the provided relative path
-// deterministically and return up to the first 3 elements of the path.
-// slash and dot prefixes are removed.
-// Trailing slashes are removed.
-// Returns 0 length if no parts are found after trimming.
-func splitPathDeterministic(in string) []string {
-	split := strings.Split(decodeDirObject(in), SlashSeparator)
-
-	// Trim empty start/end
-	for len(split) > 0 {
-		if len(split[0]) > 0 && split[0] != "." {
-			break
-		}
-		split = split[1:]
-	}
-	for len(split) > 0 {
-		if len(split[len(split)-1]) > 0 {
-			break
-		}
-		split = split[:len(split)-1]
-	}
-
-	return split
-}
-
-// bloomFilterRequest request bloom filters.
-// Current index will be updated to current and entries back to Oldest is returned.
-type bloomFilterRequest struct {
-	Oldest  uint64
-	Current uint64
-	// If set the oldest clean version will be returned in OldestIdx
-	// and the rest of the request will be ignored.
-	OldestClean string
-}
-
-type bloomFilterResponse struct {
-	// Current index being written to.
-	CurrentIdx uint64
-	// Oldest index in the returned bloom filter.
-	OldestIdx uint64
-	// Newest Index in the returned bloom filter.
-	NewestIdx uint64
-	// Are all indexes between oldest and newest filled?
-	Complete bool
-	// Binary data of the bloom filter.
-	Filter []byte
-}
-
-// NSUpdated indicates namespace has been updated.
-// The function will block until the entry has been picked up.
-func NSUpdated(bucket, prefix string) {
-	if intDataUpdateTracker != nil {
-		intDataUpdateTracker.markDirty(bucket, prefix)
-	}
-}

cmd/data-update-tracker_test.go

@@ -1,299 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"math/rand"
-	"os"
-	"path"
-	"path/filepath"
-	"sync"
-	"testing"
-
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
-	"github.com/minio/minio/internal/logger/target/types"
-)
-
-type testLoggerI interface {
-	Helper()
-	Log(args ...interface{})
-}
-
-type testingLogger struct {
-	mu sync.Mutex
-	t  testLoggerI
-}
-
-func (t *testingLogger) Endpoint() string {
-	return ""
-}
-
-func (t *testingLogger) String() string {
-	return ""
-}
-
-func (t *testingLogger) Init() error {
-	return nil
-}
-
-func (t *testingLogger) Cancel() {
-}
-
-func (t *testingLogger) Type() types.TargetType {
-	return types.TargetHTTP
-}
-
-func (t *testingLogger) IsOnline() bool {
-	return true
-}
-
-// Stats returns the target statistics.
-func (t *testingLogger) Stats() types.TargetStats {
-	return types.TargetStats{}
-}
-
-func (t *testingLogger) Send(entry interface{}) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if t.t == nil {
-		return nil
-	}
-	e, ok := entry.(log.Entry)
-	if !ok {
-		return fmt.Errorf("unexpected log entry structure %#v", entry)
-	}
-
-	t.t.Helper()
-	t.t.Log(e.Level, ":", e.Message)
-	return nil
-}
-
-func addTestingLogging(t testLoggerI) func() {
-	tl := &testingLogger{t: t}
-	logger.AddSystemTarget(tl)
-	return func() {
-		tl.mu.Lock()
-		defer tl.mu.Unlock()
-		tl.t = nil
-	}
-}
-
-func TestDataUpdateTracker(t *testing.T) {
-	dut := newDataUpdateTracker()
-	// Change some defaults.
-	dut.debug = testing.Verbose()
-	dut.input = make(chan string)
-	dut.save = make(chan struct{})
-
-	defer addTestingLogging(t)()
-
-	dut.Current.bf = dut.newBloomFilter()
-
-	tmpDir := t.TempDir()
-	err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	dut.start(ctx, tmpDir)
-
-	tests := []struct {
-		in    string
-		check []string // if not empty, check against these instead.
-		exist bool
-	}{
-		{
-			in:    "bucket/directory/file.txt",
-			check: []string{"bucket", "bucket/", "/bucket", "bucket/directory", "bucket/directory/", "bucket/directory/file.txt", "/bucket/directory/file.txt"},
-			exist: true,
-		},
-		{
-			// System bucket
-			in:    ".minio.sys/ignoreme/pls",
-			exist: false,
-		},
-		{
-			// Not a valid bucket
-			in:    "./bucket/okfile.txt",
-			check: []string{"./bucket/okfile.txt", "/bucket/okfile.txt", "bucket/okfile.txt"},
-			exist: false,
-		},
-		{
-			// Not a valid bucket
-			in:    "æ/okfile.txt",
-			check: []string{"æ/okfile.txt", "æ/okfile.txt", "æ"},
-			exist: false,
-		},
-		{
-			in:    "/bucket2/okfile2.txt",
-			check: []string{"./bucket2/okfile2.txt", "/bucket2/okfile2.txt", "bucket2/okfile2.txt", "bucket2"},
-			exist: true,
-		},
-		{
-			in:    "/bucket3/prefix/okfile2.txt",
-			check: []string{"./bucket3/prefix/okfile2.txt", "/bucket3/prefix/okfile2.txt", "bucket3/prefix/okfile2.txt", "bucket3/prefix", "bucket3"},
-			exist: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.in, func(t *testing.T) {
-			dut.input <- tt.in
-			dut.input <- "" // Sending empty string ensures the previous is added to filter.
-			dut.mu.Lock()
-			defer dut.mu.Unlock()
-			if len(tt.check) == 0 {
-				got := dut.Current.bf.containsDir(tt.in)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
-				}
-				return
-			}
-			for _, check := range tt.check {
-				got := dut.Current.bf.containsDir(check)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
-				}
-				continue
-			}
-		})
-	}
-	// Cycle to history
-	req := bloomFilterRequest{
-		Oldest:  1,
-		Current: 2,
-	}
-
-	_, err = dut.cycleFilter(ctx, req)
-	if err != nil {
-		t.Fatal(err)
-	}
-	dut.input <- "cycle2/file.txt"
-	dut.input <- "" // Sending empty string ensures the previous is added to filter.
-
-	tests = append(tests, struct {
-		in    string
-		check []string
-		exist bool
-	}{in: "cycle2/file.txt", exist: true})
-
-	// Shut down
-	cancel()
-	<-dut.saveExited
-
-	if dut.current() != 2 {
-		t.Fatal("wrong current idx after save. want 2, got:", dut.current())
-	}
-
-	ctx, cancel = context.WithCancel(context.Background())
-	// Reload...
-	dut = newDataUpdateTracker()
-	dut.start(ctx, tmpDir)
-	defer func() {
-		cancel()
-		<-dut.saveExited
-	}()
-
-	if dut.current() != 2 {
-		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
-	}
-	req = bloomFilterRequest{
-		Oldest:  1,
-		Current: 3,
-	}
-	bfr2, err := dut.cycleFilter(ctx, req)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bfr2.Complete {
-		t.Fatal("Wanted complete, didn't get it")
-	}
-	if bfr2.CurrentIdx != 3 {
-		t.Fatal("wanted index 3, got", bfr2.CurrentIdx)
-	}
-	if bfr2.OldestIdx != 1 {
-		t.Fatal("wanted oldest index 3, got", bfr2.OldestIdx)
-	}
-
-	t.Logf("Size of filter %d bytes, M: %d, K:%d", len(bfr2.Filter), dut.Current.bf.Cap(), dut.Current.bf.K())
-	// Rerun test with returned bfr2
-	bf := dut.newBloomFilter()
-	_, err = bf.ReadFrom(bytes.NewReader(bfr2.Filter))
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, tt := range tests {
-		t.Run(tt.in+"-reloaded", func(t *testing.T) {
-			if len(tt.check) == 0 {
-				got := bf.containsDir(tt.in)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
-				}
-				return
-			}
-			for _, check := range tt.check {
-				got := bf.containsDir(check)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
-				}
-				continue
-			}
-		})
-	}
-}
-
-func BenchmarkDataUpdateTracker(b *testing.B) {
-	dut := newDataUpdateTracker()
-	// Change some defaults.
-	dut.debug = false
-	dut.input = make(chan string)
-	dut.save = make(chan struct{})
-
-	defer addTestingLogging(b)()
-
-	dut.Current.bf = dut.newBloomFilter()
-	// We do this unbuffered. This will very significantly reduce throughput, so this is a worst case.
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go dut.startCollector(ctx)
-	input := make([]string, 1000)
-	rng := rand.New(rand.NewSource(0xabad1dea))
-	tmp := []string{"bucket", "aprefix", "nextprefixlevel", "maybeobjname", "evendeeper", "ok-one-morelevel", "final.object"}
-	for i := range input {
-		tmp := tmp[:1+rng.Intn(cap(tmp)-1)]
-		input[i] = path.Join(tmp...)
-	}
-	b.SetBytes(1)
-	b.ResetTimer()
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		dut.input <- input[rng.Intn(len(input))]
-	}
-}

cmd/data-usage-cache.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"path"
 	"path/filepath"
@@ -47,6 +48,9 @@ type dataUsageHash string
 // sizeHistogram is a size histogram.
 type sizeHistogram [dataUsageBucketLen]uint64
 
+// versionsHistogram is a histogram of number of versions in an object.
+type versionsHistogram [dataUsageVersionLen]uint64
+
 type dataUsageEntry struct {
 	Children dataUsageHashMap `msg:"ch"`
 	// These fields do no include any children.
@@ -54,6 +58,7 @@ type dataUsageEntry struct {
 	Objects          uint64               `msg:"os"`
 	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.
 	ObjSizes         sizeHistogram        `msg:"szs"`
+	ObjVersions      versionsHistogram    `msg:"vh"`
 	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
 	AllTierStats     *allTierStats        `msg:"ats,omitempty"`
 	Compacted        bool                 `msg:"c"`
@@ -276,7 +281,6 @@ type dataUsageCacheInfo struct {
 	// indicates if the disk is being healed and scanner
 	// should skip healing the disk
 	SkipHealing bool
-	BloomFilter []byte `msg:"BloomFilter,omitempty"`
 
 	// Active lifecycle, if any on the bucket
 	lifeCycle *lifecycle.Lifecycle `msg:"-"`
@@ -292,6 +296,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.Size += summary.totalSize
 	e.Versions += summary.versions
 	e.ObjSizes.add(summary.totalSize)
+	e.ObjVersions.add(summary.versions)
 
 	if e.ReplicationStats == nil {
 		e.ReplicationStats = &replicationAllStats{
@@ -352,6 +357,10 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 		e.ObjSizes[i] += v
 	}
 
+	for i, v := range other.ObjVersions[:] {
+		e.ObjVersions[i] += v
+	}
+
 	if other.AllTierStats != nil {
 		if e.AllTierStats == nil {
 			e.AllTierStats = newAllTierStats()
@@ -649,10 +658,7 @@ func (d *dataUsageCache) reduceChildrenOf(path dataUsageHash, limit int, compact
 // StringAll returns a detailed string representation of all entries in the cache.
 func (d *dataUsageCache) StringAll() string {
 	// Remove bloom filter from print.
-	bf := d.Info.BloomFilter
-	d.Info.BloomFilter = nil
 	s := fmt.Sprintf("info:%+v\n", d.Info)
-	d.Info.BloomFilter = bf
 	for k, v := range d.Cache {
 		s += fmt.Sprintf("\t%v: %+v\n", k, v)
 	}
@@ -698,7 +704,7 @@ func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
 func (h *sizeHistogram) add(size int64) {
 	// Fetch the histogram interval corresponding
 	// to the passed object size.
-	for i, interval := range ObjectsHistogramIntervals {
+	for i, interval := range ObjectsHistogramIntervals[:] {
 		if size >= interval.start && size <= interval.end {
 			h[i]++
 			break
@@ -715,6 +721,27 @@ func (h *sizeHistogram) toMap() map[string]uint64 {
 	return res
 }
 
+// add a version count to the histogram.
+func (h *versionsHistogram) add(versions uint64) {
+	// Fetch the histogram interval corresponding
+	// to the passed object size.
+	for i, interval := range ObjectsVersionCountIntervals[:] {
+		if versions >= uint64(interval.start) && versions <= uint64(interval.end) {
+			h[i]++
+			break
+		}
+	}
+}
+
+// toMap returns the map to a map[string]uint64.
+func (h *versionsHistogram) toMap() map[string]uint64 {
+	res := make(map[string]uint64, dataUsageVersionLen)
+	for i, count := range h {
+		res[ObjectsVersionCountIntervals[i].name] = count
+	}
+	return res
+}
+
 func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats {
 	dst := newAllTierStats()
 	for _, bucket := range buckets {
@@ -745,10 +772,11 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		}
 		flat := d.flatten(*e)
 		bui := BucketUsageInfo{
-			Size:                 uint64(flat.Size),
-			VersionsCount:        flat.Versions,
-			ObjectsCount:         flat.Objects,
-			ObjectSizesHistogram: flat.ObjSizes.toMap(),
+			Size:                    uint64(flat.Size),
+			VersionsCount:           flat.Versions,
+			ObjectsCount:            flat.Objects,
+			ObjectSizesHistogram:    flat.ObjSizes.toMap(),
+			ObjectVersionsHistogram: flat.ObjVersions.toMap(),
 		}
 		if flat.ReplicationStats != nil {
 			bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
@@ -851,37 +879,54 @@ type objectIO interface {
 
 // load the cache content with name from minioMetaBackgroundOpsBucket.
 // Only backend errors are returned as errors.
+// The loader is optimistic and has no locking, but tries 5 times before giving up.
 // If the object is not found or unable to deserialize d is cleared and nil error is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
 	// Abandon if more than 5 minutes, so we don't hold up scanner.
 	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
 	defer cancel()
 
-	r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, readLock, ObjectOptions{})
-	if err != nil {
-		switch err.(type) {
-		case ObjectNotFound:
-		case BucketNotFound:
-		case InsufficientReadQuorum:
-		case StorageErr:
-		default:
-			return toObjectErr(err, dataUsageBucket, name)
+	// Caches are read+written without locks,
+	retries := 0
+	for retries < 5 {
+		r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, noLock, ObjectOptions{NoLock: true})
+		if err != nil {
+			switch err.(type) {
+			case ObjectNotFound, BucketNotFound:
+			case InsufficientReadQuorum, StorageErr:
+				retries++
+				time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
+				continue
+			default:
+				return toObjectErr(err, dataUsageBucket, name)
+			}
+			*d = dataUsageCache{}
+			return nil
 		}
-		*d = dataUsageCache{}
+		if err := d.deserialize(r); err != nil {
+			r.Close()
+			retries++
+			time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
+			continue
+		}
+		r.Close()
 		return nil
 	}
-	defer r.Close()
-	if err := d.deserialize(r); err != nil {
-		*d = dataUsageCache{}
-		logger.LogOnceIf(ctx, err, err.Error())
-	}
+	*d = dataUsageCache{}
 	return nil
 }
 
+// Maximum running concurrent saves on server.
+var maxConcurrentScannerSaves = make(chan struct{}, 4)
+
 // save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
+// Note that no locking is done when saving.
 func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
 	var r io.Reader
-
+	maxConcurrentScannerSaves <- struct{}{}
+	defer func() {
+		<-maxConcurrentScannerSaves
+	}()
 	// If big, do streaming...
 	size := int64(-1)
 	if len(d.Cache) > 10000 {
@@ -913,7 +958,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 		dataUsageBucket,
 		name,
 		NewPutObjReader(hr),
-		ObjectOptions{})
+		ObjectOptions{NoLock: true})
 	if isErrBucketNotFound(err) {
 		return nil
 	}

cmd/data-usage-cache_gen.go

@@ -629,12 +629,6 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "SkipHealing")
 				return
 			}
-		case "BloomFilter":
-			z.BloomFilter, err = dc.ReadBytes(z.BloomFilter)
-			if err != nil {
-				err = msgp.WrapError(err, "BloomFilter")
-				return
-			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -648,24 +642,9 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// omitempty: check for empty values
-	zb0001Len := uint32(5)
-	var zb0001Mask uint8 /* 5 bits */
-	_ = zb0001Mask
-	if z.BloomFilter == nil {
-		zb0001Len--
-		zb0001Mask |= 0x10
-	}
-	// variable map header, size zb0001Len
-	err = en.Append(0x80 | uint8(zb0001Len))
-	if err != nil {
-		return
-	}
-	if zb0001Len == 0 {
-		return
-	}
+	// map header, size 4
 	// write "Name"
-	err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -704,39 +683,15 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "SkipHealing")
 		return
 	}
-	if (zb0001Mask & 0x10) == 0 { // if not empty
-		// write "BloomFilter"
-		err = en.Append(0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
-		if err != nil {
-			return
-		}
-		err = en.WriteBytes(z.BloomFilter)
-		if err != nil {
-			err = msgp.WrapError(err, "BloomFilter")
-			return
-		}
-	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// omitempty: check for empty values
-	zb0001Len := uint32(5)
-	var zb0001Mask uint8 /* 5 bits */
-	_ = zb0001Mask
-	if z.BloomFilter == nil {
-		zb0001Len--
-		zb0001Mask |= 0x10
-	}
-	// variable map header, size zb0001Len
-	o = append(o, 0x80|uint8(zb0001Len))
-	if zb0001Len == 0 {
-		return
-	}
+	// map header, size 4
 	// string "Name"
-	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "NextCycle"
 	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
@@ -747,11 +702,6 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "SkipHealing"
 	o = append(o, 0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67)
 	o = msgp.AppendBool(o, z.SkipHealing)
-	if (zb0001Mask & 0x10) == 0 { // if not empty
-		// string "BloomFilter"
-		o = append(o, 0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
-		o = msgp.AppendBytes(o, z.BloomFilter)
-	}
 	return
 }
 
@@ -797,12 +747,6 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "SkipHealing")
 				return
 			}
-		case "BloomFilter":
-			z.BloomFilter, bts, err = msgp.ReadBytesBytes(bts, z.BloomFilter)
-			if err != nil {
-				err = msgp.WrapError(err, "BloomFilter")
-				return
-			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -817,7 +761,7 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize
 	return
 }
 
@@ -1596,6 +1540,24 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "vh":
+			var zb0003 uint32
+			zb0003, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "ObjVersions")
+				return
+			}
+			if zb0003 != uint32(dataUsageVersionLen) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
+				return
+			}
+			for za0002 := range z.ObjVersions {
+				z.ObjVersions[za0002], err = dc.ReadUint64()
+				if err != nil {
+					err = msgp.WrapError(err, "ObjVersions", za0002)
+					return
+				}
+			}
 		case "rs":
 			if dc.IsNil() {
 				err = dc.ReadNil()
@@ -1652,16 +1614,16 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
-	zb0001Len := uint32(8)
-	var zb0001Mask uint8 /* 8 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.ReplicationStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x20
+		zb0001Mask |= 0x40
 	}
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x40
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	err = en.Append(0x80 | uint8(zb0001Len))
@@ -1728,7 +1690,24 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	if (zb0001Mask & 0x20) == 0 { // if not empty
+	// write "vh"
+	err = en.Append(0xa2, 0x76, 0x68)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
+	if err != nil {
+		err = msgp.WrapError(err, "ObjVersions")
+		return
+	}
+	for za0002 := range z.ObjVersions {
+		err = en.WriteUint64(z.ObjVersions[za0002])
+		if err != nil {
+			err = msgp.WrapError(err, "ObjVersions", za0002)
+			return
+		}
+	}
+	if (zb0001Mask & 0x40) == 0 { // if not empty
 		// write "rs"
 		err = en.Append(0xa2, 0x72, 0x73)
 		if err != nil {
@@ -1747,7 +1726,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x40) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not empty
 		// write "ats"
 		err = en.Append(0xa3, 0x61, 0x74, 0x73)
 		if err != nil {
@@ -1783,16 +1762,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// omitempty: check for empty values
-	zb0001Len := uint32(8)
-	var zb0001Mask uint8 /* 8 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.ReplicationStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x20
+		zb0001Mask |= 0x40
 	}
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x40
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
@@ -1821,7 +1800,13 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	for za0001 := range z.ObjSizes {
 		o = msgp.AppendUint64(o, z.ObjSizes[za0001])
 	}
-	if (zb0001Mask & 0x20) == 0 { // if not empty
+	// string "vh"
+	o = append(o, 0xa2, 0x76, 0x68)
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
+	for za0002 := range z.ObjVersions {
+		o = msgp.AppendUint64(o, z.ObjVersions[za0002])
+	}
+	if (zb0001Mask & 0x40) == 0 { // if not empty
 		// string "rs"
 		o = append(o, 0xa2, 0x72, 0x73)
 		if z.ReplicationStats == nil {
@@ -1834,7 +1819,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x40) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not empty
 		// string "ats"
 		o = append(o, 0xa3, 0x61, 0x74, 0x73)
 		if z.AllTierStats == nil {
@@ -1913,6 +1898,24 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "vh":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjVersions")
+				return
+			}
+			if zb0003 != uint32(dataUsageVersionLen) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
+				return
+			}
+			for za0002 := range z.ObjVersions {
+				z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "ObjVersions", za0002)
+					return
+				}
+			}
 		case "rs":
 			if msgp.IsNil(bts) {
 				bts, err = msgp.ReadNilBytes(bts)
@@ -1967,7 +1970,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageEntry) Msgsize() (s int) {
-	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3
+	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
 	if z.ReplicationStats == nil {
 		s += msgp.NilSize
 	} else {
@@ -3760,3 +3763,81 @@ func (z tierStats) Msgsize() (s int) {
 	s = 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
 	return
 }
+
+// DecodeMsg implements msgp.Decodable
+func (z *versionsHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageVersionLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *versionsHistogram) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for za0001 := range z {
+		err = en.WriteUint64(z[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *versionsHistogram) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
+	for za0001 := range z {
+		o = msgp.AppendUint64(o, z[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *versionsHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageVersionLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *versionsHistogram) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size))
+	return
+}

cmd/data-usage-cache_gen_test.go

@@ -1196,3 +1196,116 @@ func BenchmarkDecodetierStats(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalversionsHistogram(t *testing.T) {
+	v := versionsHistogram{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeversionsHistogram(t *testing.T) {
+	v := versionsHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeversionsHistogram Msgsize() is inaccurate")
+	}
+
+	vn := versionsHistogram{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/data-usage-utils.go

@@ -58,11 +58,12 @@ type BucketUsageInfo struct {
 	// Total number of objects that failed replication
 	ReplicationFailedCountV1 uint64 `json:"objectsFailedReplicationCount"`
 
-	ObjectsCount         uint64                           `json:"objectsCount"`
-	ObjectSizesHistogram map[string]uint64                `json:"objectsSizesHistogram"`
-	VersionsCount        uint64                           `json:"versionsCount"`
-	ReplicaSize          uint64                           `json:"objectReplicaTotalSize"`
-	ReplicationInfo      map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
+	ObjectsCount            uint64                           `json:"objectsCount"`
+	ObjectSizesHistogram    map[string]uint64                `json:"objectsSizesHistogram"`
+	ObjectVersionsHistogram map[string]uint64                `json:"objectsVersionsHistogram"`
+	VersionsCount           uint64                           `json:"versionsCount"`
+	ReplicaSize             uint64                           `json:"objectReplicaTotalSize"`
+	ReplicationInfo         map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
 }
 
 // DataUsageInfo represents data usage stats of the underlying Object API

cmd/disk-cache.go

@@ -25,7 +25,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
@@ -35,6 +34,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
+	xnet "github.com/minio/pkg/net"
 	"github.com/minio/pkg/wildcard"
 )
 
@@ -448,7 +448,10 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 			return cachedObjInfo, nil
 		}
 		c.cacheStats.incMiss()
-		return ObjectInfo{}, BackendDown{}
+		if xnet.IsNetworkOrHostDown(err, false) {
+			return ObjectInfo{}, BackendDown{Err: err.Error()}
+		}
+		return ObjectInfo{}, err
 	}
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
@@ -893,8 +896,16 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 				cacheDiskStats[i].UsageSize = info.Used
 				cacheDiskStats[i].TotalCapacity = info.Total
 				cacheDiskStats[i].Dir = dcache.stats.Dir
-				atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&dcache.stats.UsageState))
-				atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&dcache.stats.UsagePercent))
+				if info.Total != 0 {
+					// UsageState
+					gcTriggerPct := dcache.quotaPct * dcache.highWatermark / 100
+					usedPercent := float64(info.Used) * 100 / float64(info.Total)
+					if usedPercent >= float64(gcTriggerPct) {
+						cacheDiskStats[i].UsageState = 1
+					}
+					// UsagePercent
+					cacheDiskStats[i].UsagePercent = uint64(usedPercent)
+				}
 			}
 		}
 		return cacheDiskStats

cmd/encryption-v1.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -34,7 +34,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"
@@ -48,10 +48,11 @@ import (
 
 var (
 	// AWS errors for invalid SSE-C requests.
-	errEncryptedObject      = errors.New("The object was stored using a form of SSE")
-	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
-	errKMSNotConfigured     = errors.New("KMS not configured for a server side encrypted object")
-	errKMSKeyNotFound       = errors.New("Invalid KMS keyId")
+	errEncryptedObject                = errors.New("The object was stored using a form of SSE")
+	errInvalidSSEParameters           = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
+	errKMSNotConfigured               = errors.New("KMS not configured for a server side encrypted objects")
+	errKMSKeyNotFound                 = errors.New("Unknown KMS key ID")
+	errKMSDefaultKeyAlreadyConfigured = errors.New("A default encryption already exists on KMS")
 	// Additional MinIO errors for SSE-C requests.
 	errObjectTampered = errors.New("The requested object was modified and may be compromised")
 	// error returned when invalid encryption parameters are specified
@@ -109,7 +110,7 @@ func kmsKeyIDFromMetadata(metadata map[string]string) string {
 //
 // DecryptETags uses a KMS bulk decryption API, if available, which
 // is more efficient than decrypting ETags sequentually.
-func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error {
+func DecryptETags(ctx context.Context, k kms.KMS, objects []ObjectInfo) error {
 	const BatchSize = 250 // We process the objects in batches - 250 is a reasonable default.
 	var (
 		metadata = make([]map[string]string, 0, BatchSize)
@@ -169,7 +170,7 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error
 		// For all SSE-S3 single-part objects we have to
 		// fetch their decryption keys. We do this using
 		// a Bulk-Decryption API call, if available.
-		keys, err := crypto.S3.UnsealObjectKeys(ctx, KMS, metadata, buckets, names)
+		keys, err := crypto.S3.UnsealObjectKeys(ctx, k, metadata, buckets, names)
 		if err != nil {
 			return err
 		}

cmd/endpoint_test.go

@@ -62,13 +62,14 @@ func TestNewEndpoint(t *testing.T) {
 				err = endpoint.UpdateIsLocal()
 			}
 
-			if test.expectedErr == nil {
+			switch {
+			case test.expectedErr == nil:
 				if err != nil {
 					t.Errorf("error: expected = <nil>, got = %v", err)
 				}
-			} else if err == nil {
+			case err == nil:
 				t.Errorf("error: expected = %v, got = <nil>", test.expectedErr)
-			} else if test.expectedErr.Error() != err.Error() {
+			case test.expectedErr.Error() != err.Error():
 				t.Errorf("error: expected = %v, got = %v", test.expectedErr, err)
 			}
 
@@ -115,13 +116,14 @@ func TestNewEndpoints(t *testing.T) {
 
 	for _, testCase := range testCases {
 		_, err := NewEndpoints(testCase.args...)
-		if testCase.expectedErr == nil {
+		switch {
+		case testCase.expectedErr == nil:
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)
 			}
-		} else if err == nil {
+		case err == nil:
 			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
+		case testCase.expectedErr.Error() != err.Error():
 			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
 		}
 	}

cmd/erasure-encode_test.go

@@ -24,7 +24,7 @@ import (
 	"io"
 	"testing"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 )
 
 type badDisk struct{ StorageAPI }
@@ -41,10 +41,6 @@ func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset
 	return nil, errFaultyDisk
 }
 
-func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
-	return nil, errFaultyDisk
-}
-
 func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
 	return errFaultyDisk
 }

cmd/erasure-healing.go

@@ -66,16 +66,12 @@ func (fi FileInfo) DataShardFixed() bool {
 	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
 }
 
-// Heals a bucket if it doesn't exist on one of the disks, additionally
+// HealBucket heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
 func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (
 	result madmin.HealResultItem, err error,
 ) {
-	if !opts.DryRun {
-		defer NSUpdated(bucket, slashSeparator)
-	}
-
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 
@@ -298,6 +294,26 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 	return false
 }
 
+const xMinIOHealing = ReservedMetadataPrefix + "healing"
+
+// SetHealing marks object (version) as being healed.
+// Note: this is to be used only from healObject
+func (fi *FileInfo) SetHealing() {
+	if fi.Metadata == nil {
+		fi.Metadata = make(map[string]string)
+	}
+	fi.Metadata[xMinIOHealing] = "true"
+}
+
+// Healing returns true if object is being healed (i.e fi is being passed down
+// from healObject)
+func (fi FileInfo) Healing() bool {
+	if _, ok := fi.Metadata[xMinIOHealing]; ok {
+		return true
+	}
+	return false
+}
+
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
 	dryRun := opts.DryRun
@@ -411,6 +427,11 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			erasure.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
 	}
 
+	result.ObjectSize, err = latestMeta.GetActualSize()
+	if err != nil {
+		return result, err
+	}
+
 	// Loop to find number of disks with valid data, per-drive
 	// data state and a list of outdated disks on which data needs
 	// to be healed.
@@ -421,9 +442,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		switch {
 		case v != nil:
 			driveState = madmin.DriveStateOk
-			// If data is sane on any one disk, we can
-			// extract the correct object size.
-			result.ObjectSize = partsMetadata[i].Size
 		case errs[i] == errDiskNotFound, dataErrs[i] == errDiskNotFound:
 			driveState = madmin.DriveStateOffline
 		case errs[i] == errFileNotFound, errs[i] == errFileVersionNotFound, errs[i] == errVolumeNotFound:
@@ -507,8 +525,34 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	migrateDataDir := mustGetUUID()
 
 	// Reorder so that we have data disks first and parity disks next.
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(availableDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from available disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, availableDisks, bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
+
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(outDatedDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from outdated disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, outDatedDisks, bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution)
+
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(partsMetadata) {
+		err := fmt.Errorf("unexpected file distribution (%v) from metadata entries (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, len(partsMetadata), bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution)
 
 	copyPartsMetadata := make([]FileInfo, len(partsMetadata))
@@ -640,6 +684,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		partsMetadata[i].Erasure.Index = i + 1
 
 		// Attempt a rename now from healed data to final location.
+		partsMetadata[i].SetHealing()
 		if _, err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
 			logger.LogIf(ctx, err)
 			return result, err
@@ -673,9 +718,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		})
 	}
 
-	// Set the size of the object in the heal result
-	result.ObjectSize = latestMeta.Size
-
 	return result, nil
 }
 
@@ -755,7 +797,6 @@ func (er *erasureObjects) healObjectDir(ctx context.Context, bucket, object stri
 				}(index, disk)
 			}
 			wg.Wait()
-			NSUpdated(bucket, object)
 		}
 	}
 
@@ -916,13 +957,14 @@ func isObjectDirDangling(errs []error) (ok bool) {
 	var foundNotEmpty int
 	var otherFound int
 	for _, readErr := range errs {
-		if readErr == nil {
+		switch {
+		case readErr == nil:
 			found++
-		} else if readErr == errFileNotFound || readErr == errVolumeNotFound {
+		case readErr == errFileNotFound || readErr == errVolumeNotFound:
 			notFound++
-		} else if readErr == errVolumeNotEmpty {
+		case readErr == errVolumeNotEmpty:
 			foundNotEmpty++
-		} else {
+		default:
 			otherFound++
 		}
 	}
@@ -944,11 +986,12 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 			diskNotFoundCount int
 		)
 		for _, readErr := range cerrs {
-			if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
+			switch {
+			case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
 				notFoundCount++
-			} else if errors.Is(readErr, errFileCorrupt) {
+			case errors.Is(readErr, errFileCorrupt):
 				corruptedCount++
-			} else if errors.Is(readErr, errDiskNotFound) {
+			case errors.Is(readErr, errDiskNotFound):
 				diskNotFoundCount++
 			}
 		}
@@ -1063,7 +1106,12 @@ func healTrace(funcName healingMetric, startTime time.Time, bucket, object strin
 		Path:      pathJoin(bucket, decodeDirObject(object)),
 	}
 	if opts != nil {
-		tr.Message = fmt.Sprintf("dry:%v, rm:%v, recreate:%v mode:%v", opts.DryRun, opts.Remove, opts.Recreate, opts.ScanMode)
+		tr.Custom = map[string]string{
+			"dry":      fmt.Sprint(opts.DryRun),
+			"remove":   fmt.Sprint(opts.Remove),
+			"recreate": fmt.Sprint(opts.Recreate),
+			"mode":     fmt.Sprint(opts.ScanMode),
+		}
 	}
 	if err != nil {
 		tr.Error = err.Error()

cmd/erasure-healing_test.go

@@ -1621,10 +1621,10 @@ func TestHealLastDataShard(t *testing.T) {
 			}
 
 			firstGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
-			defer firstGr.Close()
 			if err != nil {
 				t.Fatal(err)
 			}
+			defer firstGr.Close()
 
 			firstHealedH := sha256.New()
 			_, err = io.Copy(firstHealedH, firstGr)
@@ -1651,10 +1651,10 @@ func TestHealLastDataShard(t *testing.T) {
 			}
 
 			secondGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
-			defer secondGr.Close()
 			if err != nil {
 				t.Fatal(err)
 			}
+			defer secondGr.Close()
 
 			secondHealedH := sha256.New()
 			_, err = io.Copy(secondHealedH, secondGr)

cmd/erasure-metadata.go

@@ -23,15 +23,18 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/sio"
 )
 
 // Object was stored with additional erasure codes due to degraded system at upload time
@@ -104,6 +107,52 @@ func (fi FileInfo) IsValid() bool {
 		correctIndexes)
 }
 
+func (fi FileInfo) checkMultipart() (int64, bool) {
+	if len(fi.Parts) == 0 {
+		return 0, false
+	}
+	if !crypto.IsMultiPart(fi.Metadata) {
+		return 0, false
+	}
+	var size int64
+	for _, part := range fi.Parts {
+		psize, err := sio.DecryptedSize(uint64(part.Size))
+		if err != nil {
+			return 0, false
+		}
+		size += int64(psize)
+	}
+
+	return size, len(extractETag(fi.Metadata)) != 32
+}
+
+// GetActualSize - returns the actual size of the stored object
+func (fi FileInfo) GetActualSize() (int64, error) {
+	if _, ok := fi.Metadata[ReservedMetadataPrefix+"compression"]; ok {
+		sizeStr, ok := fi.Metadata[ReservedMetadataPrefix+"actual-size"]
+		if !ok {
+			return -1, errInvalidDecompressedSize
+		}
+		size, err := strconv.ParseInt(sizeStr, 10, 64)
+		if err != nil {
+			return -1, errInvalidDecompressedSize
+		}
+		return size, nil
+	}
+	if _, ok := crypto.IsEncrypted(fi.Metadata); ok {
+		size, ok := fi.checkMultipart()
+		if !ok {
+			size, err := sio.DecryptedSize(uint64(fi.Size))
+			if err != nil {
+				err = errObjectTampered // assign correct error type
+			}
+			return int64(size), err
+		}
+		return size, nil
+	}
+	return fi.Size, nil
+}
+
 // ToObjectInfo - Converts metadata to object info.
 func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInfo {
 	object = decodeDirObject(object)

cmd/erasure-multipart.go

@@ -1216,7 +1216,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
 	}
-	defer NSUpdated(bucket, object)
 
 	if !opts.Speedtest && versionsDisparity {
 		listAndHeal(ctx, bucket, object, &er, healObjectVersionsDisparity)

cmd/erasure-object.go

@@ -72,8 +72,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, NotImplemented{}
 	}
 
-	defer NSUpdated(dstBucket, dstObject)
-
 	if !dstOpts.NoLock {
 		lk := er.NewNSLock(dstBucket, dstObject)
 		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
@@ -188,6 +186,14 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	auditObjectErasureSet(ctx, object, &er)
 
+	// This is a special call attempted first to check for SOS-API calls.
+	gr, err = veeamSOSAPIGetObject(ctx, bucket, object, rs, opts)
+	if err == nil {
+		return gr, nil
+	}
+	// reset any error to 'nil'
+	err = nil
+
 	var unlockOnDefer bool
 	nsUnlocker := func() {}
 	defer func() {
@@ -475,7 +481,6 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		if opts.VersionID != "" {
 			err = errFileVersionNotFound
 		}
-		defer NSUpdated(bucket, object)
 
 		fi := FileInfo{
 			VersionID: m.VersionID,
@@ -483,7 +488,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		if opts.VersionID != "" {
 			fi.VersionID = opts.VersionID
 		}
-
+		fi.SetTierFreeVersionID(mustGetUUID())
 		disks := er.getDisks()
 		g := errgroup.WithNErrs(len(disks))
 		for index := range disks {
@@ -653,6 +658,13 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		return fi, nil, nil, err
 	}
 
+	if !fi.Deleted && len(fi.Erasure.Distribution) != len(onlineDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			fi.Erasure.Distribution, onlineDisks, bucket, object, opts.VersionID)
+		logger.LogIf(ctx, err)
+		return fi, nil, nil, toObjectErr(err, bucket, object, opts.VersionID)
+	}
+
 	filterOnlineDisksInplace(fi, metaArr, onlineDisks)
 
 	// if one of the disk is offline, return right here no need
@@ -967,17 +979,14 @@ func healObjectVersionsDisparity(bucket string, entry metaCacheEntry) error {
 
 	fivs, err := entry.fileInfoVersions(bucket)
 	if err != nil {
-		go healObject(bucket, entry.name, "", madmin.HealNormalScan)
+		go healObject(bucket, entry.name, "", madmin.HealDeepScan)
 		return err
 	}
 
-	if len(fivs.Versions) > 2 {
-		return fmt.Errorf("bucket(%s)/object(%s) object with %d versions needs healing, allowing lazy healing via scanner instead here for versions greater than 2",
-			bucket, entry.name, len(fivs.Versions))
-	}
-
-	for _, version := range fivs.Versions {
-		go healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
+	if len(fivs.Versions) <= 2 {
+		for _, version := range fivs.Versions {
+			go healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
+		}
 	}
 
 	return nil
@@ -989,7 +998,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	if opts.CheckPrecondFn != nil {
 		obj, err := er.getObjectInfo(ctx, bucket, object, opts)
-		if err != nil && !isErrVersionNotFound(err) {
+		if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
 			return objInfo, err
 		}
 		if opts.CheckPrecondFn(obj) {
@@ -1263,8 +1272,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
-	defer NSUpdated(bucket, object)
-
 	for i := 0; i < len(onlineDisks); i++ {
 		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
 			// Object info is the same in all disks, so we can pick
@@ -1467,14 +1474,15 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		} else {
 			errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName)
 		}
-
-		defer NSUpdated(bucket, objects[objIndex].ObjectName)
 	}
 
 	// Check failed deletes across multiple objects
 	for i, dobj := range dobjects {
-		// This object errored, no need to attempt a heal.
-		if errs[i] != nil {
+		// This object errored, we should attempt a heal just in case.
+		if errs[i] != nil && !isErrVersionNotFound(errs[i]) && !isErrObjectNotFound(errs[i]) {
+			// all other direct versionId references we should
+			// ensure no dangling file is left over.
+			er.addPartial(bucket, dobj.ObjectName, dobj.VersionID, -1)
 			continue
 		}
 
@@ -1560,16 +1568,12 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
-	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
-	defer NSUpdated(bucket, object)
-
 	storageDisks := er.getDisks()
 	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
 	goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
 	if gerr != nil && goi.Name == "" {
-		switch gerr.(type) {
-		case InsufficientReadQuorum:
+		if _, ok := gerr.(InsufficientReadQuorum); ok {
 			return objInfo, InsufficientWriteQuorum{}
 		}
 		// For delete marker replication, versionID being replicated will not exist on disk
@@ -1647,6 +1651,18 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	}
 	fvID := mustGetUUID()
 
+	defer func() {
+		// attempt a heal before returning if there are offline disks
+		// for both del marker and permanent delete situations.
+		for _, disk := range storageDisks {
+			if disk != nil && disk.IsOnline() {
+				continue
+			}
+			er.addPartial(bucket, object, opts.VersionID, -1)
+			break
+		}
+	}()
+
 	if markDelete && (opts.Versioned || opts.VersionSuspended) {
 		if !deleteMarker {
 			// versioning suspended means we add `null` version as
@@ -1694,14 +1710,6 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 
-	for _, disk := range storageDisks {
-		if disk != nil && disk.IsOnline() {
-			continue
-		}
-		er.addPartial(bucket, object, opts.VersionID, -1)
-		break
-	}
-
 	return dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
@@ -1932,7 +1940,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	if fi.TransitionStatus == lifecycle.TransitionComplete {
 		return nil
 	}
-	defer NSUpdated(bucket, object)
 
 	if fi.XLV1 {
 		if _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{NoLock: true}); err != nil {
@@ -1946,6 +1953,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 			return toObjectErr(err, bucket, object)
 		}
 	}
+	traceFn := globalLifecycleSys.trace(fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended))
 
 	destObj, err := genTransitionObjName(bucket)
 	if err != nil {
@@ -1990,9 +1998,10 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		EventName:  eventName,
 		BucketName: bucket,
 		Object:     objInfo,
-		Host:       "Internal: [ILM-Transition]",
+		UserAgent:  "Internal: [ILM-Transition]",
+		Host:       globalLocalNodeName,
 	})
-	auditLogLifecycle(ctx, objInfo, ILMTransition)
+	auditLogLifecycle(ctx, objInfo, ILMTransition, traceFn)
 	return err
 }
 
@@ -2096,3 +2105,46 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 	})
 	return setRestoreHeaderFn(oi, err)
 }
+
+// DecomTieredObject - moves tiered object to another pool during decommissioning.
+func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
+	if opts.UserDefined == nil {
+		opts.UserDefined = make(map[string]string)
+	}
+	// overlay Erasure info for this set of disks
+	storageDisks := er.getDisks()
+	// Get parity and data drive count based on storage class metadata
+	parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
+	if parityDrives < 0 {
+		parityDrives = er.defaultParityCount
+	}
+	dataDrives := len(storageDisks) - parityDrives
+
+	// we now know the number of blocks this object needs for data and parity.
+	// writeQuorum is dataBlocks + 1
+	writeQuorum := dataDrives
+	if dataDrives == parityDrives {
+		writeQuorum++
+	}
+
+	// Initialize parts metadata
+	partsMetadata := make([]FileInfo, len(storageDisks))
+
+	fi2 := newFileInfo(pathJoin(bucket, object), dataDrives, parityDrives)
+	fi.Erasure = fi2.Erasure
+	// Initialize erasure metadata.
+	for index := range partsMetadata {
+		partsMetadata[index] = fi
+		partsMetadata[index].Erasure.Index = index + 1
+	}
+
+	// Order disks according to erasure distribution
+	var onlineDisks []StorageAPI
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi)
+
+	if _, err := writeUniqueFileInfo(ctx, onlineDisks, bucket, object, partsMetadata, writeQuorum); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	return nil
+}

cmd/erasure-server-pool-decom.go

@@ -32,7 +32,6 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v2"
-	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
@@ -735,18 +734,18 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			}
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
+
 			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
-			switch evt.Action {
-			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
-				// Skip this entry.
-				return true
-			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-				globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
-				// Skip this entry.
+			switch {
+			case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
+				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt.Action.DeleteRestored())
+				return false
+			case evt.Action.Delete():
+				globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
 				return true
+			default:
+				return false
 			}
-			return false
 		}
 
 		decommissionEntry := func(entry metaCacheEntry) {
@@ -770,12 +769,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 			var decommissionedCount int
 			for _, version := range fivs.Versions {
-				// TODO: Skip transitioned objects for now.
-				if version.IsRemote() {
-					logger.LogIf(ctx, fmt.Errorf("found %s/%s transitioned object, transitioned object won't be decommissioned", bi.Name, version.Name))
-					continue
-				}
-
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bi.Name, version.Name, version) {
 					logger.LogIf(ctx, fmt.Errorf("found %s/%s (%s) expired object based on ILM rules, skipping and scheduled for deletion", bi.Name, version.Name, version.VersionID))
@@ -824,6 +817,22 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				var failure, ignore bool
 				// gr.Close() is ensured by decommissionObject().
 				for try := 0; try < 3; try++ {
+					if version.IsRemote() {
+						stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
+						if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
+							VersionID:   version.VersionID,
+							MTime:       version.ModTime,
+							UserDefined: version.Metadata,
+						}); err != nil {
+							stopFn(err)
+							failure = true
+							logger.LogIf(ctx, err)
+							continue
+						}
+						stopFn(nil)
+						failure = false
+						break
+					}
 					gr, err := set.GetObjectNInfo(ctx,
 						bi.Name,
 						encodeDirObject(version.Name),
@@ -898,8 +907,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			defer wg.Done()
 			err := set.listObjectsToDecommission(ctx, bi,
 				func(entry metaCacheEntry) {
-					parallelWorkers <- struct{}{}
+					// Wait must be synchronized here.
 					wg.Add(1)
+					parallelWorkers <- struct{}{}
 					go decommissionEntry(entry)
 				},
 			)
@@ -1270,17 +1280,6 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, indices ...i
 		z.HealBucket(ctx, bucket.Name, madmin.HealOpts{})
 	}
 
-	// TODO: Support decommissioning transition tiers.
-	for _, bucket := range decomBuckets {
-		if lc, err := globalLifecycleSys.Get(bucket.Name); err == nil {
-			if lc.HasTransition() {
-				return decomError{
-					Err: fmt.Sprintf("Bucket is part of transitioned tier %s: decommission is not allowed in Tier'd setups", bucket.Name),
-				}
-			}
-		}
-	}
-
 	// Create .minio.sys/conifg, .minio.sys/buckets paths if missing,
 	// this code is present to avoid any missing meta buckets on other
 	// pools.

cmd/erasure-server-pool-rebalance.go

@@ -32,7 +32,6 @@ import (
 
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v2"
-	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
@@ -463,17 +462,13 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 			}
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
-			event := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
-			switch action := event.Action; action {
-			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
-				// Skip this entry.
-				return true
-			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-				globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
-				// Skip this entry.
+
+			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			if evt.Action.Delete() {
+				globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
 				return true
 			}
+
 			return false
 		}
 

cmd/erasure-server-pool.go

@@ -37,7 +37,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -56,9 +55,6 @@ type erasureServerPools struct {
 
 	serverPools []*erasureSets
 
-	// Shut down async operations
-	shutdown context.CancelFunc
-
 	// Active decommission canceler
 	decommissionCancelers []context.CancelFunc
 
@@ -160,14 +156,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		break
 	}
 
-	drives := make([]string, 0, len(localDrives))
-	for _, localDrive := range localDrives {
-		drives = append(drives, localDrive.Endpoint().Path)
-	}
-
+	globalLocalDrivesMu.Lock()
 	globalLocalDrives = localDrives
-	ctx, z.shutdown = context.WithCancel(ctx)
-	go intDataUpdateTracker.start(ctx, drives...)
+	defer globalLocalDrivesMu.Unlock()
 	return z, nil
 }
 
@@ -355,11 +346,17 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 	g.Wait()
 
 	for i, zinfo := range storageInfos {
-		var available uint64
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(zinfo, size) {
+		if zinfo == nil {
 			serverPools[i] = poolAvailableSpace{Index: i}
 			continue
 		}
+		var available uint64
+		if !isMinioMetaBucketName(bucket) {
+			if avail, err := hasSpaceFor(zinfo, size); err != nil && !avail {
+				serverPools[i] = poolAvailableSpace{Index: i}
+				continue
+			}
+		}
 		var maxUsedPct int
 		for _, disk := range zinfo {
 			if disk == nil || disk.Total == 0 {
@@ -431,36 +428,48 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
 		return mtime1.After(mtime2)
 	})
 
+	defPool := PoolObjInfo{Index: -1}
 	for _, pinfo := range poolObjInfos {
 		// skip all objects from suspended pools if asked by the
 		// caller.
-		if z.IsSuspended(pinfo.Index) && opts.SkipDecommissioned {
+		if opts.SkipDecommissioned && z.IsSuspended(pinfo.Index) {
 			continue
 		}
 		// Skip object if it's from pools participating in a rebalance operation.
 		if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) {
 			continue
 		}
-
-		if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
+		if pinfo.Err == nil {
+			// found a pool
+			return pinfo, nil
+		}
+		if isErrReadQuorum(pinfo.Err) {
+			// read quorum is returned when the object is visibly
+			// present but its unreadable, we simply ask the writes to
+			// schedule to this pool instead. If there is no quorum
+			// it will fail anyways, however if there is quorum available
+			// with enough disks online but sufficiently inconsistent to
+			// break parity threshold, allow them to be overwritten
+			// or allow new versions to be added.
+			return pinfo, nil
+		}
+		defPool = pinfo
+		if !isErrObjectNotFound(pinfo.Err) {
 			return pinfo, pinfo.Err
 		}
 
-		if isErrObjectNotFound(pinfo.Err) {
-			// No object exists or its a delete marker,
-			// check objInfo to confirm.
-			if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
-				return pinfo, nil
-			}
-
-			// objInfo is not valid, truly the object doesn't
-			// exist proceed to next pool.
-			continue
+		// No object exists or its a delete marker,
+		// check objInfo to confirm.
+		if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
+			return pinfo, nil
 		}
-
-		return pinfo, nil
 	}
-
+	if opts.ReplicationRequest && opts.DeleteMarker && defPool.Index >= 0 {
+		// If the request is a delete marker replication request, return a default pool
+		// in cases where the object does not exist.
+		// This is to ensure that the delete marker is replicated to the destination.
+		return defPool, nil
+	}
 	return PoolObjInfo{}, toObjectErr(errFileNotFound, bucket, object)
 }
 
@@ -524,8 +533,6 @@ func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object stri
 }
 
 func (z *erasureServerPools) Shutdown(ctx context.Context) error {
-	defer z.shutdown()
-
 	g := errgroup.WithNErrs(len(z.serverPools))
 
 	for index := range z.serverPools {
@@ -594,7 +601,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context) StorageInfo {
 	return globalNotificationSys.StorageInfo(z)
 }
 
-func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
+func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
 	// Updates must be closed before we return.
 	defer close(updates)
 
@@ -639,7 +646,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 					}
 				}()
 				// Start scanner. Blocks until done.
-				err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates, healScanMode)
+				err := erObj.nsScanner(ctx, allBuckets, wantCycle, updates, healScanMode)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					mu.Lock()
@@ -701,9 +708,11 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 	case updateCloser <- ch:
 		<-ch
 	case <-ctx.Done():
+		mu.Lock()
 		if firstErr == nil {
 			firstErr = ctx.Err()
 		}
+		defer mu.Unlock()
 	}
 	return firstErr
 }
@@ -712,8 +721,6 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 // even if one of the sets fail to create buckets, we proceed all the successful
 // operations.
 func (z *erasureServerPools) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
-	defer NSUpdated(bucket, slashSeparator)
-
 	// Verify if bucket is valid.
 	if !isMinioMetaBucketName(bucket) {
 		if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
@@ -940,8 +947,15 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 	object = encodeDirObject(object)
 
 	if z.SinglePool() {
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), data.Size()) {
-			return ObjectInfo{}, toObjectErr(errDiskFull)
+		if !isMinioMetaBucketName(bucket) {
+			avail, err := hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), data.Size())
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return ObjectInfo{}, toObjectErr(errErasureWriteQuorum)
+			}
+			if !avail {
+				return ObjectInfo{}, toObjectErr(errDiskFull)
+			}
 		}
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
 	}
@@ -999,8 +1013,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	gopts.NoLock = true
 	pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts)
 	if err != nil {
-		switch err.(type) {
-		case InsufficientReadQuorum:
+		if _, ok := err.(InsufficientReadQuorum); ok {
 			return objInfo, InsufficientWriteQuorum{}
 		}
 		return objInfo, err
@@ -1207,6 +1220,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	if marker == "" && versionMarker != "" {
 		return loi, NotImplemented{}
 	}
+
 	opts := listPathOptions{
 		Bucket:      bucket,
 		Prefix:      prefix,
@@ -1217,6 +1231,17 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 		AskDisks:    globalAPIConfig.getListQuorum(),
 		Versioned:   true,
 	}
+
+	// Shortcut for APN/1.0 Veeam/1.0 Backup/10.0
+	// It requests unique blocks with a specific prefix.
+	// We skip scanning the parent directory for
+	// more objects matching the prefix.
+	ri := logger.GetReqInfo(ctx)
+	if ri != nil && strings.Contains(ri.UserAgent, `1.0 Veeam/1.0 Backup`) && strings.HasSuffix(prefix, ".blk") {
+		opts.BaseDir = prefix
+		opts.Transient = true
+	}
+
 	// set bucket metadata in opts
 	opts.setBucketMeta(ctx)
 
@@ -1301,11 +1326,12 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		if err == nil {
 			if opts.Lifecycle != nil {
 				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
-				switch evt.Action {
-				case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-					fallthrough
-				case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-					return loi, nil
+				if evt.Action.Delete() {
+					globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
+					if !evt.Action.DeleteRestored() {
+						// Skip entry if ILM action was DeleteVersionAction or DeleteAction
+						return loi, nil
+					}
 				}
 			}
 			loi.Objects = append(loi.Objects, objInfo)
@@ -1393,8 +1419,15 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}
 
 	if z.SinglePool() {
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), -1) {
-			return nil, toObjectErr(errDiskFull)
+		if !isMinioMetaBucketName(bucket) {
+			avail, err := hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), -1)
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return nil, toObjectErr(errErasureWriteQuorum)
+			}
+			if !avail {
+				return nil, toObjectErr(errDiskFull)
+			}
 		}
 		return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
 	}
@@ -1454,8 +1487,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 		if err == nil {
 			return pi, nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			// Look for information on the next pool
 			continue
 		}
@@ -1486,8 +1518,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 		if err == nil {
 			return mi, nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			// upload id not found, continue to the next pool.
 			continue
 		}
@@ -1518,8 +1549,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		if err == nil {
 			return result, nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			continue
 		}
 		return ListPartsInfo{}, err
@@ -1549,8 +1579,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		if err == nil {
 			return nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			// upload id not found move to next pool
 			continue
 		}
@@ -1581,8 +1610,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 		if err == nil {
 			return objInfo, nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			// upload id not found move to next pool
 			continue
 		}
@@ -1625,7 +1653,6 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 		return BucketNameInvalid{Bucket: bucket}
 	}
 
-	defer NSUpdated(bucket, slashSeparator)
 	if !opts.NoLock {
 		// Lock the bucket name before creating.
 		lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")
@@ -1638,15 +1665,14 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 	}
 
 	err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
-	if err == nil || errors.Is(err, errVolumeNotFound) {
+	if err == nil || isErrBucketNotFound(err) {
 		// If site replication is configured, hold on to deleted bucket state until sites sync
-		switch opts.SRDeleteOp {
-		case MarkDelete:
+		if opts.SRDeleteOp == MarkDelete {
 			z.s3Peer.MakeBucket(context.Background(), pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket), MakeBucketOptions{})
 		}
 	}
 
-	if err != nil && !errors.Is(err, errVolumeNotFound) {
+	if err != nil && !isErrBucketNotFound(err) {
 		if !opts.NoRecreate {
 			z.s3Peer.MakeBucket(ctx, bucket, MakeBucketOptions{})
 		}
@@ -1745,8 +1771,7 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
 	for _, pool := range z.serverPools {
 		result, err := pool.HealBucket(ctx, bucket, opts)
 		if err != nil {
-			switch err.(type) {
-			case BucketNotFound:
+			if _, ok := err.(BucketNotFound); ok {
 				continue
 			}
 			return result, err
@@ -2079,9 +2104,14 @@ func (z *erasureServerPools) getPoolAndSet(id string) (poolIdx, setIdx, diskIdx
 	return -1, -1, -1, fmt.Errorf("DriveID(%s) %w", id, errDiskNotFound)
 }
 
+const (
+	vmware = "VMWare"
+)
+
 // HealthOptions takes input options to return sepcific information
 type HealthOptions struct {
-	Maintenance bool
+	Maintenance    bool
+	DeploymentType string
 }
 
 // HealthResult returns the current state of the system, also
@@ -2167,7 +2197,8 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	}
 
 	var aggHealStateResult madmin.BgHealState
-	if opts.Maintenance {
+	// Check if disks are healing on in-case of VMware vsphere deployments.
+	if opts.Maintenance && opts.DeploymentType == vmware {
 		// check if local disks are being healed, if they are being healed
 		// we need to tell healthy status as 'false' so that this server
 		// is not taken down for maintenance
@@ -2350,3 +2381,27 @@ func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, ob
 	}
 	return nil
 }
+
+// DecomTieredObject - moves tiered object to another pool during decommissioning.
+func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
+	object = encodeDirObject(object)
+	if z.SinglePool() {
+		return fmt.Errorf("error decommissioning %s/%s", bucket, object)
+	}
+	if !opts.NoLock {
+		ns := z.NewNSLock(bucket, object)
+		lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return err
+		}
+		ctx = lkctx.Context()
+		defer ns.Unlock(lkctx)
+		opts.NoLock = true
+	}
+	idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size)
+	if err != nil {
+		return err
+	}
+
+	return z.serverPools[idx].DecomTieredObject(ctx, bucket, object, fi, opts)
+}

cmd/erasure-sets.go

@@ -1215,6 +1215,12 @@ func (s *erasureSets) PutObjectMetadata(ctx context.Context, bucket, object stri
 	return er.PutObjectMetadata(ctx, bucket, object, opts)
 }
 
+// DecomTieredObject - moves tiered object to another pool during decommissioning.
+func (s *erasureSets) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
+	er := s.getHashedSet(object)
+	return er.DecomTieredObject(ctx, bucket, object, fi, opts)
+}
+
 // PutObjectTags - replace or add tags to an existing object
 func (s *erasureSets) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) {
 	er := s.getHashedSet(object)

cmd/erasure.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
-	"github.com/minio/pkg/console"
 )
 
 // list all errors that can be ignore in a bucket operation.
@@ -208,6 +207,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				Scanning:       info.Scanning,
 				State:          diskErrToDriveState(err),
 				FreeInodes:     info.FreeInodes,
+				UsedInodes:     info.UsedInodes,
 			}
 			di.PoolIndex, di.SetIndex, di.DiskIndex = disks[index].GetDiskLoc()
 			if info.Healing {
@@ -351,7 +351,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 
 // nsScanner will start scanning buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
-func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
+func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
 	if len(buckets) == 0 {
 		return nil
 	}
@@ -377,7 +377,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 		},
 		Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)),
 	}
-	bloom := bf.bytes()
 
 	// Put all buckets into channel.
 	bucketCh := make(chan BucketInfo, len(buckets))
@@ -414,9 +413,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 
 		for {
 			select {
-			case <-ctx.Done():
-				// Return without saving.
-				return
 			case <-t.C:
 				if cache.Info.LastUpdate.Equal(lastSave) {
 					continue
@@ -454,6 +450,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 	// Start one scanner per disk
 	var wg sync.WaitGroup
 	wg.Add(len(disks))
+
 	for i := range disks {
 		go func(i int) {
 			defer wg.Done()
@@ -473,7 +470,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				if cache.Info.Name == "" {
 					cache.Info.Name = bucket.Name
 				}
-				cache.Info.BloomFilter = bloom
 				cache.Info.SkipHealing = healing
 				cache.Info.NextCycle = wantCycle
 				if cache.Info.Name != bucket.Name {
@@ -491,13 +487,13 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				go func(name string) {
 					defer wg.Done()
 					for update := range updates {
-						bucketResults <- dataUsageEntryInfo{
+						select {
+						case <-ctx.Done():
+						case bucketResults <- dataUsageEntryInfo{
 							Name:   name,
 							Parent: dataUsageRoot,
 							Entry:  update,
-						}
-						if intDataUpdateTracker.debug {
-							console.Debugln("z:", er.poolIndex, "s:", er.setIndex, "bucket", name, "got update", update)
+						}:
 						}
 					}
 				}(cache.Info.Name)
@@ -505,7 +501,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				before := cache.Info.LastUpdate
 				var err error
 				cache, err = disk.NSScanner(ctx, cache, updates, healScanMode)
-				cache.Info.BloomFilter = nil
 				if err != nil {
 					if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {
 						logger.LogIf(ctx, cache.save(ctx, er, cacheName))
@@ -525,15 +520,16 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				if r := cache.root(); r != nil {
 					root = cache.flatten(*r)
 				}
-				t := time.Now()
-				bucketResults <- dataUsageEntryInfo{
+				select {
+				case <-ctx.Done():
+					return
+				case bucketResults <- dataUsageEntryInfo{
 					Name:   cache.Info.Name,
 					Parent: dataUsageRoot,
 					Entry:  root,
+				}:
 				}
-				// We want to avoid synchronizing up all writes in case
-				// the results are piled up.
-				time.Sleep(time.Duration(float64(time.Since(t)) * rand.Float64()))
+
 				// Save cache
 				logger.LogIf(ctx, cache.save(ctx, er, cacheName))
 			}

cmd/event-notification.go

@@ -101,7 +101,7 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
 		return errServerNotInitialized
 	}
 
-	if err := evnot.targetList.Add(globalConfigTargetList.Targets()...); err != nil {
+	if err := evnot.targetList.Add(globalNotifyTargetList.Targets()...); err != nil {
 		return err
 	}
 
@@ -248,6 +248,7 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 
 	respElements := map[string]string{
 		"x-amz-request-id": args.RespElements["requestId"],
+		"x-amz-id-2":       args.RespElements["nodeId"],
 		"x-minio-origin-endpoint": func() string {
 			if globalMinioEndpoint != "" {
 				return globalMinioEndpoint
@@ -255,17 +256,18 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 			return getAPIEndpoints()[0]
 		}(), // MinIO specific custom elements.
 	}
-	// Add deployment as part of
-	if globalDeploymentID != "" {
-		respElements["x-minio-deployment-id"] = globalDeploymentID
-	}
+
+	// Add deployment as part of response elements.
+	respElements["x-minio-deployment-id"] = globalDeploymentID
 	if args.RespElements["content-length"] != "" {
 		respElements["content-length"] = args.RespElements["content-length"]
 	}
+
 	keyName := args.Object.Name
 	if escape {
 		keyName = url.QueryEscape(args.Object.Name)
 	}
+
 	newEvent := event.Event{
 		EventVersion:      "2.0",
 		EventSource:       "minio:s3",
@@ -312,12 +314,13 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 }
 
 func sendEvent(args eventArgs) {
-	args.Object.Size, _ = args.Object.GetActualSize()
-
 	// avoid generating a notification for REPLICA creation event.
 	if _, ok := args.ReqParams[xhttp.MinIOSourceReplicationRequest]; ok {
 		return
 	}
+
+	args.Object.Size, _ = args.Object.GetActualSize()
+
 	// remove sensitive encryption entries in metadata.
 	crypto.RemoveSensitiveEntries(args.Object.UserDefined)
 	crypto.RemoveInternalEntries(args.Object.UserDefined)

cmd/generic-handlers.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"net/http"
 	"path"
+	"path/filepath"
 	"runtime/debug"
 	"strings"
 	"sync/atomic"
@@ -35,7 +36,6 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/http/stats"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
 )
@@ -111,6 +111,7 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrUnsupportedMetadata), r.URL)
 			return
 		}
@@ -121,6 +122,7 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsHeader, 1)
 			return
@@ -133,9 +135,8 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 
 // Reserved bucket.
 const (
-	minioReservedBucket              = "minio"
-	minioReservedBucketPath          = SlashSeparator + minioReservedBucket
-	minioReservedBucketPathWithSlash = SlashSeparator + minioReservedBucket + SlashSeparator
+	minioReservedBucket     = "minio"
+	minioReservedBucketPath = SlashSeparator + minioReservedBucket
 
 	loginPathPrefix = SlashSeparator + "login"
 )
@@ -277,60 +278,6 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
 	return time.Time{}, ErrMissingDateHeader
 }
 
-// splitStr splits a string into n parts, empty strings are added
-// if we are not able to reach n elements
-func splitStr(path, sep string, n int) []string {
-	splits := strings.SplitN(path, sep, n)
-	// Add empty strings if we found elements less than nr
-	for i := n - len(splits); i > 0; i-- {
-		splits = append(splits, "")
-	}
-	return splits
-}
-
-func url2Bucket(p string) (bucket string) {
-	tokens := splitStr(p, SlashSeparator, 3)
-	return tokens[1]
-}
-
-// setHttpStatsHandler sets a http Stats handler to gather HTTP statistics
-func setHTTPStatsHandler(h http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Meters s3 connection stats.
-		meteredRequest := &stats.IncomingTrafficMeter{ReadCloser: r.Body}
-		meteredResponse := &stats.OutgoingTrafficMeter{ResponseWriter: w}
-
-		// Execute the request
-		r.Body = meteredRequest
-		h.ServeHTTP(meteredResponse, r)
-
-		if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
-			strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
-			strings.HasPrefix(r.URL.Path, peerS3Prefix) ||
-			strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
-			globalConnStats.incInputBytes(meteredRequest.BytesRead())
-			globalConnStats.incOutputBytes(meteredResponse.BytesWritten())
-			return
-		}
-		if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
-			globalConnStats.incAdminInputBytes(meteredRequest.BytesRead())
-			globalConnStats.incAdminOutputBytes(meteredResponse.BytesWritten())
-			return
-		}
-
-		globalConnStats.incS3InputBytes(meteredRequest.BytesRead())
-		globalConnStats.incS3OutputBytes(meteredResponse.BytesWritten())
-
-		if r.URL != nil {
-			bucket := url2Bucket(r.URL.Path)
-			if bucket != "" && bucket != minioReservedBucket {
-				globalBucketConnStats.incS3InputBytes(bucket, meteredRequest.BytesRead())
-				globalBucketConnStats.incS3OutputBytes(bucket, meteredResponse.BytesWritten())
-			}
-		}
-	})
-}
-
 // Bad path components to be rejected by the path validity handler.
 const (
 	dotdotComponent = ".."
@@ -349,7 +296,7 @@ func hasBadHost(host string) error {
 // Check if the incoming path has bad path components,
 // such as ".." and "."
 func hasBadPathComponent(path string) bool {
-	path = strings.TrimSpace(path)
+	path = filepath.ToSlash(strings.TrimSpace(path)) // For windows '\' must be converted to '/'
 	for _, p := range strings.Split(path, SlashSeparator) {
 		switch strings.TrimSpace(p) {
 		case dotdotComponent:
@@ -388,6 +335,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (%s)", invalidReq.Description, err)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -402,6 +350,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 			return
@@ -415,6 +364,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 						tc.ResponseRecorder.LogErrBody = true
 					}
 
+					defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 					writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 					atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 					return
@@ -427,6 +377,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (request has multiple authentication types, please use one)", invalidReq.Description)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -441,6 +392,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 					tc.FuncName = "handler.ValidRequest"
 					tc.ResponseRecorder.LogErrBody = true
 				}
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL)
 				return
 			}
@@ -453,6 +405,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = false
 				}
 
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest))
 			} else {
 				if ok {
@@ -460,6 +413,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = true
 				}
 
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL)
 			}
 			return
@@ -510,6 +464,7 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 		}
 		sr, err := globalDNSConfig.Get(bucket)
 		if err != nil {
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			if err == dns.ErrNoEntriesFound {
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrNoSuchBucket), r.URL)
 			} else {
@@ -552,9 +507,9 @@ func addCustomHeaders(h http.Handler) http.Handler {
 		// Set custom headers such as x-amz-request-id for each request.
 		w.Header().Set(xhttp.AmzRequestID, mustGetRequestID(UTCNow()))
 		if globalLocalNodeName != "" {
-			w.Header().Set(xhttp.AmzRequestNodeID, globalLocalNodeNameHex)
+			w.Header().Set(xhttp.AmzRequestHostID, globalLocalNodeNameHex)
 		}
-		h.ServeHTTP(xhttp.NewResponseRecorder(w), r)
+		h.ServeHTTP(w, r)
 	})
 }
 
@@ -592,6 +547,7 @@ func setUploadForwardingHandler(h http.Handler) http.Handler {
 			h.ServeHTTP(w, r)
 			return
 		}
+
 		bucket, object := request2BucketObjectName(r)
 		uploadID := r.Form.Get(xhttp.UploadID)
 
@@ -608,6 +564,7 @@ func setUploadForwardingHandler(h http.Handler) http.Handler {
 			}
 			// forward request to peer handling this upload
 			if globalBucketTargetSys.isOffline(remote.EndpointURL) {
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrReplicationRemoteConnectionError), r.URL)
 				return
 			}