Add minio_bucket_usage_version_total metric to Prometheus (#17023)

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>

.github/workflows/go-cross.yml

@@ -20,11 +20,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
-      - uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/go-fips.yml

@@ -20,21 +20,28 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
       - name: Setup dockerfile for build test
         run: |
-          echo "FROM golang:1.19.4" >> Dockerfile.fips.test
-          echo "COPY . /minio" >> Dockerfile.fips.test
-          echo "WORKDIR /minio" >> Dockerfile.fips.test
-          echo "ENV GOEXPERIMENT=boringcrypto" >> Dockerfile.fips.test
-          echo "RUN make" >> Dockerfile.fips.test
+          GO_VERSION=$(go version | cut -d ' ' -f 3 | sed 's/go//')
+          echo Detected go version $GO_VERSION
+          cat > Dockerfile.fips.test <<EOF
+          FROM golang:${GO_VERSION}
+          COPY . /minio
+          WORKDIR /minio
+          ENV GOEXPERIMENT=boringcrypto
+          RUN make
+          EOF
 
       - name: Build
         uses: docker/build-push-action@v3
@@ -49,4 +56,4 @@ jobs:
       - name: Test binary
         run: |
           docker run --rm minio/fips-test:latest ./minio --version
-          docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio' | grep -q FIPS
+          docker run --rm -i minio/fips-test:latest /bin/bash -c 'go tool nm ./minio | grep FIPS | grep -q FIPS'

.github/workflows/go-healing.yml

@@ -20,10 +20,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/go-lint.yml

@@ -20,10 +20,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest, windows-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/go.yml

@@ -20,10 +20,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/iam-integrations.yaml

@@ -61,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]
@@ -75,7 +75,7 @@ jobs:
             openid: "http://127.0.0.1:5556/dex"
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/markdown-lint.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     - name: Lint all docs
       run: |

.github/workflows/mint.yml

@@ -0,0 +1,57 @@
+name: Mint Tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  mint-test:
+    runs-on: mint
+    timeout-minutes: 120
+    steps:
+      - name: cleanup #https://github.com/actions/checkout/issues/273
+        run: |
+          sudo -S rm -rf ${GITHUB_WORKSPACE}
+          mkdir ${GITHUB_WORKSPACE}
+      - name: checkout-step
+        uses: actions/checkout@v3
+
+      - name: setup-go-step
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.20.x
+
+      - name: github sha short
+        id: vars
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: build-minio
+        run: |
+          make install
+          docker build . -t "minio/minio:${{ steps.vars.outputs.sha_short }}"
+
+      - name: compress and encrypt
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "compress-encrypt" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+
+      - name: multiple pools
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "pools" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+
+      - name: standalone erasure
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+          docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
+
+
+

.github/workflows/mint/minio-compress-encrypt.yaml

@@ -0,0 +1,79 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_COMPRESS: "true"
+    MINIO_COMPRESS_MIMETYPES: "*"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - cdata1-1:/cdata1
+      - cdata1-2:/cdata2
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - cdata2-1:/cdata1
+      - cdata2-2:/cdata2
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - cdata3-1:/cdata1
+      - cdata3-2:/cdata2
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - cdata4-1:/cdata1
+      - cdata4-2:/cdata2
+
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  cdata1-1:
+  cdata1-2:
+  cdata2-1:
+  cdata2-2:
+  cdata3-1:
+  cdata3-2:
+  cdata4-1:
+  cdata4-2:

.github/workflows/mint/minio-erasure.yaml

@@ -0,0 +1,53 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" edata{1...4}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_COMPRESS: "true"
+    MINIO_COMPRESS_MIMETYPES: "*"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - edata1-1:/edata1
+      - edata1-2:/edata2
+      - edata1-3:/edata3
+      - edata1-4:/edata4
+
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-1-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  edata1-1:
+  edata1-2:
+  edata1-3:
+  edata1-4:

.github/workflows/mint/minio-pools.yaml

@@ -0,0 +1,117 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: minio/minio:${JOB_NAME}
+  command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
+  expose:
+    - "9000"
+    - "9001"
+  environment:
+    MINIO_CI_CD: "on"
+    MINIO_ROOT_USER: "minio"
+    MINIO_ROOT_PASSWORD: "minio123"
+    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+  healthcheck:
+    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+    interval: 30s
+    timeout: 20s
+    retries: 3
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  minio1:
+    <<: *minio-common
+    hostname: minio1
+    volumes:
+      - pdata1-1:/pdata1
+      - pdata1-2:/pdata2
+
+  minio2:
+    <<: *minio-common
+    hostname: minio2
+    volumes:
+      - pdata2-1:/pdata1
+      - pdata2-2:/pdata2
+
+  minio3:
+    <<: *minio-common
+    hostname: minio3
+    volumes:
+      - pdata3-1:/pdata1
+      - pdata3-2:/pdata2
+
+  minio4:
+    <<: *minio-common
+    hostname: minio4
+    volumes:
+      - pdata4-1:/pdata1
+      - pdata4-2:/pdata2
+
+  minio5:
+    <<: *minio-common
+    hostname: minio5
+    volumes:
+      - pdata5-1:/pdata1
+      - pdata5-2:/pdata2
+
+  minio6:
+    <<: *minio-common
+    hostname: minio6
+    volumes:
+      - pdata6-1:/pdata1
+      - pdata6-2:/pdata2
+
+  minio7:
+    <<: *minio-common
+    hostname: minio7
+    volumes:
+      - pdata7-1:/pdata1
+      - pdata7-2:/pdata2
+
+  minio8:
+    <<: *minio-common
+    hostname: minio8
+    volumes:
+      - pdata8-1:/pdata1
+      - pdata8-2:/pdata2
+      
+  nginx:
+    image: nginx:1.19.2-alpine
+    hostname: nginx
+    volumes:
+      - ./nginx-8-node.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    depends_on:
+      - minio1
+      - minio2
+      - minio3
+      - minio4
+      - minio5
+      - minio6
+      - minio7
+      - minio8
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  pdata1-1:
+  pdata1-2:
+  pdata2-1:
+  pdata2-2:
+  pdata3-1:
+  pdata3-2:
+  pdata4-1:
+  pdata4-2:
+  pdata5-1:
+  pdata5-2:
+  pdata6-1:
+  pdata6-2:
+  pdata7-1:
+  pdata7-2:
+  pdata8-1:
+  pdata8-2:

.github/workflows/mint/nginx-1-node.conf

@@ -0,0 +1,100 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/mint/nginx-4-node.conf

@@ -0,0 +1,106 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/mint/nginx-8-node.conf

@@ -0,0 +1,114 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+        server minio5:9000;
+        server minio6:9000;
+        server minio7:9000;
+        server minio8:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+        server minio5:9001;
+        server minio6:9001;
+        server minio7:9001;
+        server minio8:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/mint/nginx.conf

@@ -0,0 +1,106 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server minio1:9000;
+        server minio2:9000;
+        server minio3:9000;
+        server minio4:9000;
+    }
+
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
+    server {
+        listen       9000;
+        listen  [::]:9000;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            
+            # To support websocket
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+            
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
+}

.github/workflows/replication.yaml

@@ -21,10 +21,10 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/run-mint.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+
+set -ex
+
+export MODE="$1"
+export ACCESS_KEY="$2"
+export SECRET_KEY="$3"
+export JOB_NAME="$4"
+export MINT_MODE="full"
+
+docker system prune -f
+docker volume prune -f
+
+## change working directory
+cd .github/workflows/mint
+
+docker-compose -f minio-${MODE}.yaml up -d
+sleep 5m
+
+docker run --rm --net=host \
+       --name="mint-${MODE}-${JOB_NAME}" \
+       -e SERVER_ENDPOINT="127.0.0.1:9000" \
+       -e ACCESS_KEY="${ACCESS_KEY}" \
+       -e SECRET_KEY="${SECRET_KEY}" \
+       -e ENABLE_HTTPS=0 \
+       -e MINT_MODE="${MINT_MODE}" \
+       docker.io/minio/mint:edge \
+	 aws-sdk-go   \
+	 aws-sdk-java \
+	 aws-sdk-php  \
+	 aws-sdk-ruby \
+	 awscli       \
+	 healthcheck  \
+	 mc           \
+	 minio-go     \
+	 minio-java   \
+	 minio-js     \
+	 minio-py     \
+	 s3cmd        \
+	 s3select     \
+	 versioning
+
+docker-compose -f minio-${MODE}.yaml down || true
+sleep 10s
+
+docker system prune -f || true
+docker volume prune -f || true
+
+## change working directory
+cd ../../../
+

.github/workflows/upgrade-ci-cd.yaml

@@ -20,11 +20,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.19.x]
+        go-version: [1.20.x]
         os: [ubuntu-latest]
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}

.github/workflows/vulncheck.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19
+          go-version: 1.20.x
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.gitignore

@@ -38,3 +38,4 @@ docs/debugging/s3-check-md5/s3-check-md5
 docs/debugging/hash-set/hash-set
 docs/debugging/healing-bin/healing-bin
 docs/debugging/inspect/inspect
+.bin/

.golangci.yml

@@ -1,39 +1,34 @@
 linters-settings:
   gofumpt:
-    lang-version: "1.18"
+    simplify: true
 
   misspell:
     locale: US
 
+  staticcheck:
+    checks: ['all', '-ST1005', '-ST1000', '-SA4000', '-SA9004', '-SA1019', '-SA1008', '-U1000', '-ST1016']
+
 linters:
   disable-all: true
   enable:
-    - typecheck
-    - goimports
-    - misspell
-    - govet
-    - revive
-    - ineffassign
-    - gomodguard
+    - durationcheck
+    - gocritic
     - gofmt
+    - gofumpt
+    - goimports
+    - gomodguard
+    - govet
+    - ineffassign
+    - misspell
+    - revive
+    - staticcheck
+    - tenv
+    - typecheck
     - unconvert
     - unused
-    - gocritic
-    - gofumpt
-    - tenv
-    - durationcheck
 
 issues:
   exclude-use-default: false
   exclude:
-      - should have a package comment
-      - error strings should not be capitalized or end with punctuation or a newline
-      # todo fix these when we get enough time.
-      - "singleCaseSwitch: should rewrite switch statement to if statement"
-      - "unlambda: replace"
-      - "captLocal:"
-      - "ifElseChain:"
-      - "elseif:"
-
-service:
-  golangci-lint-version: 1.43.0 # use the fixed version to not introduce new linters unexpectedly
+    - should have a package comment
+    - error strings should not be capitalized or end with punctuation or a newline

Makefile

@@ -8,6 +8,10 @@ GOOS := $(shell go env GOOS)
 VERSION ?= $(shell git describe --tags)
 TAG ?= "minio/minio:$(VERSION)"
 
+GOLANGCI_VERSION = v1.51.2
+GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
+GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
+
 all: build
 
 checks: ## check dependencies
@@ -19,24 +23,26 @@ help: ## print this help
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION)
 	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
-	@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
 
 crosscompile: ## cross compile minio
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps lint check-gen
+verifiers: lint check-gen
 
 check-gen: ## check for updated autogenerated files
 	@go generate ./... >/dev/null
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 
-lint: ## runs golangci-lint suite of linters
+lint: getdeps ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
-	@${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
-	@${GOPATH}/bin/staticcheck --tests=false ./...
+	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
+
+lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
+	@echo "Running $@ check"
+	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
 
 check: test
 test: verifiers build ## builds minio, runs linters, tests

README.md

@@ -86,8 +86,6 @@ chmod +x minio
 ./minio server /data
 ```
 
-Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
-
 The following table lists supported architectures. Replace the `wget` URL with the architecture for your Linux host.
 
 | Architecture                   | URL                                                        |

buildscripts/race.sh

@@ -5,7 +5,6 @@ set -e
 export GORACE="history_size=7"
 export MINIO_API_REQUESTS_MAX=10000
 
-## TODO remove `dsync` from race detector once this is merged and released https://go-review.googlesource.com/c/go/+/333529/
-for d in $(go list ./... | grep -v dsync); do
+for d in $(go list ./...); do
     CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
 done

buildscripts/verify-healing-with-root-disks.sh

@@ -45,16 +45,18 @@ function start_minio() {
 
 # Prepare fake disks with losetup
 function prepare_block_devices() {
-        mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
-        for i in 1 2 3 4; do
-                dd if=/dev/zero of=${WORK_DIR}/disks/img.$i bs=1M count=2048
-                mkfs.ext4 -F ${WORK_DIR}/disks/img.$i
-                sudo mknod /dev/minio-loopdisk$i b 7 $[256-$i]
-                sudo losetup /dev/minio-loopdisk$i ${WORK_DIR}/disks/img.$i
-                mkdir -p ${WORK_DIR}/mnt/disk$i/
-                sudo mount /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
-                sudo chown "$(id -u):$(id -g)" /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
-        done
+    set -e
+    mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
+    sudo modprobe loop
+    for i in 1 2 3 4; do
+        dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000
+        device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i})
+        sudo mkfs.ext4 -F ${device}
+        mkdir -p ${WORK_DIR}/mnt/disk${i}/
+        sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/
+        sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/
+    done
+    set +e
 }
 
 # Start a distributed MinIO setup, unmount one disk and check if it is formatted

cmd/admin-bucket-handlers.go

@@ -686,8 +686,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			continue
 		}
 		bucket, fileName := slc[0], slc[1]
-		switch fileName {
-		case objectLockConfig:
+		if fileName == objectLockConfig {
 			reader, err := file.Open()
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)
@@ -756,8 +755,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			continue
 		}
 		bucket, fileName := slc[0], slc[1]
-		switch fileName {
-		case bucketVersioningConfig:
+		if fileName == bucketVersioningConfig {
 			reader, err := file.Open()
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)

cmd/admin-handlers-idp-config.go

@@ -86,7 +86,7 @@ func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.Resp
 		if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default {
 			// LDAP does not support multiple configurations. So cfgName must be
 			// empty or `madmin.Default`.
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPNonDefaultConfigName), r.URL)
 			return
 		}
 	}

cmd/admin-handlers-users-race_test.go

@@ -27,12 +27,12 @@ import (
 	"context"
 	"fmt"
 	"runtime"
-	"sync"
 	"testing"
 	"time"
 
 	"github.com/minio/madmin-go/v2"
 	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio/internal/sync/errgroup"
 )
 
 func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
@@ -129,18 +129,21 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
 		secretKeys[i] = secretKey
 	}
 
-	wg := sync.WaitGroup{}
+	g := errgroup.Group{}
 	for i := 0; i < userCount; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
-			err := s.adm.RemoveUser(ctx, accessKeys[i])
-			if err != nil {
-				c.Fatalf("unable to remove user: %v", err)
+		g.Go(func(i int) func() error {
+			return func() error {
+				uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "")
+				err := s.adm.RemoveUser(ctx, accessKeys[i])
+				if err != nil {
+					return err
+				}
+				c.mustNotListObjects(ctx, uClient, bucket)
+				return nil
 			}
-			c.mustNotListObjects(ctx, uClient, bucket)
-		}(i)
+		}(i), i)
+	}
+	if errs := g.Wait(); len(errs) > 0 {
+		c.Fatalf("unable to remove users: %v", errs)
 	}
-	wg.Wait()
 }

cmd/admin-handlers-users.go

@@ -431,7 +431,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Not allowed to add a user with same access key as root credential
-	if owner && accessKey == cred.AccessKey {
+	if accessKey == globalActiveCred.AccessKey {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
 		return
 	}
@@ -582,6 +582,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		AccountStatus: stsAccount.Status,
 		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
+		Expiration:    &stsAccount.Expiration,
 	}
 
 	data, err := json.Marshal(infoResp)
@@ -1332,11 +1333,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			// Fetch the data usage of the current bucket
 			var size uint64
 			var objectsCount uint64
-			var objectsHist map[string]uint64
+			var objectsHist, versionsHist map[string]uint64
 			if !dataUsageInfo.LastUpdate.IsZero() {
 				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
 				objectsCount = dataUsageInfo.BucketsUsage[bucket.Name].ObjectsCount
 				objectsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectSizesHistogram
+				versionsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectVersionsHistogram
 			}
 			// Fetch the prefix usage of the current bucket
 			var prefixUsage map[string]uint64
@@ -1350,12 +1352,13 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			tcfg, _, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name)
 
 			acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
-				Name:                 bucket.Name,
-				Created:              bucket.Created,
-				Size:                 size,
-				Objects:              objectsCount,
-				ObjectSizesHistogram: objectsHist,
-				PrefixUsage:          prefixUsage,
+				Name:                    bucket.Name,
+				Created:                 bucket.Created,
+				Size:                    size,
+				Objects:                 objectsCount,
+				ObjectSizesHistogram:    objectsHist,
+				ObjectVersionsHistogram: versionsHist,
+				PrefixUsage:             prefixUsage,
 				Details: &madmin.BucketDetails{
 					Versioning:          globalBucketVersioningSys.Enabled(bucket.Name),
 					VersioningSuspended: globalBucketVersioningSys.Suspended(bucket.Name),
@@ -1954,7 +1957,7 @@ func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 			UserOrGroup: userOrGroup,
 			UserType:    int(userType),
 			IsGroup:     isGroup,
-			Policy:      strings.Join(policiesToDetach, ","),
+			Policy:      newPolicies,
 		},
 		UpdatedAt: updatedAt,
 	}))
@@ -2295,7 +2298,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 			for accessKey, ureq := range userAccts {
 				// Not allowed to add a user with same access key as root credential
-				if owner && accessKey == cred.AccessKey {
+				if accessKey == globalActiveCred.AccessKey {
 					writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL)
 					return
 				}

cmd/admin-handlers-users_test.go

@@ -825,7 +825,7 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
 	if set.CreateStringSet(groups...).Contains(group) {
 		c.Fatalf("created group still present!")
 	}
-	groupInfo, err = s.adm.GetGroupDescription(ctx, group)
+	_, err = s.adm.GetGroupDescription(ctx, group)
 	if err == nil {
 		c.Fatalf("group appears to exist")
 	}

cmd/admin-handlers.go

@@ -533,16 +533,19 @@ func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string)
 func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 	now := time.Now().UTC()
 	entryMap := make(map[string]*madmin.LockEntry)
+	toEntry := func(lri lockRequesterInfo) string {
+		return fmt.Sprintf("%s/%s", lri.Name, lri.UID)
+	}
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
 			continue
 		}
 		for k, v := range peerLock.Locks {
 			for _, lockReqInfo := range v {
-				if val, ok := entryMap[lockReqInfo.Name]; ok {
+				if val, ok := entryMap[toEntry(lockReqInfo)]; ok {
 					val.ServerList = append(val.ServerList, peerLock.Addr)
 				} else {
-					entryMap[lockReqInfo.Name] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr)
+					entryMap[toEntry(lockReqInfo)] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr)
 				}
 			}
 		}
@@ -1526,6 +1529,11 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Publish bootstrap events that have already occurred before client could subscribe.
+	if traceOpts.TraceTypes().Contains(madmin.TraceBootstrap) {
+		go globalBootstrapTracer.Publish(ctx, globalTrace)
+	}
+
 	for _, peer := range peers {
 		if peer == nil {
 			continue
@@ -2429,7 +2437,7 @@ func assignPoolNumbers(servers []madmin.ServerProperties) {
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
-	for _, tgt := range globalConfigTargetList.Targets() {
+	for _, tgt := range globalNotifyTargetList.Targets() {
 		targetIDStatus := make(map[string]madmin.Status)
 		active, _ := tgt.IsActive()
 		targetID := tgt.ID()

cmd/admin-handlers_test.go

@@ -21,10 +21,12 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"sort"
 	"sync"
 	"testing"
 	"time"
@@ -380,3 +382,146 @@ func TestExtractHealInitParams(t *testing.T) {
 		}
 	}
 }
+
+type byResourceUID struct{ madmin.LockEntries }
+
+func (b byResourceUID) Less(i, j int) bool {
+	toUniqLock := func(entry madmin.LockEntry) string {
+		return fmt.Sprintf("%s/%s", entry.Resource, entry.ID)
+	}
+	return toUniqLock(b.LockEntries[i]) < toUniqLock(b.LockEntries[j])
+}
+
+func TestTopLockEntries(t *testing.T) {
+	locksHeld := make(map[string][]lockRequesterInfo)
+	var owners []string
+	for i := 0; i < 4; i++ {
+		owners = append(owners, fmt.Sprintf("node-%d", i))
+	}
+
+	// Simulate DeleteObjects of 10 objects in a single request. i.e same lock
+	// request UID, but 10 different resource names associated with it.
+	var lris []lockRequesterInfo
+	uuid := mustGetUUID()
+	for i := 0; i < 10; i++ {
+		resource := fmt.Sprintf("bucket/delete-object-%d", i)
+		lri := lockRequesterInfo{
+			Name:   resource,
+			Writer: true,
+			UID:    uuid,
+			Owner:  owners[i%len(owners)],
+			Group:  true,
+			Quorum: 3,
+		}
+		lris = append(lris, lri)
+		locksHeld[resource] = []lockRequesterInfo{lri}
+	}
+
+	// Add a few concurrent read locks to the mix
+	for i := 0; i < 50; i++ {
+		resource := fmt.Sprintf("bucket/get-object-%d", i)
+		lri := lockRequesterInfo{
+			Name:   resource,
+			UID:    mustGetUUID(),
+			Owner:  owners[i%len(owners)],
+			Quorum: 2,
+		}
+		lris = append(lris, lri)
+		locksHeld[resource] = append(locksHeld[resource], lri)
+		// concurrent read lock, same resource different uid
+		lri.UID = mustGetUUID()
+		lris = append(lris, lri)
+		locksHeld[resource] = append(locksHeld[resource], lri)
+	}
+
+	var peerLocks []*PeerLocks
+	for _, owner := range owners {
+		peerLocks = append(peerLocks, &PeerLocks{
+			Addr:  owner,
+			Locks: locksHeld,
+		})
+	}
+	var exp madmin.LockEntries
+	for _, lri := range lris {
+		lockType := func(lri lockRequesterInfo) string {
+			if lri.Writer {
+				return "WRITE"
+			}
+			return "READ"
+		}
+		exp = append(exp, madmin.LockEntry{
+			Resource:   lri.Name,
+			Type:       lockType(lri),
+			ServerList: owners,
+			Owner:      lri.Owner,
+			ID:         lri.UID,
+			Quorum:     lri.Quorum,
+		})
+	}
+
+	testCases := []struct {
+		peerLocks []*PeerLocks
+		expected  madmin.LockEntries
+	}{
+		{
+			peerLocks: peerLocks,
+			expected:  exp,
+		},
+	}
+
+	// printEntries := func(entries madmin.LockEntries) {
+	// 	for i, entry := range entries {
+	// 		fmt.Printf("%d: %s %s %s %s %v %d\n", i, entry.Resource, entry.ID, entry.Owner, entry.Type, entry.ServerList, entry.Elapsed)
+	// 	}
+	// }
+
+	check := func(exp, got madmin.LockEntries) (int, bool) {
+		if len(exp) != len(got) {
+			return 0, false
+		}
+		sort.Sort(byResourceUID{exp})
+		sort.Sort(byResourceUID{got})
+		// printEntries(exp)
+		// printEntries(got)
+		for i, e := range exp {
+			if !e.Timestamp.Equal(got[i].Timestamp) {
+				return i, false
+			}
+			// Skip checking elapsed since it's time sensitive.
+			// if e.Elapsed != got[i].Elapsed {
+			// 	return false
+			// }
+			if e.Resource != got[i].Resource {
+				return i, false
+			}
+			if e.Type != got[i].Type {
+				return i, false
+			}
+			if e.Source != got[i].Source {
+				return i, false
+			}
+			if e.Owner != got[i].Owner {
+				return i, false
+			}
+			if e.ID != got[i].ID {
+				return i, false
+			}
+			if len(e.ServerList) != len(got[i].ServerList) {
+				return i, false
+			}
+			for j := range e.ServerList {
+				if e.ServerList[j] != got[i].ServerList[j] {
+					return i, false
+				}
+			}
+		}
+		return 0, true
+	}
+
+	for i, tc := range testCases {
+		got := topLockEntries(tc.peerLocks, false)
+		if idx, ok := check(tc.expected, got); !ok {
+			t.Fatalf("%d: mismatch at %d \n expected %#v but got %#v", i, idx, tc.expected[idx], got[idx])
+		}
+	}
+}

cmd/admin-router.go

@@ -237,6 +237,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
 			gz(httpTraceHdrs(adminAPI.DescribeBatchJob)))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/cancel-job").HandlerFunc(
+			gz(httpTraceHdrs(adminAPI.CancelBatchJob)))
 
 		// Bucket migration operations
 		// ExportBucketMetaHandler

cmd/api-errors.go

@@ -38,10 +38,12 @@ import (
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/versioning"
+	levent "github.com/minio/minio/internal/config/lambda/event"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/pkg/bucket/policy"
@@ -266,6 +268,7 @@ const (
 	ErrAdminNoSuchUser
 	ErrAdminNoSuchGroup
 	ErrAdminGroupNotEmpty
+	ErrAdminGroupDisabled
 	ErrAdminNoSuchJob
 	ErrAdminNoSuchPolicy
 	ErrAdminPolicyChangeAlreadyApplied
@@ -279,6 +282,7 @@ const (
 	ErrAdminConfigEnvOverridden
 	ErrAdminConfigDuplicateKeys
 	ErrAdminConfigInvalidIDPType
+	ErrAdminConfigLDAPNonDefaultConfigName
 	ErrAdminConfigLDAPValidation
 	ErrAdminConfigIDPCfgNameAlreadyExists
 	ErrAdminConfigIDPCfgNameDoesNotExist
@@ -410,6 +414,10 @@ const (
 
 	ErrInvalidChecksum
 
+	// Lambda functions
+	ErrLambdaARNInvalid
+	ErrLambdaARNNotFound
+
 	apiErrCodeEnd // This is used only for the testing code
 )
 
@@ -424,8 +432,7 @@ func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError
 		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
 	}
 	if globalSite.Region != "" {
-		switch errCode {
-		case ErrAuthorizationHeaderMalformed:
+		if errCode == ErrAuthorizationHeaderMalformed {
 			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
 			return apiErr
 		}
@@ -1261,6 +1268,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The specified group is not empty - cannot remove it.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminGroupDisabled: {
+		Code:           "XMinioAdminGroupDisabled",
+		Description:    "The specified group is disabled.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminNoSuchPolicy: {
 		Code:           "XMinioAdminNoSuchPolicy",
 		Description:    "The canned policy does not exist.",
@@ -1323,6 +1335,11 @@ var errorCodes = errorCodeMap{
 		Description:    fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigLDAPNonDefaultConfigName: {
+		Code:           "XMinioAdminConfigLDAPNonDefaultConfigName",
+		Description:    "Only a single LDAP configuration is supported - config name must be empty or `_`",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigLDAPValidation: {
 		Code:           "XMinioAdminConfigLDAPValidation",
 		Description:    "LDAP Configuration validation failed",
@@ -1959,6 +1976,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid checksum provided.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrLambdaARNInvalid: {
+		Code:           "LambdaARNInvalid",
+		Description:    "The specified lambda ARN is invalid",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrLambdaARNNotFound: {
+		Code:           "LambdaARNNotFound",
+		Description:    "The specified lambda ARN does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrPolicyAlreadyAttached: {
 		Code:           "XMinioPolicyAlreadyAttached",
 		Description:    "The specified policy is already attached.",
@@ -1982,15 +2009,15 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 
 	// Only return ErrClientDisconnected if the provided context is actually canceled.
 	// This way downstream context.Canceled will still report ErrOperationTimedOut
-	if contextCanceled(ctx) {
-		if ctx.Err() == context.Canceled {
-			return ErrClientDisconnected
-		}
+	if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
+		return ErrClientDisconnected
 	}
 
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
+	case errNoSuchPolicy:
+		apiErr = ErrAdminNoSuchPolicy
 	case errNoSuchUser:
 		apiErr = ErrAdminNoSuchUser
 	case errNoSuchServiceAccount:
@@ -2019,6 +2046,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidSecretKey
 	case errInvalidStorageClass:
 		apiErr = ErrInvalidStorageClass
+	case errErasureReadQuorum:
+		apiErr = ErrSlowDown
+	case errErasureWriteQuorum:
+		apiErr = ErrSlowDown
 	// SSE errors
 	case errInvalidEncryptionParameters:
 		apiErr = ErrInvalidEncryptionParameters
@@ -2066,16 +2097,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectLockInvalidHeaders
 	case objectlock.ErrMalformedXML:
 		apiErr = ErrMalformedXML
-	default:
-		switch {
-		case errors.Is(err, errNoSuchPolicy):
-			apiErr = ErrAdminNoSuchPolicy
-		}
 	}
 
 	// Compression errors
-	switch err {
-	case errInvalidDecompressedSize:
+	if err == errInvalidDecompressedSize {
 		apiErr = ErrInvalidDecompressedSize
 	}
 
@@ -2086,7 +2111,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 
 	// etcd specific errors, a key is always a bucket for us return
 	// ErrNoSuchBucket in such a case.
-	if err == dns.ErrNoEntriesFound {
+	if errors.Is(err, dns.ErrNoEntriesFound) {
 		return ErrNoSuchBucket
 	}
 
@@ -2211,6 +2236,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrARNNotification
 	case *event.ErrARNNotFound:
 		apiErr = ErrARNNotification
+	case *levent.ErrInvalidARN:
+		apiErr = ErrLambdaARNInvalid
+	case *levent.ErrARNNotFound:
+		apiErr = ErrLambdaARNNotFound
 	case *event.ErrUnknownRegion:
 		apiErr = ErrRegionNotification
 	case *event.ErrInvalidFilterName:
@@ -2274,38 +2303,27 @@ func toAPIError(ctx context.Context, err error) APIError {
 	}
 
 	apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
-	e, ok := err.(dns.ErrInvalidBucketName)
-	if ok {
-		code := toAPIErrorCode(ctx, e)
-		apiErr = errorCodes.ToAPIErrWithErr(code, e)
-	}
-
-	if apiErr.Code == "NotImplemented" {
-		switch e := err.(type) {
-		case NotImplemented:
-			desc := e.Error()
-			if desc == "" {
-				desc = apiErr.Description
-			}
-			apiErr = APIError{
-				Code:           apiErr.Code,
-				Description:    desc,
-				HTTPStatusCode: apiErr.HTTPStatusCode,
-			}
-			return apiErr
+	switch apiErr.Code {
+	case "NotImplemented":
+		desc := fmt.Sprintf("%s (%v)", apiErr.Description, err)
+		apiErr = APIError{
+			Code:           apiErr.Code,
+			Description:    desc,
+			HTTPStatusCode: apiErr.HTTPStatusCode,
 		}
-	}
-
-	if apiErr.Code == "XMinioBackendDown" {
+	case "XMinioBackendDown":
 		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
-		return apiErr
-	}
-
-	if apiErr.Code == "InternalError" {
+	case "InternalError":
 		// If we see an internal error try to interpret
 		// any underlying errors if possible depending on
 		// their internal error types.
 		switch e := err.(type) {
+		case kms.Error:
+			apiErr = APIError{
+				Description:    e.Err.Error(),
+				Code:           e.APICode,
+				HTTPStatusCode: e.HTTPStatusCode,
+			}
 		case batchReplicationJobError:
 			apiErr = APIError(e)
 		case InvalidArgument:
@@ -2316,22 +2334,20 @@ func toAPIError(ctx context.Context, err error) APIError {
 			}
 		case *xml.SyntaxError:
 			apiErr = APIError{
-				Code: "MalformedXML",
-				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description,
-					e.Error()),
+				Code:           "MalformedXML",
+				Description:    fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, e),
 				HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
 			}
 		case url.EscapeError:
 			apiErr = APIError{
-				Code: "XMinioInvalidObjectName",
-				Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description,
-					e.Error()),
+				Code:           "XMinioInvalidObjectName",
+				Description:    fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description, e),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
 		case versioning.Error:
 			apiErr = APIError{
 				Code:           "IllegalVersioningConfigurationException",
-				Description:    fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e.Error()),
+				Description:    fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
 		case lifecycle.Error:
@@ -2397,19 +2413,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 			// Add more other SDK related errors here if any in future.
 		default:
 			//nolint:gocritic
-			if errors.Is(err, errMalformedEncoding) {
-				apiErr = APIError{
-					Code:           "BadRequest",
-					Description:    err.Error(),
-					HTTPStatusCode: http.StatusBadRequest,
-				}
-			} else if errors.Is(err, errChunkTooBig) {
-				apiErr = APIError{
-					Code:           "BadRequest",
-					Description:    err.Error(),
-					HTTPStatusCode: http.StatusBadRequest,
-				}
-			} else if errors.Is(err, strconv.ErrRange) {
+			if errors.Is(err, errMalformedEncoding) || errors.Is(err, errChunkTooBig) || errors.Is(err, strconv.ErrRange) {
 				apiErr = APIError{
 					Code:           "BadRequest",
 					Description:    err.Error(),

cmd/api-response.go

@@ -35,6 +35,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 	xxml "github.com/minio/xxml"
 )
 
@@ -359,6 +360,7 @@ type Object struct {
 
 	// UserMetadata user-defined metadata
 	UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
+	UserTags     string    `xml:"UserTags,omitempty"`
 }
 
 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -492,7 +494,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 }
 
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
-func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
+func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
 
 	owner := Owner{
@@ -500,11 +502,19 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		DisplayName: "minio",
 	}
 	data := ListVersionsResponse{}
+	var lastObjMetaName string
+	var tagErr, metaErr APIErrorCode = -1, -1
 
 	for _, object := range resp.Objects {
 		if object.Name == "" {
 			continue
 		}
+		// Cache checks for the same object
+		if metadata != nil && lastObjMetaName != object.Name {
+			tagErr = metadata(object.Name, policy.GetObjectTaggingAction)
+			metaErr = metadata(object.Name, policy.GetObjectAction)
+			lastObjMetaName = object.Name
+		}
 		content := ObjectVersion{}
 		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
@@ -517,6 +527,32 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		} else {
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
+		if tagErr == ErrNone {
+			content.UserTags = object.UserTags
+		}
+		if metaErr == ErrNone {
+			content.UserMetadata = &Metadata{}
+			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
+			case crypto.S3:
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+			case crypto.S3KMS:
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
+			case crypto.SSEC:
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
+			}
+			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+					// Do not need to send any internal metadata
+					// values to client.
+					continue
+				}
+				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+					continue
+				}
+				content.UserMetadata.Set(k, v)
+			}
+		}
 		content.Owner = owner
 		content.VersionID = object.VersionID
 		if content.VersionID == "" {
@@ -600,7 +636,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata metaCheckFn) ListObjectsV2Response {
 	contents := make([]Object, 0, len(objects))
 	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
@@ -625,27 +661,32 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
 		content.Owner = owner
-		if metadata {
-			content.UserMetadata = &Metadata{}
-			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
-			case crypto.S3:
-				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
-			case crypto.S3KMS:
-				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
-			case crypto.SSEC:
-				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
+		if metadata != nil {
+			if metadata(object.Name, policy.GetObjectTaggingAction) == ErrNone {
+				content.UserTags = object.UserTags
 			}
-			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
-				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
-					// Do not need to send any internal metadata
-					// values to client.
-					continue
+			if metadata(object.Name, policy.GetObjectAction) == ErrNone {
+				content.UserMetadata = &Metadata{}
+				switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
+				case crypto.S3:
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+				case crypto.S3KMS:
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
+				case crypto.SSEC:
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 				}
-				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
-					continue
+				for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+					if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+						// Do not need to send any internal metadata
+						// values to client.
+						continue
+					}
+					// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+					if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+						continue
+					}
+					content.UserMetadata.Set(k, v)
 				}
-				content.UserMetadata.Set(k, v)
 			}
 		}
 		contents = append(contents, content)
@@ -673,6 +714,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	return data
 }
 
+type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
+
 // generates CopyObjectResponse from etag and lastModified time.
 func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
 	return CopyObjectResponse{

cmd/api-router.go

@@ -285,7 +285,10 @@ func registerAPIRouter(router *mux.Router) {
 		// GetObjectLegalHold
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
-		// GetObject - note gzip compression is *not* added due to Range requests.
+		// GetObject with lambda ARNs
+		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectLambdaHandler))))).Queries("lambdaArn", "{lambdaArn:.*}")
+		// GetObject
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("getobject", maxClients(gz(httpTraceHdrs(api.GetObjectHandler)))))
 		// CopyObject
@@ -389,6 +392,9 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
 		// ListObjectVersions
+		router.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true")
+		// ListObjectVersions
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
 		// GetBucketPolicyStatus
@@ -431,8 +437,9 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodHead).HandlerFunc(
 			collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler)))))
 		// PostPolicy
-		router.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
-			collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
+		router.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool {
+			return isRequestPostPolicySignatureV4(r)
+		}).HandlerFunc(collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
 		// DeleteMultipleObjects
 		router.Methods(http.MethodPost).HandlerFunc(
 			collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "")

cmd/api-utils.go

@@ -94,14 +94,8 @@ func s3URLEncode(s string) string {
 }
 
 // s3EncodeName encodes string in response when encodingType is specified in AWS S3 requests.
-func s3EncodeName(name string, encodingType string) (result string) {
-	// Quick path to exit
-	if encodingType == "" {
-		return name
-	}
-	encodingType = strings.ToLower(encodingType)
-	switch encodingType {
-	case "url":
+func s3EncodeName(name, encodingType string) string {
+	if strings.ToLower(encodingType) == "url" {
 		return s3URLEncode(name)
 	}
 	return name

cmd/auth-handler.go

@@ -25,6 +25,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"io"
+	"mime"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -74,8 +75,11 @@ func isRequestPresignedSignatureV2(r *http.Request) bool {
 
 // Verify if request has AWS Post policy Signature Version '4'.
 func isRequestPostPolicySignatureV4(r *http.Request) bool {
-	return strings.Contains(r.Header.Get(xhttp.ContentType), "multipart/form-data") &&
-		r.Method == http.MethodPost
+	mediaType, _, err := mime.ParseMediaType(r.Header.Get(xhttp.ContentType))
+	if err != nil {
+		return false
+	}
+	return mediaType == "multipart/form-data" && r.Method == http.MethodPost
 }
 
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
@@ -154,6 +158,10 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 		return cred, owner, s3Err
 	}
 
+	logger.GetReqInfo(ctx).Cred = cred
+	logger.GetReqInfo(ctx).Owner = owner
+	logger.GetReqInfo(ctx).Region = globalSite.Region
+
 	return cred, owner, ErrNone
 }
 
@@ -203,7 +211,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 	// that clients cannot decode the token using the temp
 	// secret keys and generate an entirely new claim by essentially
 	// hijacking the policies. We need to make sure that this is
-	// based an admin credential such that token cannot be decoded
+	// based on admin credential such that token cannot be decoded
 	// on the client side and is treated like an opaque value.
 	claims, err := auth.ExtractClaims(token, secret)
 	if err != nil {
@@ -344,6 +352,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
 
 	logger.GetReqInfo(ctx).Cred = cred
 	logger.GetReqInfo(ctx).Owner = owner
+	logger.GetReqInfo(ctx).Region = globalSite.Region
 
 	// region is valid only for CreateBucketAction.
 	var region string
@@ -585,6 +594,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 				// All our internal APIs are sensitive towards Date
 				// header, for all requests where Date header is not
 				// present we will reject such clients.
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -598,6 +608,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = true
 				}
 
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -613,6 +624,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 			tc.ResponseRecorder.LogErrBody = true
 		}
 
+		defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
 		atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1)
 	})

cmd/background-newdisks-heal-ops.go

@@ -271,6 +271,9 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
+	globalLocalDrivesMu.RLock()
+	globalLocalDrives := globalLocalDrives
+	globalLocalDrivesMu.RUnlock()
 	for _, disk := range globalLocalDrives {
 		_, err := disk.GetDiskID()
 		if errors.Is(err, errUnformattedDisk) {
@@ -450,7 +453,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 					globalBackgroundHealState.setDiskHealingStatus(disk, true)
 					if err := healFreshDisk(ctx, z, disk); err != nil {
 						globalBackgroundHealState.setDiskHealingStatus(disk, false)
-						if !errors.Is(err, context.Canceled) {
+						timedout := OperationTimedOut{}
+						if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) {
 							printEndpointError(disk, err, false)
 						}
 						return

cmd/batch-handlers.go

@@ -28,6 +28,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"path"
 	"runtime"
 	"strconv"
 	"strings"
@@ -37,10 +38,14 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/workers"
@@ -189,6 +194,11 @@ type BatchJobReplicateCredentials struct {
 	SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"`
 }
 
+// Empty indicates if credentials are not set
+func (c BatchJobReplicateCredentials) Empty() bool {
+	return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == ""
+}
+
 // Validate validates if credentials are valid
 func (c BatchJobReplicateCredentials) Validate() error {
 	if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) {
@@ -227,6 +237,11 @@ type BatchJobReplicateV1 struct {
 	clnt *miniogo.Core `msg:"-"`
 }
 
+// RemoteToLocal returns true if source is remote and target is local
+func (r BatchJobReplicateV1) RemoteToLocal() bool {
+	return !r.Source.Creds.Empty()
+}
+
 // BatchJobRequest this is an internal data structure not for external consumption.
 type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
@@ -234,6 +249,8 @@ type BatchJobRequest struct {
 	Started   time.Time            `yaml:"-" json:"started"`
 	Location  string               `yaml:"-" json:"location"`
 	Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"`
+	KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"`
+	ctx       context.Context      `msg:"-"`
 }
 
 // Notify notifies notification endpoint if configured regarding job failure or success.
@@ -269,10 +286,360 @@ func (r BatchJobReplicateV1) Notify(ctx context.Context, body io.Reader) error {
 }
 
 // ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local.
-func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, c *miniogo.Core, srcObject string) error {
+func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool) error {
+	srcBucket := r.Source.Bucket
+	tgtBucket := r.Target.Bucket
+	srcObject := srcObjInfo.Name
+	tgtObject := srcObjInfo.Name
+	if r.Target.Prefix != "" {
+		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+	}
+
+	versioned := globalBucketVersioningSys.PrefixEnabled(tgtBucket, tgtObject)
+	versionSuspended := globalBucketVersioningSys.PrefixSuspended(tgtBucket, tgtObject)
+
+	if srcObjInfo.DeleteMarker {
+		_, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{
+			VersionID:          srcObjInfo.VersionID,
+			VersionSuspended:   versionSuspended,
+			Versioned:          versioned,
+			MTime:              srcObjInfo.ModTime,
+			DeleteMarker:       srcObjInfo.DeleteMarker,
+			ReplicationRequest: true,
+		})
+		return err
+	}
+
+	opts := ObjectOptions{
+		VersionID:        srcObjInfo.VersionID,
+		Versioned:        versioned,
+		VersionSuspended: versionSuspended,
+		MTime:            srcObjInfo.ModTime,
+		PreserveETag:     srcObjInfo.ETag,
+		UserDefined:      srcObjInfo.UserDefined,
+	}
+	if crypto.S3.IsEncrypted(srcObjInfo.UserDefined) {
+		opts.ServerSideEncryption = encrypt.NewSSE()
+	}
+	slc := strings.Split(srcObjInfo.ETag, "-")
+	if len(slc) == 2 {
+		partsCount, err := strconv.Atoi(slc[1])
+		if err != nil {
+			return err
+		}
+		return r.copyWithMultipartfromSource(ctx, api, core, srcObjInfo, opts, partsCount)
+	}
+	gopts := minio.GetObjectOptions{
+		VersionID: srcObjInfo.VersionID,
+	}
+	if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil {
+		return err
+	}
+	rd, objInfo, _, err := core.GetObject(ctx, srcBucket, srcObject, gopts)
+	if err != nil {
+		return err
+	}
+	defer rd.Close()
+
+	hr, err := hash.NewReader(rd, objInfo.Size, "", "", objInfo.Size)
+	if err != nil {
+		return err
+	}
+	pReader := NewPutObjReader(hr)
+	_, err = api.PutObject(ctx, tgtBucket, tgtObject, pReader, opts)
+	return err
+}
+
+func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, opts ObjectOptions, partsCount int) (err error) {
+	srcBucket := r.Source.Bucket
+	tgtBucket := r.Target.Bucket
+	srcObject := srcObjInfo.Name
+	tgtObject := srcObjInfo.Name
+	if r.Target.Prefix != "" {
+		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+	}
+
+	var uploadedParts []CompletePart
+	res, err := api.NewMultipartUpload(context.Background(), tgtBucket, tgtObject, opts)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			// block and abort remote upload upon failure.
+			attempts := 1
+			for attempts <= 3 {
+				aerr := api.AbortMultipartUpload(ctx, tgtBucket, tgtObject, res.UploadID, ObjectOptions{})
+				if aerr == nil {
+					return
+				}
+				logger.LogIf(ctx,
+					fmt.Errorf("trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
+						humanize.Ordinal(attempts), res.UploadID, tgtBucket, tgtObject, aerr))
+				attempts++
+				time.Sleep(time.Second)
+			}
+		}
+	}()
+
+	var (
+		hr    *hash.Reader
+		pInfo PartInfo
+	)
+
+	for i := 0; i < partsCount; i++ {
+		gopts := minio.GetObjectOptions{
+			VersionID:  srcObjInfo.VersionID,
+			PartNumber: i + 1,
+		}
+		if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil {
+			return err
+		}
+		rd, objInfo, _, err := c.GetObject(ctx, srcBucket, srcObject, gopts)
+		if err != nil {
+			return err
+		}
+		defer rd.Close()
+
+		hr, err = hash.NewReader(rd, objInfo.Size, "", "", objInfo.Size)
+		if err != nil {
+			return err
+		}
+		pReader := NewPutObjReader(hr)
+		opts.PreserveETag = ""
+		pInfo, err = api.PutObjectPart(ctx, tgtBucket, tgtObject, res.UploadID, i+1, pReader, opts)
+		if err != nil {
+			return err
+		}
+		if pInfo.Size != objInfo.Size {
+			return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, objInfo.Size)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+	_, err = api.CompleteMultipartUpload(ctx, tgtBucket, tgtObject, res.UploadID, uploadedParts, opts)
+	return err
+}
+
+// StartFromSource starts the batch replication job from remote source, resumes if there was a pending job via "job.ID"
+func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	ri := &batchJobInfo{
+		JobID:     job.ID,
+		JobType:   string(job.Type()),
+		StartTime: job.Started,
+	}
+	if err := ri.load(ctx, api, job); err != nil {
+		return err
+	}
+	globalBatchJobsMetrics.save(job.ID, ri)
+
+	delay := job.Replicate.Flags.Retry.Delay
+	if delay == 0 {
+		delay = batchReplJobDefaultRetryDelay
+	}
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	skip := func(oi ObjectInfo) (ok bool) {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan {
+			// skip all objects that are newer than specified older duration
+			return true
+		}
+
+		if r.Flags.Filter.NewerThan > 0 && time.Since(oi.ModTime) >= r.Flags.Filter.NewerThan {
+			// skip all objects that are older than specified newer duration
+			return true
+		}
+
+		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(oi.ModTime) {
+			// skip all objects that are created before the specified time.
+			return true
+		}
+
+		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(oi.ModTime) {
+			// skip all objects that are created after the specified time.
+			return true
+		}
+		if len(r.Flags.Filter.Tags) > 0 {
+			// Only parse object tags if tags filter is specified.
+			tagMap := map[string]string{}
+			tagStr := oi.UserTags
+			if len(tagStr) != 0 {
+				t, err := tags.ParseObjectTags(tagStr)
+				if err != nil {
+					return false
+				}
+				tagMap = t.ToMap()
+			}
+			for _, kv := range r.Flags.Filter.Tags {
+				for t, v := range tagMap {
+					if kv.Match(BatchJobReplicateKV{Key: t, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided tags filter match skip the object
+			return false
+		}
+
+		if len(r.Flags.Filter.Metadata) > 0 {
+			for _, kv := range r.Flags.Filter.Metadata {
+				for k, v := range oi.UserDefined {
+					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
+						continue
+					}
+					// We only need to match x-amz-meta or standardHeaders
+					if kv.Match(BatchJobReplicateKV{Key: k, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided metadata filters match skip the object.
+			return false
+		}
+
+		return false
+	}
+
+	u, err := url.Parse(r.Source.Endpoint)
+	if err != nil {
+		return err
+	}
+
+	cred := r.Source.Creds
+
+	c, err := miniogo.New(u.Host, &miniogo.Options{
+		Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+		Secure:    u.Scheme == "https",
+		Transport: getRemoteInstanceTransport,
+	})
+	if err != nil {
+		return err
+	}
+
+	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
+	core := &minio.Core{Client: c}
+
+	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+	if err != nil {
+		return err
+	}
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		// invalid worker size.
+		return err
+	}
+
+	retryAttempts := ri.RetryAttempts
+	retry := false
+	for attempts := 1; attempts <= retryAttempts; attempts++ {
+		attempts := attempts
+
+		ctx, cancel := context.WithCancel(ctx)
+		objInfoCh := c.ListObjects(ctx, r.Source.Bucket, minio.ListObjectsOptions{
+			Prefix:       r.Source.Prefix,
+			WithVersions: true,
+			Recursive:    true,
+			WithMetadata: true,
+		})
+		for obj := range objInfoCh {
+			oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
+			if skip(oi) {
+				continue
+			}
+			wk.Take()
+			go func() {
+				defer wk.Give()
+				stopFn := globalBatchJobsMetrics.trace(batchReplicationMetricObject, job.ID, attempts, oi)
+				success := true
+				if err := r.ReplicateFromSource(ctx, api, core, oi, retry); err != nil {
+					// object must be deleted concurrently, allow these failures but do not count them
+					if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
+						return
+					}
+					stopFn(err)
+					logger.LogIf(ctx, err)
+					success = false
+				} else {
+					stopFn(nil)
+				}
+				ri.trackCurrentBucketObject(r.Target.Bucket, oi, success)
+				globalBatchJobsMetrics.save(job.ID, ri)
+				// persist in-memory state to disk after every 10secs.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+			}()
+		}
+		wk.Wait()
+
+		ri.RetryAttempts = attempts
+		ri.Complete = ri.ObjectsFailed == 0
+		ri.Failed = ri.ObjectsFailed > 0
+
+		globalBatchJobsMetrics.save(job.ID, ri)
+		// persist in-memory state to disk.
+		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+
+		buf, _ := json.Marshal(ri)
+		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+		}
+
+		cancel()
+		if ri.Failed {
+			ri.ObjectsFailed = 0
+			ri.Bucket = ""
+			ri.Object = ""
+			ri.Objects = 0
+			ri.BytesFailed = 0
+			ri.BytesTransferred = 0
+			retry = true // indicate we are retrying..
+			time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+			continue
+		}
+
+		break
+	}
+
 	return nil
 }
 
+// toObjectInfo converts minio.ObjectInfo to ObjectInfo
+func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo {
+	tags, _ := tags.MapToObjectTags(objInfo.UserTags)
+	oi := ObjectInfo{
+		Bucket:                    bucket,
+		Name:                      object,
+		ModTime:                   objInfo.LastModified,
+		Size:                      objInfo.Size,
+		ETag:                      objInfo.ETag,
+		VersionID:                 objInfo.VersionID,
+		IsLatest:                  objInfo.IsLatest,
+		DeleteMarker:              objInfo.IsDeleteMarker,
+		ContentType:               objInfo.ContentType,
+		Expires:                   objInfo.Expires,
+		StorageClass:              objInfo.StorageClass,
+		ReplicationStatusInternal: objInfo.ReplicationStatus,
+		UserTags:                  tags.String(),
+	}
+	oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
+	for k, v := range objInfo.Metadata {
+		oi.UserDefined[k] = v[0]
+	}
+	ce, ok := oi.UserDefined[xhttp.ContentEncoding]
+	if !ok {
+		ce, ok = oi.UserDefined[strings.ToLower(xhttp.ContentEncoding)]
+	}
+	if ok {
+		oi.ContentEncoding = ce
+	}
+	return oi
+}
+
 // ReplicateToTarget read from source and replicate to configured target
 func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *miniogo.Core, srcObjInfo ObjectInfo, retry bool) error {
 	srcBucket := r.Source.Bucket
@@ -399,14 +766,34 @@ const (
 )
 
 func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
-	data, err := readConfig(ctx, api, pathJoin(job.Location, batchReplName))
+	var fileName string
+	var format, version uint16
+	switch {
+	case job.Replicate != nil:
+		fileName = batchReplName
+		version = batchReplVersionV1
+		format = batchReplFormat
+	case job.KeyRotate != nil:
+		fileName = batchKeyRotationName
+		version = batchKeyRotateVersionV1
+		format = batchKeyRotationFormat
+
+	}
+	data, err := readConfig(ctx, api, pathJoin(job.Location, fileName))
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) {
-			ri.Version = batchReplVersionV1
-			if job.Replicate.Flags.Retry.Attempts > 0 {
-				ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
-			} else {
+			ri.Version = int(version)
+			switch {
+			case job.Replicate != nil:
 				ri.RetryAttempts = batchReplJobDefaultRetries
+				if job.Replicate.Flags.Retry.Attempts > 0 {
+					ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
+				}
+			case job.KeyRotate != nil:
+				ri.RetryAttempts = batchKeyRotateJobDefaultRetries
+				if job.KeyRotate.Flags.Retry.Attempts > 0 {
+					ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
+				}
 			}
 			return nil
 		}
@@ -417,18 +804,18 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 		return nil
 	}
 	if len(data) <= 4 {
-		return fmt.Errorf("batchRepl: no data")
+		return fmt.Errorf("%s: no data", ri.JobType)
 	}
 	// Read header
 	switch binary.LittleEndian.Uint16(data[0:2]) {
-	case batchReplFormat:
+	case format:
 	default:
-		return fmt.Errorf("batchRepl: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+		return fmt.Errorf("%s: unknown format: %d", ri.JobType, binary.LittleEndian.Uint16(data[0:2]))
 	}
 	switch binary.LittleEndian.Uint16(data[2:4]) {
-	case batchReplVersion:
+	case version:
 	default:
-		return fmt.Errorf("batchRepl: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+		return fmt.Errorf("%s: unknown version: %d", ri.JobType, binary.LittleEndian.Uint16(data[2:4]))
 	}
 
 	ri.mu.Lock()
@@ -442,7 +829,7 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 	switch ri.Version {
 	case batchReplVersionV1:
 	default:
-		return fmt.Errorf("unexpected batch repl meta version: %d", ri.Version)
+		return fmt.Errorf("unexpected batch %s meta version: %d", ri.JobType, ri.Version)
 	}
 
 	return nil
@@ -491,31 +878,51 @@ func (ri *batchJobInfo) countItem(size int64, dmarker, success bool) {
 	}
 }
 
-func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, duration time.Duration, jobLocation string) error {
+func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, duration time.Duration, job BatchJobRequest) error {
 	if ri == nil {
 		return errInvalidArgument
 	}
 	now := UTCNow()
 	ri.mu.Lock()
+	var (
+		format, version  uint16
+		jobTyp, fileName string
+	)
+
 	if now.Sub(ri.LastUpdate) >= duration {
+		switch job.Type() {
+		case madmin.BatchJobReplicate:
+			format = batchReplFormat
+			version = batchReplVersion
+			jobTyp = string(job.Type())
+			fileName = batchReplName
+			ri.Version = batchReplVersionV1
+		case madmin.BatchJobKeyRotate:
+			format = batchKeyRotationFormat
+			version = batchKeyRotateVersion
+			jobTyp = string(job.Type())
+			fileName = batchKeyRotationName
+			ri.Version = batchKeyRotateVersionV1
+		default:
+			return errInvalidArgument
+		}
 		if serverDebugLog {
-			console.Debugf("batchReplicate: persisting batchReplication info on drive: threshold:%s, batchRepl:%#v\n", now.Sub(ri.LastUpdate), ri)
+			console.Debugf("%s: persisting info on drive: threshold:%s, %s:%#v\n", jobTyp, now.Sub(ri.LastUpdate), jobTyp, ri)
 		}
 		ri.LastUpdate = now
-		ri.Version = batchReplVersionV1
 
 		data := make([]byte, 4, ri.Msgsize()+4)
 
 		// Initialize the header.
-		binary.LittleEndian.PutUint16(data[0:2], batchReplFormat)
-		binary.LittleEndian.PutUint16(data[2:4], batchReplVersion)
+		binary.LittleEndian.PutUint16(data[0:2], format)
+		binary.LittleEndian.PutUint16(data[2:4], version)
 
 		buf, err := ri.MarshalMsg(data)
 		ri.mu.Unlock()
 		if err != nil {
 			return err
 		}
-		return saveConfig(ctx, api, pathJoin(jobLocation, batchReplName), buf)
+		return saveConfig(ctx, api, pathJoin(job.Location, fileName), buf)
 	}
 	ri.mu.Unlock()
 	return nil
@@ -689,7 +1096,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job.Location))
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 			}()
 		}
 		wk.Wait()
@@ -700,11 +1107,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 		globalBatchJobsMetrics.save(job.ID, ri)
 		// persist in-memory state to disk.
-		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job.Location))
+		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
 		buf, _ := json.Marshal(ri)
 		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to notify %v", err))
+			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
 		cancel()
@@ -750,13 +1157,18 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	if r.Source.Bucket == "" {
 		return errInvalidArgument
 	}
-
-	info, err := o.GetBucketInfo(ctx, r.Source.Bucket, BucketOptions{})
+	var isRemoteToLocal bool
+	localBkt := r.Source.Bucket
+	if r.Source.Endpoint != "" {
+		localBkt = r.Target.Bucket
+		isRemoteToLocal = true
+	}
+	info, err := o.GetBucketInfo(ctx, localBkt, BucketOptions{})
 	if err != nil {
 		if isErrBucketNotFound(err) {
 			return batchReplicationJobError{
 				Code:           "NoSuchSourceBucket",
-				Description:    "The specified source bucket does not exist",
+				Description:    fmt.Sprintf("The specified bucket %s does not exist", localBkt),
 				HTTPStatusCode: http.StatusNotFound,
 			}
 		}
@@ -766,8 +1178,20 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	if err := r.Source.Type.Validate(); err != nil {
 		return err
 	}
+	if r.Source.Creds.Empty() && r.Target.Creds.Empty() {
+		return errInvalidArgument
+	}
 
-	if r.Target.Endpoint == "" {
+	if !r.Source.Creds.Empty() {
+		if err := r.Source.Creds.Validate(); err != nil {
+			return err
+		}
+	}
+	if r.Target.Endpoint == "" && !r.Target.Creds.Empty() {
+		return errInvalidArgument
+	}
+
+	if r.Source.Endpoint == "" && !r.Source.Creds.Empty() {
 		return errInvalidArgument
 	}
 
@@ -775,8 +1199,14 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		return errInvalidArgument
 	}
 
-	if err := r.Target.Creds.Validate(); err != nil {
-		return err
+	if !r.Target.Creds.Empty() {
+		if err := r.Target.Creds.Validate(); err != nil {
+			return err
+		}
+	}
+
+	if r.Source.Creds.Empty() && r.Target.Creds.Empty() {
+		return errInvalidArgument
 	}
 
 	if err := r.Target.Type.Validate(); err != nil {
@@ -799,13 +1229,21 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		return err
 	}
 
-	u, err := url.Parse(r.Target.Endpoint)
+	remoteEp := r.Target.Endpoint
+	remoteBkt := r.Target.Bucket
+	cred := r.Target.Creds
+
+	if r.Source.Endpoint != "" {
+		remoteEp = r.Source.Endpoint
+		cred = r.Source.Creds
+		remoteBkt = r.Source.Bucket
+	}
+
+	u, err := url.Parse(remoteEp)
 	if err != nil {
 		return err
 	}
 
-	cred := r.Target.Creds
-
 	c, err := miniogo.NewCore(u.Host, &miniogo.Options{
 		Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 		Secure:    u.Scheme == "https",
@@ -816,7 +1254,7 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	}
 	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
-	vcfg, err := c.GetBucketVersioning(ctx, r.Target.Bucket)
+	vcfg, err := c.GetBucketVersioning(ctx, remoteBkt)
 	if err != nil {
 		if miniogo.ToErrorResponse(err).Code == "NoSuchBucket" {
 			return batchReplicationJobError{
@@ -827,8 +1265,8 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		}
 		return err
 	}
-
-	if info.Versioning && !vcfg.Enabled() {
+	// If source has versioning enabled, target must have versioning enabled
+	if (info.Versioning && !vcfg.Enabled() && !isRemoteToLocal) || (!info.Versioning && vcfg.Enabled() && isRemoteToLocal) {
 		return batchReplicationJobError{
 			Code: "InvalidBucketState",
 			Description: fmt.Sprintf("The source '%s' has versioning enabled, target '%s' must have versioning enabled",
@@ -843,8 +1281,11 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 
 // Type returns type of batch job, currently only supports 'replicate'
 func (j BatchJobRequest) Type() madmin.BatchJobType {
-	if j.Replicate != nil {
+	switch {
+	case j.Replicate != nil:
 		return madmin.BatchJobReplicate
+	case j.KeyRotate != nil:
+		return madmin.BatchJobKeyRotate
 	}
 	return madmin.BatchJobType("unknown")
 }
@@ -852,22 +1293,28 @@ func (j BatchJobRequest) Type() madmin.BatchJobType {
 // Validate validates the current job, used by 'save()' before
 // persisting the job request
 func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
-	if j.Replicate != nil {
+	switch {
+	case j.Replicate != nil:
 		return j.Replicate.Validate(ctx, j, o)
+	case j.KeyRotate != nil:
+		return j.KeyRotate.Validate(ctx, j, o)
 	}
 	return errInvalidArgument
 }
 
 func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
-	if j.Replicate != nil {
+	switch {
+	case j.Replicate != nil:
 		deleteConfig(ctx, api, pathJoin(j.Location, batchReplName))
+	case j.KeyRotate != nil:
+		deleteConfig(ctx, api, pathJoin(j.Location, batchKeyRotationName))
 	}
 	globalBatchJobsMetrics.delete(j.ID)
 	deleteConfig(ctx, api, j.Location)
 }
 
 func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
-	if j.Replicate == nil {
+	if j.Replicate == nil && j.KeyRotate == nil {
 		return errInvalidArgument
 	}
 
@@ -1061,6 +1508,34 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 	writeSuccessResponseJSON(w, buf)
 }
 
+// CancelBatchJob cancels a job in progress
+func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "CancelBatchJob")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CancelBatchJobAction)
+	if objectAPI == nil {
+		return
+	}
+	jobID := r.Form.Get("id")
+	if jobID == "" {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL)
+		return
+	}
+	if err := globalBatchJobPool.canceler(jobID, true); err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
+		return
+	}
+	j := BatchJobRequest{
+		ID:       jobID,
+		Location: pathJoin(batchJobPrefix, jobID),
+	}
+	j.delete(ctx, objectAPI)
+
+	writeSuccessNoContent(w)
+}
+
 //msgp:ignore BatchJobPool
 
 // BatchJobPool batch job pool
@@ -1070,6 +1545,8 @@ type BatchJobPool struct {
 	once         sync.Once
 	mu           sync.Mutex
 	jobCh        chan *BatchJobRequest
+	jmu          sync.Mutex // protects jobCancelers
+	jobCancelers map[string]context.CancelFunc
 	workerKillCh chan struct{}
 	workerSize   int
 }
@@ -1083,6 +1560,7 @@ func newBatchJobPool(ctx context.Context, o ObjectLayer, workers int) *BatchJobP
 		objLayer:     o,
 		jobCh:        make(chan *BatchJobRequest, 10000),
 		workerKillCh: make(chan struct{}, workers),
+		jobCancelers: make(map[string]context.CancelFunc),
 	}
 	jpool.ResizeWorkers(workers)
 	jpool.resume()
@@ -1098,6 +1576,10 @@ func (j *BatchJobPool) resume() {
 		return
 	}
 	for result := range results {
+		// ignore batch-replicate.bin and batch-rotate.bin entries
+		if strings.HasSuffix(result.Name, slashSeparator) {
+			continue
+		}
 		req := &BatchJobRequest{}
 		if err := req.load(ctx, j.objLayer, result.Name); err != nil {
 			logger.LogIf(ctx, err)
@@ -1124,15 +1606,36 @@ func (j *BatchJobPool) AddWorker() {
 				return
 			}
 			if job.Replicate != nil {
-				if err := job.Replicate.Start(j.ctx, j.objLayer, *job); err != nil {
+				if job.Replicate.RemoteToLocal() {
+					if err := job.Replicate.StartFromSource(job.ctx, j.objLayer, *job); err != nil {
+						if !isErrBucketNotFound(err) {
+							logger.LogIf(j.ctx, err)
+							j.canceler(job.ID, false)
+							continue
+						}
+						// Bucket not found proceed to delete such a job.
+					}
+				} else {
+					if err := job.Replicate.Start(job.ctx, j.objLayer, *job); err != nil {
+						if !isErrBucketNotFound(err) {
+							logger.LogIf(j.ctx, err)
+							j.canceler(job.ID, false)
+							continue
+						}
+						// Bucket not found proceed to delete such a job.
+					}
+				}
+			}
+			if job.KeyRotate != nil {
+				if err := job.KeyRotate.Start(job.ctx, j.objLayer, *job); err != nil {
 					if !isErrBucketNotFound(err) {
 						logger.LogIf(j.ctx, err)
 						continue
 					}
-					// Bucket not found proceed to delete such a job.
 				}
 			}
 			job.delete(j.ctx, j.objLayer)
+			j.canceler(job.ID, false)
 		case <-j.workerKillCh:
 			return
 		}
@@ -1162,6 +1665,12 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
 	if j == nil {
 		return errInvalidArgument
 	}
+	jctx, jcancel := context.WithCancel(j.ctx)
+	j.jmu.Lock()
+	j.jobCancelers[req.ID] = jcancel
+	j.jmu.Unlock()
+	req.ctx = jctx
+
 	select {
 	case <-j.ctx.Done():
 		j.once.Do(func() {
@@ -1174,6 +1683,22 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
 	return nil
 }
 
+// delete canceler from the map, cancel job if requested
+func (j *BatchJobPool) canceler(jobID string, cancel bool) error {
+	if j == nil {
+		return errInvalidArgument
+	}
+	j.jmu.Lock()
+	defer j.jmu.Unlock()
+	if canceler, ok := j.jobCancelers[jobID]; ok {
+		if cancel {
+			canceler()
+		}
+	}
+	delete(j.jobCancelers, jobID)
+	return nil
+}
+
 //msgp:ignore batchJobMetrics
 type batchJobMetrics struct {
 	sync.RWMutex
@@ -1184,25 +1709,32 @@ var globalBatchJobsMetrics = batchJobMetrics{
 	metrics: make(map[string]*batchJobInfo),
 }
 
-//msgp:ignore batchReplicationMetric
-//go:generate stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric $GOFILE
-type batchReplicationMetric uint8
+//msgp:ignore batchJobMetric
+//go:generate stringer -type=batchJobMetric -trimprefix=batchJobMetric $GOFILE
+type batchJobMetric uint8
 
 const (
-	batchReplicationMetricObject batchReplicationMetric = iota
+	batchReplicationMetricObject batchJobMetric = iota
+	batchKeyRotationMetricObject
 )
 
-func batchReplicationTrace(d batchReplicationMetric, job string, startTime time.Time, duration time.Duration, info ObjectInfo, attempts int, err error) madmin.TraceInfo {
+func batchJobTrace(d batchJobMetric, job string, startTime time.Time, duration time.Duration, info ObjectInfo, attempts int, err error) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
 	}
-	funcName := fmt.Sprintf("batchReplication.%s (job-name=%s)", d.String(), job)
+	jobKind := "batchReplication"
+	traceType := madmin.TraceBatchReplication
+	if d == batchKeyRotationMetricObject {
+		jobKind = "batchKeyRotation"
+		traceType = madmin.TraceBatchKeyRotation
+	}
+	funcName := fmt.Sprintf("%s.%s (job-name=%s)", jobKind, d.String(), job)
 	if attempts > 0 {
-		funcName = fmt.Sprintf("batchReplication.%s (job-name=%s,attempts=%s)", d.String(), job, humanize.Ordinal(attempts))
+		funcName = fmt.Sprintf("%s.%s (job-name=%s,attempts=%s)", jobKind, d.String(), job, humanize.Ordinal(attempts))
 	}
 	return madmin.TraceInfo{
-		TraceType: madmin.TraceBatchReplication,
+		TraceType: traceType,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
 		FuncName:  funcName,
@@ -1234,6 +1766,12 @@ func (m *batchJobMetrics) report(jobID string) (metrics *madmin.BatchJobMetrics)
 				BytesTransferred: job.BytesTransferred,
 				BytesFailed:      job.BytesFailed,
 			},
+			KeyRotate: &madmin.KeyRotationInfo{
+				Bucket:        job.Bucket,
+				Object:        job.Object,
+				Objects:       job.Objects,
+				ObjectsFailed: job.ObjectsFailed,
+			},
 		}
 		if match {
 			break
@@ -1256,12 +1794,19 @@ func (m *batchJobMetrics) save(jobID string, ri *batchJobInfo) {
 	m.metrics[jobID] = ri.clone()
 }
 
-func (m *batchJobMetrics) trace(d batchReplicationMetric, job string, attempts int, info ObjectInfo) func(err error) {
+func (m *batchJobMetrics) trace(d batchJobMetric, job string, attempts int, info ObjectInfo) func(err error) {
 	startTime := time.Now()
 	return func(err error) {
 		duration := time.Since(startTime)
-		if globalTrace.NumSubscribers(madmin.TraceBatchReplication) > 0 {
-			globalTrace.Publish(batchReplicationTrace(d, job, startTime, duration, info, attempts, err))
+		switch d {
+		case batchReplicationMetricObject:
+			if globalTrace.NumSubscribers(madmin.TraceBatchReplication) > 0 {
+				globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err))
+			}
+		case batchKeyRotationMetricObject:
+			if globalTrace.NumSubscribers(madmin.TraceBatchKeyRotation) > 0 {
+				globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err))
+			}
 		}
 	}
 }

cmd/batch-handlers_gen.go

@@ -1538,6 +1538,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "KeyRotate":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "KeyRotate")
+					return
+				}
+				z.KeyRotate = nil
+			} else {
+				if z.KeyRotate == nil {
+					z.KeyRotate = new(BatchJobKeyRotateV1)
+				}
+				err = z.KeyRotate.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "KeyRotate")
+					return
+				}
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -1551,9 +1569,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 5
+	// map header, size 6
 	// write "ID"
-	err = en.Append(0x85, 0xa2, 0x49, 0x44)
+	err = en.Append(0x86, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -1609,15 +1627,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "KeyRotate"
+	err = en.Append(0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65)
+	if err != nil {
+		return
+	}
+	if z.KeyRotate == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.KeyRotate.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "KeyRotate")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
+	// map header, size 6
 	// string "ID"
-	o = append(o, 0x85, 0xa2, 0x49, 0x44)
+	o = append(o, 0x86, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -1639,6 +1674,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 			return
 		}
 	}
+	// string "KeyRotate"
+	o = append(o, 0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65)
+	if z.KeyRotate == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.KeyRotate.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "KeyRotate")
+			return
+		}
+	}
 	return
 }
 
@@ -1701,6 +1747,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "KeyRotate":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.KeyRotate = nil
+			} else {
+				if z.KeyRotate == nil {
+					z.KeyRotate = new(BatchJobKeyRotateV1)
+				}
+				bts, err = z.KeyRotate.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "KeyRotate")
+					return
+				}
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -1721,6 +1784,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
 	} else {
 		s += z.Replicate.Msgsize()
 	}
+	s += 10
+	if z.KeyRotate == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.KeyRotate.Msgsize()
+	}
 	return
 }
 

cmd/batch-rotate.go

@@ -0,0 +1,564 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/workers"
+	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/wildcard"
+)
+
+// keyrotate:
+//   apiVersion: v1
+//   bucket: BUCKET
+//   prefix: PREFIX
+//   encryption:
+//     type: sse-s3 # valid values are sse-s3 and sse-kms
+//     key: <new-kms-key> # valid only for sse-kms
+//     context: <new-kms-key-context> # valid only for sse-kms
+// # optional flags based filtering criteria
+// # for all objects
+// flags:
+//   filter:
+//     newerThan: "7d" # match objects newer than this value (e.g. 7d10h31s)
+//     olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+//     createdAfter: "date" # match objects created after "date"
+//     createdBefore: "date" # match objects created before "date"
+//     tags:
+//       - key: "name"
+//         value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
+//     metadata:
+//       - key: "content-type"
+//         value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
+//     kmskey: "key-id" # match objects with KMS key-id (applicable only for sse-kms)
+//   notify:
+//     endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
+//     token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+
+//   retry:
+//     attempts: 10 # number of retries for the job before giving up
+//     delay: "500ms" # least amount of delay between each retry
+
+//go:generate msgp -file $GOFILE -unexported
+
+// BatchKeyRotateKV is a datatype that holds key and values for filtering of objects
+// used by metadata filter as well as tags based filtering.
+type BatchKeyRotateKV struct {
+	Key   string `yaml:"key" json:"key"`
+	Value string `yaml:"value" json:"value"`
+}
+
+// Validate returns an error if key is empty
+func (kv BatchKeyRotateKV) Validate() error {
+	if kv.Key == "" {
+		return errInvalidArgument
+	}
+	return nil
+}
+
+// Empty indicates if kv is not set
+func (kv BatchKeyRotateKV) Empty() bool {
+	return kv.Key == "" && kv.Value == ""
+}
+
+// Match matches input kv with kv, value will be wildcard matched depending on the user input
+func (kv BatchKeyRotateKV) Match(ikv BatchKeyRotateKV) bool {
+	if kv.Empty() {
+		return true
+	}
+	if strings.EqualFold(kv.Key, ikv.Key) {
+		return wildcard.Match(kv.Value, ikv.Value)
+	}
+	return false
+}
+
+// BatchKeyRotateRetry datatype represents total retry attempts and delay between each retries.
+type BatchKeyRotateRetry struct {
+	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
+	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
+}
+
+// Validate validates input replicate retries.
+func (r BatchKeyRotateRetry) Validate() error {
+	if r.Attempts < 0 {
+		return errInvalidArgument
+	}
+
+	if r.Delay < 0 {
+		return errInvalidArgument
+	}
+
+	return nil
+}
+
+// BatchKeyRotationType defines key rotation type
+type BatchKeyRotationType string
+
+const (
+	sses3  BatchKeyRotationType = "sse-s3"
+	ssekms BatchKeyRotationType = "sse-kms"
+)
+
+// BatchJobKeyRotateEncryption defines key rotation encryption options passed
+type BatchJobKeyRotateEncryption struct {
+	Type       BatchKeyRotationType `yaml:"type" json:"type"`
+	Key        string               `yaml:"key" json:"key"`
+	Context    string               `yaml:"context" json:"context"`
+	kmsContext kms.Context          `msg:"-"`
+}
+
+// Validate validates input key rotation encryption options.
+func (e BatchJobKeyRotateEncryption) Validate() error {
+	if e.Type != sses3 && e.Type != ssekms {
+		return errInvalidArgument
+	}
+	spaces := strings.HasPrefix(e.Key, " ") || strings.HasSuffix(e.Key, " ")
+	if e.Type == ssekms && spaces {
+		return crypto.ErrInvalidEncryptionKeyID
+	}
+	if e.Type == ssekms && GlobalKMS != nil {
+		ctx := kms.Context{}
+		if e.Context != "" {
+			b, err := base64.StdEncoding.DecodeString(e.Context)
+			if err != nil {
+				return err
+			}
+
+			json := jsoniter.ConfigCompatibleWithStandardLibrary
+			if err := json.Unmarshal(b, &ctx); err != nil {
+				return err
+			}
+		}
+		e.kmsContext = kms.Context{}
+		for k, v := range ctx {
+			e.kmsContext[k] = v
+		}
+		ctx["MinIO batch API"] = "batchrotate" // Context for a test key operation
+		if _, err := GlobalKMS.GenerateKey(GlobalContext, e.Key, ctx); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// BatchKeyRotateFilter holds all the filters currently supported for batch replication
+type BatchKeyRotateFilter struct {
+	NewerThan     time.Duration      `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     time.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time          `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchKeyRotateKV `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchKeyRotateKV `yaml:"metadata,omitempty" json:"metadata"`
+	KMSKeyID      string             `yaml:"kmskeyid" json:"kmskey"`
+}
+
+// BatchKeyRotateNotification success or failure notification endpoint for each job attempts
+type BatchKeyRotateNotification struct {
+	Endpoint string `yaml:"endpoint" json:"endpoint"`
+	Token    string `yaml:"token" json:"token"`
+}
+
+// BatchJobKeyRotateFlags various configurations for replication job definition currently includes
+// - filter
+// - notify
+// - retry
+type BatchJobKeyRotateFlags struct {
+	Filter BatchKeyRotateFilter       `yaml:"filter" json:"filter"`
+	Notify BatchKeyRotateNotification `yaml:"notify" json:"notify"`
+	Retry  BatchKeyRotateRetry        `yaml:"retry" json:"retry"`
+}
+
+// BatchJobKeyRotateV1 v1 of batch key rotation job
+type BatchJobKeyRotateV1 struct {
+	APIVersion string                      `yaml:"apiVersion" json:"apiVersion"`
+	Flags      BatchJobKeyRotateFlags      `yaml:"flags" json:"flags"`
+	Bucket     string                      `yaml:"bucket" json:"bucket"`
+	Prefix     string                      `yaml:"prefix" json:"prefix"`
+	Endpoint   string                      `yaml:"endpoint" json:"endpoint"`
+	Encryption BatchJobKeyRotateEncryption `yaml:"encryption" json:"encryption"`
+}
+
+// Notify notifies notification endpoint if configured regarding job failure or success.
+func (r BatchJobKeyRotateV1) Notify(ctx context.Context, body io.Reader) error {
+	if r.Flags.Notify.Endpoint == "" {
+		return nil
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.Flags.Notify.Endpoint, body)
+	if err != nil {
+		return err
+	}
+
+	if r.Flags.Notify.Token != "" {
+		req.Header.Set("Authorization", r.Flags.Notify.Token)
+	}
+
+	clnt := http.Client{Transport: getRemoteInstanceTransport}
+	resp, err := clnt.Do(req)
+	if err != nil {
+		return err
+	}
+
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(resp.Status)
+	}
+
+	return nil
+}
+
+// KeyRotate rotates encryption key of an object
+func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error {
+	srcBucket := r.Bucket
+	srcObject := objInfo.Name
+
+	if objInfo.DeleteMarker || !objInfo.VersionPurgeStatus.Empty() {
+		return nil
+	}
+	sseKMS := crypto.S3KMS.IsEncrypted(objInfo.UserDefined)
+	sseS3 := crypto.S3.IsEncrypted(objInfo.UserDefined)
+	if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
+		return errInvalidEncryptionParameters
+	}
+	if sseKMS && r.Encryption.Type == sses3 { // previously encrypted with sse-kms, now sse-s3 disallowed
+		return errInvalidEncryptionParameters
+	}
+	versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
+	versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
+
+	lock := api.NewNSLock(r.Bucket, objInfo.Name)
+	lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return err
+	}
+	ctx = lkctx.Context()
+	defer lock.Unlock(lkctx)
+
+	opts := ObjectOptions{
+		VersionID:        objInfo.VersionID,
+		Versioned:        versioned,
+		VersionSuspended: versionSuspended,
+		NoLock:           true,
+	}
+	obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts)
+	if err != nil {
+		return err
+	}
+	oi := obj.Clone()
+	var (
+		newKeyID      string
+		newKeyContext kms.Context
+	)
+	encMetadata := make(map[string]string)
+	for k, v := range oi.UserDefined {
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			encMetadata[k] = v
+		}
+	}
+
+	if (sseKMS || sseS3) && r.Encryption.Type == ssekms {
+		if err = r.Encryption.Validate(); err != nil {
+			return err
+		}
+		newKeyID = strings.TrimPrefix(r.Encryption.Key, crypto.ARNPrefix)
+		newKeyContext = r.Encryption.kmsContext
+	}
+	if err = rotateKey(ctx, []byte{}, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext); err != nil {
+		return err
+	}
+
+	// Since we are rotating the keys, make sure to update the metadata.
+	oi.metadataOnly = true
+	oi.keyRotation = true
+	for k, v := range encMetadata {
+		oi.UserDefined[k] = v
+	}
+	if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{
+		VersionID: oi.VersionID,
+	}, ObjectOptions{
+		VersionID: oi.VersionID,
+		NoLock:    true,
+	}); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+const (
+	batchKeyRotationName               = "batch-rotate.bin"
+	batchKeyRotationFormat             = 1
+	batchKeyRotateVersionV1            = 1
+	batchKeyRotateVersion              = batchKeyRotateVersionV1
+	batchKeyRotateAPIVersion           = "v1"
+	batchKeyRotateJobDefaultRetries    = 3
+	batchKeyRotateJobDefaultRetryDelay = 250 * time.Millisecond
+)
+
+// Start the batch key rottion job, resumes if there was a pending job via "job.ID"
+func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	ri := &batchJobInfo{
+		JobID:     job.ID,
+		JobType:   string(job.Type()),
+		StartTime: job.Started,
+	}
+	if err := ri.load(ctx, api, job); err != nil {
+		return err
+	}
+
+	globalBatchJobsMetrics.save(job.ID, ri)
+	lastObject := ri.Object
+
+	delay := job.KeyRotate.Flags.Retry.Delay
+	if delay == 0 {
+		delay = batchKeyRotateJobDefaultRetryDelay
+	}
+	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	skip := func(info FileInfo) (ok bool) {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
+			// skip all objects that are newer than specified older duration
+			return false
+		}
+
+		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan {
+			// skip all objects that are older than specified newer duration
+			return false
+		}
+
+		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(info.ModTime) {
+			// skip all objects that are created before the specified time.
+			return false
+		}
+
+		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(info.ModTime) {
+			// skip all objects that are created after the specified time.
+			return false
+		}
+
+		if len(r.Flags.Filter.Tags) > 0 {
+			// Only parse object tags if tags filter is specified.
+			tagMap := map[string]string{}
+			tagStr := info.Metadata[xhttp.AmzObjectTagging]
+			if len(tagStr) != 0 {
+				t, err := tags.ParseObjectTags(tagStr)
+				if err != nil {
+					return false
+				}
+				tagMap = t.ToMap()
+			}
+
+			for _, kv := range r.Flags.Filter.Tags {
+				for t, v := range tagMap {
+					if kv.Match(BatchKeyRotateKV{Key: t, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided tags filter match skip the object
+			return false
+		}
+
+		if len(r.Flags.Filter.Metadata) > 0 {
+			for _, kv := range r.Flags.Filter.Metadata {
+				for k, v := range info.Metadata {
+					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
+						continue
+					}
+					// We only need to match x-amz-meta or standardHeaders
+					if kv.Match(BatchKeyRotateKV{Key: k, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided metadata filters match skip the object.
+			return false
+		}
+		if r.Flags.Filter.KMSKeyID != "" {
+			if v, ok := info.Metadata[xhttp.AmzServerSideEncryptionKmsID]; ok && strings.TrimPrefix(v, crypto.ARNPrefix) != r.Flags.Filter.KMSKeyID {
+				return false
+			}
+		}
+		return true
+	}
+
+	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_KEYROTATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+	if err != nil {
+		return err
+	}
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		// invalid worker size.
+		return err
+	}
+
+	retryAttempts := ri.RetryAttempts
+	ctx, cancel := context.WithCancel(ctx)
+
+	results := make(chan ObjectInfo, 100)
+	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, ObjectOptions{
+		WalkMarker: lastObject,
+		WalkFilter: skip,
+	}); err != nil {
+		cancel()
+		// Do not need to retry if we can't list objects on source.
+		return err
+	}
+
+	for result := range results {
+		result := result
+		sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined)
+		sseS3 := crypto.S3.IsEncrypted(result.UserDefined)
+		if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
+			continue
+		}
+		wk.Take()
+		go func() {
+			defer wk.Give()
+			for attempts := 1; attempts <= retryAttempts; attempts++ {
+				attempts := attempts
+				stopFn := globalBatchJobsMetrics.trace(batchKeyRotationMetricObject, job.ID, attempts, result)
+				success := true
+				if err := r.KeyRotate(ctx, api, result); err != nil {
+					stopFn(err)
+					logger.LogIf(ctx, err)
+					success = false
+				} else {
+					stopFn(nil)
+				}
+				ri.trackCurrentBucketObject(r.Bucket, result, success)
+				ri.RetryAttempts = attempts
+				globalBatchJobsMetrics.save(job.ID, ri)
+				// persist in-memory state to disk after every 10secs.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				if success {
+					break
+				}
+			}
+		}()
+	}
+	wk.Wait()
+
+	ri.Complete = ri.ObjectsFailed == 0
+	ri.Failed = ri.ObjectsFailed > 0
+	globalBatchJobsMetrics.save(job.ID, ri)
+	// persist in-memory state to disk.
+	logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+
+	buf, _ := json.Marshal(ri)
+	if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+	}
+
+	cancel()
+	if ri.Failed {
+		ri.ObjectsFailed = 0
+		ri.Bucket = ""
+		ri.Object = ""
+		ri.Objects = 0
+		time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+	}
+
+	return nil
+}
+
+//msgp:ignore batchKeyRotationJobError
+type batchKeyRotationJobError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+func (e batchKeyRotationJobError) Error() string {
+	return e.Description
+}
+
+// Validate validates the job definition input
+func (r *BatchJobKeyRotateV1) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
+	if r == nil {
+		return nil
+	}
+
+	if r.APIVersion != batchKeyRotateAPIVersion {
+		return errInvalidArgument
+	}
+
+	if r.Bucket == "" {
+		return errInvalidArgument
+	}
+
+	if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil {
+		if isErrBucketNotFound(err) {
+			return batchKeyRotationJobError{
+				Code:           "NoSuchSourceBucket",
+				Description:    "The specified source bucket does not exist",
+				HTTPStatusCode: http.StatusNotFound,
+			}
+		}
+		return err
+	}
+	if GlobalKMS == nil {
+		return errKMSNotConfigured
+	}
+	if err := r.Encryption.Validate(); err != nil {
+		return err
+	}
+
+	for _, tag := range r.Flags.Filter.Tags {
+		if err := tag.Validate(); err != nil {
+			return err
+		}
+	}
+
+	for _, meta := range r.Flags.Filter.Metadata {
+		if err := meta.Validate(); err != nil {
+			return err
+		}
+	}
+
+	if err := r.Flags.Retry.Validate(); err != nil {
+		return err
+	}
+	return nil
+}

cmd/batch-rotate_gen_test.go

@@ -0,0 +1,801 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBatchJobKeyRotateEncryption(t *testing.T) {
+	v := BatchJobKeyRotateEncryption{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKeyRotateEncryption(t *testing.T) {
+	v := BatchJobKeyRotateEncryption{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateEncryption Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKeyRotateEncryption{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKeyRotateEncryption(b *testing.B) {
+	v := BatchJobKeyRotateEncryption{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobKeyRotateFlags(t *testing.T) {
+	v := BatchJobKeyRotateFlags{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKeyRotateFlags(t *testing.T) {
+	v := BatchJobKeyRotateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateFlags Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKeyRotateFlags{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKeyRotateFlags(b *testing.B) {
+	v := BatchJobKeyRotateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobKeyRotateV1(t *testing.T) {
+	v := BatchJobKeyRotateV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKeyRotateV1(t *testing.T) {
+	v := BatchJobKeyRotateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateV1 Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKeyRotateV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKeyRotateV1(b *testing.B) {
+	v := BatchJobKeyRotateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateFilter(t *testing.T) {
+	v := BatchKeyRotateFilter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateFilter(t *testing.T) {
+	v := BatchKeyRotateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateFilter Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateFilter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateFilter(b *testing.B) {
+	v := BatchKeyRotateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateKV(t *testing.T) {
+	v := BatchKeyRotateKV{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateKV(t *testing.T) {
+	v := BatchKeyRotateKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateKV Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateKV{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateKV(b *testing.B) {
+	v := BatchKeyRotateKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateNotification(t *testing.T) {
+	v := BatchKeyRotateNotification{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateNotification(t *testing.T) {
+	v := BatchKeyRotateNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateNotification Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateNotification{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateNotification(b *testing.B) {
+	v := BatchKeyRotateNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchKeyRotateRetry(t *testing.T) {
+	v := BatchKeyRotateRetry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchKeyRotateRetry(t *testing.T) {
+	v := BatchKeyRotateRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateRetry Msgsize() is inaccurate")
+	}
+
+	vn := BatchKeyRotateRetry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchKeyRotateRetry(b *testing.B) {
+	v := BatchKeyRotateRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batchjobmetric_string.go

@@ -0,0 +1,24 @@
+// Code generated by "stringer -type=batchJobMetric -trimprefix=batchJobMetric batch-handlers.go"; DO NOT EDIT.
+
+package cmd
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[batchReplicationMetricObject-0]
+	_ = x[batchKeyRotationMetricObject-1]
+}
+
+const _batchJobMetric_name = "batchReplicationMetricObjectbatchKeyRotationMetricObject"
+
+var _batchJobMetric_index = [...]uint8{0, 28, 56}
+
+func (i batchJobMetric) String() string {
+	if i >= batchJobMetric(len(_batchJobMetric_index)-1) {
+		return "batchJobMetric(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _batchJobMetric_name[_batchJobMetric_index[i]:_batchJobMetric_index[i+1]]
+}

cmd/batchreplicationmetric_string.go

@@ -1,23 +0,0 @@
-// Code generated by "stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric batch-handlers.go"; DO NOT EDIT.
-
-package cmd
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[batchReplicationMetricObject-0]
-}
-
-const _batchReplicationMetric_name = "Object"
-
-var _batchReplicationMetric_index = [...]uint8{0, 6}
-
-func (i batchReplicationMetric) String() string {
-	if i >= batchReplicationMetric(len(_batchReplicationMetric_index)-1) {
-		return "batchReplicationMetric(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-	return _batchReplicationMetric_name[_batchReplicationMetric_index[i]:_batchReplicationMetric_index[i+1]]
-}

cmd/bootstrap-messages.go

@@ -0,0 +1,116 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio/internal/pubsub"
+)
+
+const bootstrapMsgsLimit = 4 << 10
+
+type bootstrapInfo struct {
+	msg    string
+	ts     time.Time
+	source string
+}
+type bootstrapTracer struct {
+	mu         sync.RWMutex
+	idx        int
+	info       [bootstrapMsgsLimit]bootstrapInfo
+	lastUpdate time.Time
+}
+
+var globalBootstrapTracer = &bootstrapTracer{}
+
+func (bs *bootstrapTracer) DropEvents() {
+	bs.mu.Lock()
+	defer bs.mu.Unlock()
+
+	if time.Now().UTC().Sub(bs.lastUpdate) > 24*time.Hour {
+		bs.info = [4096]bootstrapInfo{}
+		bs.idx = 0
+	}
+}
+
+func (bs *bootstrapTracer) Empty() bool {
+	var empty bool
+	bs.mu.RLock()
+	empty = bs.info[0].msg == ""
+	bs.mu.RUnlock()
+
+	return empty
+}
+
+func (bs *bootstrapTracer) Record(msg string, skip int) {
+	source := getSource(skip + 1)
+	bs.mu.Lock()
+	now := time.Now().UTC()
+	bs.info[bs.idx] = bootstrapInfo{
+		msg:    msg,
+		ts:     now,
+		source: source,
+	}
+	bs.lastUpdate = now
+	bs.idx = (bs.idx + 1) % bootstrapMsgsLimit
+	bs.mu.Unlock()
+}
+
+func (bs *bootstrapTracer) Events() []madmin.TraceInfo {
+	traceInfo := make([]madmin.TraceInfo, 0, bootstrapMsgsLimit)
+
+	// Add all messages in order
+	addAll := func(info []bootstrapInfo) {
+		for _, msg := range info {
+			if msg.ts.IsZero() {
+				continue // skip empty events
+			}
+			traceInfo = append(traceInfo, madmin.TraceInfo{
+				TraceType: madmin.TraceBootstrap,
+				Time:      msg.ts,
+				NodeName:  globalLocalNodeName,
+				FuncName:  "BOOTSTRAP",
+				Message:   fmt.Sprintf("%s %s", msg.source, msg.msg),
+			})
+		}
+	}
+
+	bs.mu.RLock()
+	addAll(bs.info[bs.idx:])
+	addAll(bs.info[:bs.idx])
+	bs.mu.RUnlock()
+	return traceInfo
+}
+
+func (bs *bootstrapTracer) Publish(ctx context.Context, trace *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) {
+	if bs.Empty() {
+		return
+	}
+	for _, bsEvent := range bs.Events() {
+		select {
+		case <-ctx.Done():
+		default:
+			trace.Publish(bsEvent)
+		}
+	}
+}

cmd/bootstrap-messages_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestBootstrap(t *testing.T) {
+	// Bootstrap events exceed bootstrap messages limit
+	bsTracer := &bootstrapTracer{}
+	for i := 0; i < bootstrapMsgsLimit+10; i++ {
+		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
+	}
+
+	traceInfos := bsTracer.Events()
+	if len(traceInfos) != bootstrapMsgsLimit {
+		t.Fatalf("Expected length of events %d but got %d", bootstrapMsgsLimit, len(traceInfos))
+	}
+
+	// Simulate the case where bootstrap events were updated a day ago
+	bsTracer.lastUpdate = time.Now().UTC().Add(-25 * time.Hour)
+	bsTracer.DropEvents()
+	if !bsTracer.Empty() {
+		t.Fatalf("Expected all bootstrap events to have been dropped, but found %d events", len(bsTracer.Events()))
+	}
+
+	// Fewer than 4K bootstrap events
+	for i := 0; i < 10; i++ {
+		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
+	}
+	events := bsTracer.Events()
+	if len(events) != 10 {
+		t.Fatalf("Expected length of events %d but got %d", 10, len(events))
+	}
+	for i, traceInfo := range bsTracer.Events() {
+		msg := fmt.Sprintf("msg-%d", i)
+		if !strings.HasSuffix(traceInfo.Message, msg) {
+			t.Fatalf("Expected %s but got %s", msg, traceInfo.Message)
+		}
+	}
+}

cmd/bootstrap-peer-server.go

@@ -101,10 +101,14 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 }
 
 var skipEnvs = map[string]struct{}{
-	"MINIO_OPTS":         {},
-	"MINIO_CERT_PASSWD":  {},
-	"MINIO_SERVER_DEBUG": {},
-	"MINIO_DSYNC_TRACE":  {},
+	"MINIO_OPTS":          {},
+	"MINIO_CERT_PASSWD":   {},
+	"MINIO_SERVER_DEBUG":  {},
+	"MINIO_DSYNC_TRACE":   {},
+	"MINIO_ROOT_USER":     {},
+	"MINIO_ROOT_PASSWORD": {},
+	"MINIO_ACCESS_KEY":    {},
+	"MINIO_SECRET_KEY":    {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {
@@ -118,7 +122,7 @@ func getServerSystemCfg() ServerSystemConfig {
 		if _, ok := skipEnvs[envK]; ok {
 			continue
 		}
-		envValues[envK] = env.Get(envK, "")
+		envValues[envK] = logger.HashString(env.Get(envK, ""))
 	}
 	return ServerSystemConfig{
 		MinioEndpoints: globalEndpoints,
@@ -126,11 +130,22 @@ func getServerSystemCfg() ServerSystemConfig {
 	}
 }
 
+func (b *bootstrapRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusForbidden)
+	w.Write([]byte(err.Error()))
+}
+
 // HealthHandler returns success if request is valid
 func (b *bootstrapRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {}
 
 func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "VerifyHandler")
+
+	if err := storageServerRequestValidate(r); err != nil {
+		b.writeErrorResponse(w, err)
+		return
+	}
+
 	cfg := getServerSystemCfg()
 	logger.LogIf(ctx, json.NewEncoder(w).Encode(&cfg))
 }

cmd/bucket-handlers.go

@@ -796,8 +796,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 				// Make sure to add Location information here only for bucket
-				w.Header().Set(xhttp.Location,
-					getObjectLocation(r, globalDomainNames, bucket, ""))
+				w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket))
 
 				writeSuccessResponseHeadersOnly(w)
 
@@ -848,9 +847,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	logger.LogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts))
 
 	// Make sure to add Location information here only for bucket
-	if cp := pathClean(r.URL.Path); cp != "" {
-		w.Header().Set(xhttp.Location, cp) // Clean any trailing slashes.
-	}
+	w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket))
 
 	writeSuccessResponseHeadersOnly(w)
 
@@ -1123,7 +1120,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
 
-	w.Header().Set(xhttp.Location, getObjectLocation(r, globalDomainNames, bucket, object))
+	if obj := getObjectLocation(r, globalDomainNames, bucket, object); obj != "" {
+		w.Header().Set(xhttp.Location, obj)
+	}
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{

cmd/bucket-lifecycle.go

@@ -24,12 +24,15 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
@@ -38,6 +41,8 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
+	"github.com/minio/minio/internal/workers"
+	"github.com/minio/pkg/env"
 )
 
 const (
@@ -66,6 +71,30 @@ func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
 
+func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string) madmin.TraceInfo {
+	return madmin.TraceInfo{
+		TraceType: madmin.TraceILM,
+		Time:      startTime,
+		NodeName:  globalLocalNodeName,
+		FuncName:  event,
+		Duration:  duration,
+		Path:      pathJoin(oi.Bucket, oi.Name),
+		Error:     "",
+		Message:   getSource(4),
+		Custom:    map[string]string{"version-id": oi.VersionID},
+	}
+}
+
+func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string) {
+	startTime := time.Now()
+	return func(event string) {
+		duration := time.Since(startTime)
+		if globalTrace.NumSubscribers(madmin.TraceILM) > 0 {
+			globalTrace.Publish(ilmTrace(startTime, duration, oi, event))
+		}
+	}
+}
+
 type expiryTask struct {
 	objInfo        ObjectInfo
 	versionExpiry  bool
@@ -116,26 +145,49 @@ var globalExpiryState *expiryState
 
 func newExpiryState() *expiryState {
 	return &expiryState{
-		byDaysCh:            make(chan expiryTask, 10000),
-		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 10000),
+		byDaysCh:            make(chan expiryTask, 100000),
+		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 100000),
 	}
 }
 
 func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 	globalExpiryState = newExpiryState()
+
+	workerSize, _ := strconv.Atoi(env.Get("_MINIO_ILM_EXPIRY_WORKERS", strconv.Itoa((runtime.GOMAXPROCS(0)+1)/2)))
+
+	ewk, err := workers.New(workerSize)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
+	nwk, err := workers.New(workerSize)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+
 	go func() {
 		for t := range globalExpiryState.byDaysCh {
-			if t.objInfo.TransitionedObject.Status != "" {
-				applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
-			} else {
-				applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
-			}
+			ewk.Take()
+			go func(t expiryTask) {
+				defer ewk.Give()
+				if t.objInfo.TransitionedObject.Status != "" {
+					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
+				} else {
+					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
+				}
+			}(t)
 		}
+		ewk.Wait()
 	}()
 	go func() {
 		for t := range globalExpiryState.byNewerNoncurrentCh {
-			deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+			nwk.Take()
+			go func(t newerNoncurrentTask) {
+				defer nwk.Give()
+				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+			}(t)
 		}
+		nwk.Wait()
 	}()
 }
 
@@ -333,6 +385,7 @@ const (
 // 1. when a restored (via PostRestoreObject API) object expires.
 // 2. when a transitioned object expires (based on an ILM rule).
 func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error {
+	traceFn := globalLifecycleSys.trace(*oi)
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name)
 	opts.VersionID = lcOpts.VersionID
@@ -358,7 +411,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		}
 
 		// Send audit for the lifecycle delete operation
-		auditLogLifecycle(ctx, *oi, ILMExpiry)
+		defer auditLogLifecycle(ctx, *oi, ILMExpiry, traceFn)
 
 		eventName := event.ObjectRemovedDelete
 		if lcOpts.DeleteMarker {
@@ -374,7 +427,8 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 			EventName:  eventName,
 			BucketName: oi.Bucket,
 			Object:     objInfo,
-			Host:       "Internal: [ILM-Expiry]",
+			UserAgent:  "Internal: [ILM-Expiry]",
+			Host:       globalLocalNodeName,
 		})
 
 	case expireRestoredObj:

cmd/bucket-listobjects-handlers.go

@@ -59,10 +59,18 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max
 	return ErrNone
 }
 
-// ListObjectVersions - GET Bucket Object versions
+func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+	api.listObjectVersionsHandler(w, r, false)
+}
+
+func (api objectAPIHandlers) ListObjectVersionsMHandler(w http.ResponseWriter, r *http.Request) {
+	api.listObjectVersionsHandler(w, r, true)
+}
+
+// ListObjectVersionsHandler - GET Bucket Object versions
 // You can use the versions subresource to list metadata about all
 // of the versions of objects in a bucket.
-func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) listObjectVersionsHandler(w http.ResponseWriter, r *http.Request, metadata bool) {
 	ctx := newContext(r, w, "ListObjectVersions")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -80,7 +88,12 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
-
+	var checkObjMeta metaCheckFn
+	if metadata {
+		checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
+			return checkRequestAuthType(ctx, r, action, bucket, name)
+		}
+	}
 	urlValues := r.Form
 
 	// Extract all the listBucketVersions query params to their native values.
@@ -111,8 +124,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
-	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
+	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo, checkObjMeta)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponseList(response))
@@ -128,64 +140,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 // MinIO continues to support ListObjectsV1 and V2 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2M")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-
-	urlValues := r.Form
-
-	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
-	if errCode != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
-		return
-	}
-
-	// Validate the query params before beginning to serve the request.
-	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
-		return
-	}
-
-	listObjectsV2 := objectAPI.ListObjectsV2
-
-	// Inititate a list objects operation based on the input params.
-	// On success would return back ListObjectsInfo object to be
-	// marshaled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
-	// The next continuation token has id@node_index format to optimize paginated listing
-	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-
-	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
-		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
-		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true)
-
-	// Write success response.
-	writeSuccessResponseXML(w, encodeResponseList(response))
+	api.listObjectsV2Handler(ctx, w, r, true)
 }
 
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
@@ -198,7 +153,11 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 // MinIO continues to support ListObjectsV1 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
+	api.listObjectsV2Handler(ctx, w, r, false)
+}
 
+// listObjectsV2Handler performs listing either with or without extra metadata.
+func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.ResponseWriter, r *http.Request, metadata bool) {
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
@@ -215,6 +174,12 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
+	var checkObjMeta metaCheckFn
+	if metadata {
+		checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
+			return checkRequestAuthType(ctx, r, action, bucket, name)
+		}
+	}
 	urlValues := r.Form
 
 	// Extract all the listObjectsV2 query params to their native values.
@@ -257,7 +222,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
-		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
+		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, checkObjMeta)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponseList(response))

cmd/bucket-metadata.go

@@ -323,8 +323,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
 
 		configData, err := readConfig(ctx, objectAPI, configFile)
 		if err != nil {
-			switch err.(type) {
-			case ObjectExistsAsDirectory:
+			if _, ok := err.(ObjectExistsAsDirectory); ok {
 				// in FS mode it possible that we have actual
 				// files in this folder with `.minio.sys/buckets/bucket/configFile`
 				continue

cmd/bucket-object-lock.go

@@ -90,8 +90,8 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 
 	opts.VersionID = object.VersionID
 	if gerr != nil { // error from GetObjectInfo
-		switch gerr.(type) {
-		case MethodNotAllowed: // This happens usually for a delete marker
+		if _, ok := gerr.(MethodNotAllowed); ok {
+			// This happens usually for a delete marker
 			if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 				// Delete marker should be present and valid.
 				return ErrNone
@@ -192,8 +192,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 				days, objRetention.RetainUntilDate.Time,
 				objRetention.Mode, byPassSet, r, cred,
 				owner)
-			switch apiErr {
-			case ErrAccessDenied:
+			if apiErr == ErrAccessDenied {
 				return errAuthentication
 			}
 			return nil
@@ -210,8 +209,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 					return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
 				}
 			}
-			switch govPerm {
-			case ErrAccessDenied:
+			if govPerm == ErrAccessDenied {
 				return errAuthentication
 			}
 			return nil
@@ -224,8 +222,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 			apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
 				days, objRetention.RetainUntilDate.Time, objRetention.Mode,
 				false, r, cred, owner)
-			switch apiErr {
-			case ErrAccessDenied:
+			if apiErr == ErrAccessDenied {
 				return errAuthentication
 			}
 			return nil
@@ -236,8 +233,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 	apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
 		days, objRetention.RetainUntilDate.Time,
 		objRetention.Mode, byPassSet, r, cred, owner)
-	switch apiErr {
-	case ErrAccessDenied:
+	if apiErr == ErrAccessDenied {
 		return errAuthentication
 	}
 	return nil

cmd/bucket-replication-handlers.go

@@ -277,6 +277,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 				Bucket: bucket,
 				Err:    fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
 			}), r.URL)
+			return
 		}
 	}
 	resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24))
@@ -347,6 +348,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		}
+		return
 	}
 	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	if err != nil {

cmd/bucket-replication-utils.go

@@ -496,8 +496,7 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 			oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus)
 		}
 		for k, v := range oi.UserDefined {
-			switch {
-			case strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset):
+			if strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset) {
 				delete(oi.UserDefined, k)
 				oi.UserDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
 			}

cmd/bucket-replication.go

@@ -387,7 +387,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -403,7 +404,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -424,7 +426,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -449,7 +452,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 					VersionID:    versionID,
 					DeleteMarker: dobj.DeleteMarker,
 				},
-				Host:      "Internal: [Replication]",
+				UserAgent: "Internal: [Replication]",
+				Host:      globalLocalNodeName,
 				EventName: event.ObjectReplicationNotTracked,
 			})
 			continue
@@ -508,7 +512,6 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
 	if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on.
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %s", bucket, dobj.ObjectName, versionID, err))
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -517,14 +520,16 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: eventName,
 		})
 	} else {
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object:     dobjInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 			EventName:  eventName,
 		})
 	}
@@ -561,7 +566,8 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 				VersionID:    dobj.VersionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
+			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		if dobj.VersionID == "" {
@@ -586,11 +592,13 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 				return
 			}
 		}
-		// mark delete marker replication as failed if target cluster not ready to receive
-		// this request yet (object version not replicated yet)
-		if err != nil && !toi.ReplicationReady {
-			rinfo.ReplicationStatus = replication.Failed
-			return
+		if !isErrObjectNotFound(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
+			// mark delete marker replication as failed if target cluster not ready to receive
+			// this request yet (object version not replicated yet)
+			if err != nil && !toi.ReplicationReady {
+				rinfo.ReplicationStatus = replication.Failed
+				return
+			}
 		}
 	}
 
@@ -802,10 +810,12 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 		oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID {
 		return replicateNone
 	}
+	sz, _ := oi1.GetActualSize()
+
 	// needs full replication
 	if oi1.ETag != oi2.ETag ||
 		oi1.VersionID != oi2.VersionID ||
-		oi1.Size != oi2.Size ||
+		sz != oi2.Size ||
 		oi1.DeleteMarker != oi2.IsDeleteMarker ||
 		oi1.ModTime.Unix() != oi2.LastModified.Unix() {
 		return replicateAll
@@ -917,7 +927,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -935,7 +946,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
 		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, ri.TargetArn))
@@ -955,7 +967,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			continue
 		}
@@ -1001,10 +1014,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			},
 		}
 
-		if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w",
-				bucket, objInfo.Name, objInfo.VersionID, err))
-		}
+		_, _ = objectAPI.PutObjectMetadata(ctx, bucket, object, popts)
 		opType := replication.MetadataReplicationType
 		if rinfos.Action() == replicateAll {
 			opType = replication.ObjectReplicationType
@@ -1020,7 +1030,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		EventName:  eventName,
 		BucketName: bucket,
 		Object:     objInfo,
-		Host:       "Internal: [Replication]",
+		UserAgent:  "Internal: [Replication]",
+		Host:       globalLocalNodeName,
 	})
 
 	// re-queue failures once more - keep a retry count to avoid flooding the queue if
@@ -1065,7 +1076,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1084,7 +1096,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
@@ -1101,7 +1114,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1112,7 +1126,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return rinfo
 	}
@@ -1137,7 +1152,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1202,19 +1218,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		ReplicationAction:     rAction,
 	}
 
-	if ri.ObjectInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
-		rinfo.ReplicationStatus = replication.Completed
-		rinfo.ReplicationResynced = true
-		return
-	}
-
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1233,7 +1244,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
@@ -1243,6 +1255,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 
 	objInfo = gr.ObjInfo
 
+	// use latest ObjectInfo to check if previous replication attempt succeeded
+	if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
+		rinfo.ReplicationStatus = replication.Completed
+		rinfo.ReplicationResynced = true
+		return
+	}
+
 	size, err := objInfo.GetActualSize()
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -1250,18 +1269,20 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return
 	}
 
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("Unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
+			Host:       globalLocalNodeName,
 		})
 		return rinfo
 	}
@@ -1286,12 +1307,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		if rAction == replicateNone {
 			if ri.OpType == replication.ExistingObjectReplicationType &&
 				objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID {
-				logger.LogIf(ctx, fmt.Errorf("Unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
 				sendEvent(eventArgs{
 					EventName:  event.ObjectReplicationNotTracked,
 					BucketName: bucket,
 					Object:     objInfo,
-					Host:       "Internal: [Replication]",
+					UserAgent:  "Internal: [Replication]",
+					Host:       globalLocalNodeName,
 				})
 			}
 			// object with same VersionID already exists, replication kicked off by
@@ -1328,7 +1350,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 		if _, err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); err != nil {
 			rinfo.ReplicationStatus = replication.Failed
-			logger.LogIf(ctx, fmt.Errorf("Unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 		}
 	} else {
 		var putOpts minio.PutObjectOptions
@@ -1339,7 +1361,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
 			})
 			return
 		}
@@ -1365,7 +1388,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				r, objInfo, putOpts); err != nil {
 				if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1374,7 +1397,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
 				if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1419,7 +1442,12 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		if err != nil {
 			return err
 		}
-		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, "", "", opts.ServerSideEncryption)
+
+		popts := minio.PutObjectPartOptions{
+			SSE: opts.ServerSideEncryption,
+		}
+
+		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
 		if err != nil {
 			return err
 		}

cmd/bucket-targets.go

@@ -208,7 +208,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	}
 	clnt, err := sys.getRemoteTargetClient(tgt)
 	if err != nil {
-		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket, Err: err}
 	}
 	// validate if target credentials are ok
 	exists, err := clnt.BucketExists(ctx, tgt.TargetBucket)

cmd/common-main.go

@@ -257,6 +257,9 @@ func initConsoleServer() (*restapi.Server, error) {
 		Path: globalCertsCADir.Get(),
 	}
 
+	// set certs before other console initialization
+	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
+
 	swaggerSpec, err := loads.Embedded(restapi.SwaggerJSON, restapi.FlatSwaggerJSON)
 	if err != nil {
 		return nil, err
@@ -283,8 +286,6 @@ func initConsoleServer() (*restapi.Server, error) {
 	// register all APIs
 	server.ConfigureAPI()
 
-	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
-
 	consolePort, _ := strconv.Atoi(globalMinioConsolePort)
 
 	server.Host = globalMinioConsoleHost
@@ -661,6 +662,7 @@ func handleCommonEnvVars() {
 		}
 		u.Path = "" // remove any path component such as `/`
 		globalMinioEndpoint = u.String()
+		globalMinioEndpointURL = u
 	}
 
 	globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff))
@@ -783,8 +785,7 @@ func handleCommonEnvVars() {
 // Initialize KMS global variable after valiadating and loading the configuration.
 // It depends on KMS env variables and global cli flags.
 func handleKMSConfig() {
-	switch {
-	case env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint):
+	if env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint) {
 		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKMSSecretKey, kms.EnvKESEndpoint))
 	}
 

cmd/config-current.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/config/identity/openid"
 	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	xtls "github.com/minio/minio/internal/config/identity/tls"
+	"github.com/minio/minio/internal/config/lambda"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
@@ -74,6 +75,9 @@ func initHelp() {
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
 	}
+	for k, v := range lambda.DefaultLambdaKVS {
+		kvs[k] = v
+	}
 	if globalIsErasure {
 		kvs[config.StorageClassSubSys] = storageclass.DefaultKVS
 		kvs[config.HealSubSys] = heal.DefaultKVS
@@ -196,6 +200,11 @@ func initHelp() {
 			Description:     "publish bucket notifications to Redis datastores",
 			MultipleTargets: true,
 		},
+		config.HelpKV{
+			Key:             config.LambdaWebhookSubSys,
+			Description:     "manage remote lambda functions",
+			MultipleTargets: true,
+		},
 		config.HelpKV{
 			Key:         config.EtcdSubSys,
 			Description: "persist IAM assets externally to etcd",
@@ -246,6 +255,7 @@ func initHelp() {
 		config.NotifyRedisSubSys:    notify.HelpRedis,
 		config.NotifyWebhookSubSys:  notify.HelpWebhook,
 		config.NotifyESSubSys:       notify.HelpES,
+		config.LambdaWebhookSubSys:  lambda.HelpWebhook,
 		config.SubnetSubSys:         subnet.HelpSubnet,
 		config.CallhomeSubSys:       callhome.HelpCallhome,
 	}
@@ -387,6 +397,13 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			return err
 		}
 	}
+
+	if config.LambdaSubSystems.Contains(subSys) {
+		if err := lambda.TestSubSysLambdaTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -423,6 +440,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 	}
 	if err == nil && dnsURL != "" {
+		bootstrapTrace("initialize remote bucket DNS store")
 		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
 			dns.Authentication(dnsUser, dnsPass),
 			dns.RootCAs(globalRootCAs))
@@ -437,6 +455,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	if etcdCfg.Enabled {
+		bootstrapTrace("initialize etcd store")
 		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
@@ -495,12 +514,18 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	transport := NewHTTPTransport()
 
-	bootstrapTrace("lookup the event notification targets")
-	globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
+	bootstrapTrace("initialize the event notification targets")
+	globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
+	bootstrapTrace("initialize the lambda targets")
+	globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
+	}
+
 	bootstrapTrace("applying the dynamic configuration")
 	// Apply dynamic config values
 	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {

cmd/crossdomain-xml-handler.go

@@ -33,8 +33,7 @@ const crossDomainXMLEntity = "/crossdomain.xml"
 func setCrossDomainPolicy(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Look for 'crossdomain.xml' in the incoming request.
-		switch r.URL.Path {
-		case crossDomainXMLEntity:
+		if r.URL.Path == crossDomainXMLEntity {
 			// Write the standard cross domain policy xml.
 			w.Write([]byte(crossDomainXML))
 			// Request completed, no need to serve to other handlers.

cmd/data-scanner.go

@@ -957,6 +957,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 // applyTierObjSweep removes remote object pending deletion and the free-version
 // tracking this information.
 func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
+	traceFn := globalLifecycleSys.trace(oi)
 	if !oi.TransitionedObject.FreeVersion {
 		// nothing to be done
 		return
@@ -982,7 +983,7 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 		InclFreeVersions: true,
 	})
 	if err == nil {
-		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete)
+		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, traceFn)
 	}
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
@@ -991,27 +992,35 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 
 // applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
 // Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
-func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
-	if i.lifeCycle == nil {
-		return fivs, nil
-	}
+func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
 	done := globalScannerMetrics.time(scannerMetricApplyNonCurrent)
 	defer done()
 
-	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
-	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
-		return fivs, nil
-	}
-
-	overflowVersions := fivs[lim+1:]
-	// current version + most recent lim noncurrent versions
-	fivs = fivs[:lim+1]
-
 	rcfg, _ := globalBucketObjectLockSys.Get(i.bucket)
 	vcfg, _ := globalBucketVersioningSys.Get(i.bucket)
 
 	versioned := vcfg != nil && vcfg.Versioned(i.objectPath())
 
+	// current version + most recent lim noncurrent versions
+	objectInfos := make([]ObjectInfo, 0, len(fivs))
+
+	if i.lifeCycle == nil {
+		for _, fi := range fivs {
+			objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
+		}
+		return objectInfos, nil
+	}
+
+	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
+		for _, fi := range fivs {
+			objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
+		}
+		return objectInfos, nil
+	}
+
+	overflowVersions := fivs[lim+1:]
+
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
 		obj := fi.ToObjectInfo(i.bucket, i.objectPath(), versioned)
@@ -1026,7 +1035,7 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 			}
 			// add this version back to remaining versions for
 			// subsequent lifecycle policy applications
-			fivs = append(fivs, fi)
+			objectInfos = append(objectInfos, obj)
 			continue
 		}
 
@@ -1040,19 +1049,19 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 
 		toDel = append(toDel, ObjectToDelete{
 			ObjectV: ObjectV{
-				ObjectName: fi.Name,
-				VersionID:  fi.VersionID,
+				ObjectName: obj.Name,
+				VersionID:  obj.VersionID,
 			},
 		})
 	}
 
 	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel)
-	return fivs, nil
+	return objectInfos, nil
 }
 
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
-func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
+func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
 	if i.heal.enabled {
 		if healDeleteDangling {
 			done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
@@ -1063,13 +1072,14 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			}
 		}
 	}
-	fivs, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+
+	objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
 	if err != nil {
-		return fivs, err
+		return nil, err
 	}
 
 	// Check if we have many versions after applyNewerNoncurrentVersionLimit.
-	if len(fivs) > dataScannerExcessiveVersionsThreshold {
+	if len(objInfos) > dataScannerExcessiveVersionsThreshold {
 		// Notify object accessed via a GET request.
 		sendEvent(eventArgs{
 			EventName:  event.ObjectManyVersions,
@@ -1077,13 +1087,13 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			Object: ObjectInfo{
 				Name: i.objectPath(),
 			},
-			UserAgent:    "scanner",
-			Host:         globalMinioHost,
+			UserAgent:    "Internal: [Scanner]",
+			Host:         globalLocalNodeName,
 			RespElements: map[string]string{"x-minio-versions": strconv.Itoa(len(fivs))},
 		})
 	}
 
-	return fivs, nil
+	return objInfos, nil
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -1168,6 +1178,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 }
 
 func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
+	traceFn := globalLifecycleSys.trace(obj)
 	opts := ObjectOptions{
 		Expiration: ExpirationOptions{Expire: true},
 	}
@@ -1191,7 +1202,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	}
 
 	// Send audit for the lifecycle delete operation
-	auditLogLifecycle(ctx, obj, ILMExpiry)
+	auditLogLifecycle(ctx, obj, ILMExpiry, traceFn)
 
 	eventName := event.ObjectRemovedDelete
 	if obj.DeleteMarker {
@@ -1203,7 +1214,8 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		EventName:  eventName,
 		BucketName: obj.Bucket,
 		Object:     obj,
-		Host:       "Internal: [ILM-Expiry]",
+		UserAgent:  "Internal: [ILM-Expiry]",
+		Host:       globalLocalNodeName,
 	})
 
 	return true
@@ -1273,8 +1285,7 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 		sizeS.replTargetStats[arn] = tgtSizeS
 	}
 
-	switch oi.ReplicationStatus {
-	case replication.Replica:
+	if oi.ReplicationStatus == replication.Replica {
 		sizeS.replicaSize += oi.Size
 	}
 }
@@ -1434,7 +1445,7 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, traceFn func(event string)) {
 	var apiName string
 	switch event {
 	case ILMExpiry:
@@ -1451,4 +1462,5 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string) {
 		Object:    oi.Name,
 		VersionID: oi.VersionID,
 	})
+	traceFn(event)
 }

cmd/data-usage-cache.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"path"
 	"path/filepath"
@@ -47,6 +48,9 @@ type dataUsageHash string
 // sizeHistogram is a size histogram.
 type sizeHistogram [dataUsageBucketLen]uint64
 
+// versionsHistogram is a histogram of number of versions in an object.
+type versionsHistogram [dataUsageVersionLen]uint64
+
 type dataUsageEntry struct {
 	Children dataUsageHashMap `msg:"ch"`
 	// These fields do no include any children.
@@ -54,6 +58,7 @@ type dataUsageEntry struct {
 	Objects          uint64               `msg:"os"`
 	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.
 	ObjSizes         sizeHistogram        `msg:"szs"`
+	ObjVersions      versionsHistogram    `msg:"vh"`
 	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
 	AllTierStats     *allTierStats        `msg:"ats,omitempty"`
 	Compacted        bool                 `msg:"c"`
@@ -291,6 +296,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.Size += summary.totalSize
 	e.Versions += summary.versions
 	e.ObjSizes.add(summary.totalSize)
+	e.ObjVersions.add(summary.versions)
 
 	if e.ReplicationStats == nil {
 		e.ReplicationStats = &replicationAllStats{
@@ -351,6 +357,10 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 		e.ObjSizes[i] += v
 	}
 
+	for i, v := range other.ObjVersions[:] {
+		e.ObjVersions[i] += v
+	}
+
 	if other.AllTierStats != nil {
 		if e.AllTierStats == nil {
 			e.AllTierStats = newAllTierStats()
@@ -694,7 +704,7 @@ func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
 func (h *sizeHistogram) add(size int64) {
 	// Fetch the histogram interval corresponding
 	// to the passed object size.
-	for i, interval := range ObjectsHistogramIntervals {
+	for i, interval := range ObjectsHistogramIntervals[:] {
 		if size >= interval.start && size <= interval.end {
 			h[i]++
 			break
@@ -711,6 +721,27 @@ func (h *sizeHistogram) toMap() map[string]uint64 {
 	return res
 }
 
+// add a version count to the histogram.
+func (h *versionsHistogram) add(versions uint64) {
+	// Fetch the histogram interval corresponding
+	// to the passed object size.
+	for i, interval := range ObjectsVersionCountIntervals[:] {
+		if versions >= uint64(interval.start) && versions <= uint64(interval.end) {
+			h[i]++
+			break
+		}
+	}
+}
+
+// toMap returns the map to a map[string]uint64.
+func (h *versionsHistogram) toMap() map[string]uint64 {
+	res := make(map[string]uint64, dataUsageVersionLen)
+	for i, count := range h {
+		res[ObjectsVersionCountIntervals[i].name] = count
+	}
+	return res
+}
+
 func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats {
 	dst := newAllTierStats()
 	for _, bucket := range buckets {
@@ -741,10 +772,11 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		}
 		flat := d.flatten(*e)
 		bui := BucketUsageInfo{
-			Size:                 uint64(flat.Size),
-			VersionsCount:        flat.Versions,
-			ObjectsCount:         flat.Objects,
-			ObjectSizesHistogram: flat.ObjSizes.toMap(),
+			Size:                    uint64(flat.Size),
+			VersionsCount:           flat.Versions,
+			ObjectsCount:            flat.Objects,
+			ObjectSizesHistogram:    flat.ObjSizes.toMap(),
+			ObjectVersionsHistogram: flat.ObjVersions.toMap(),
 		}
 		if flat.ReplicationStats != nil {
 			bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
@@ -847,37 +879,54 @@ type objectIO interface {
 
 // load the cache content with name from minioMetaBackgroundOpsBucket.
 // Only backend errors are returned as errors.
+// The loader is optimistic and has no locking, but tries 5 times before giving up.
 // If the object is not found or unable to deserialize d is cleared and nil error is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
 	// Abandon if more than 5 minutes, so we don't hold up scanner.
 	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
 	defer cancel()
 
-	r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, readLock, ObjectOptions{})
-	if err != nil {
-		switch err.(type) {
-		case ObjectNotFound:
-		case BucketNotFound:
-		case InsufficientReadQuorum:
-		case StorageErr:
-		default:
-			return toObjectErr(err, dataUsageBucket, name)
+	// Caches are read+written without locks,
+	retries := 0
+	for retries < 5 {
+		r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, noLock, ObjectOptions{NoLock: true})
+		if err != nil {
+			switch err.(type) {
+			case ObjectNotFound, BucketNotFound:
+			case InsufficientReadQuorum, StorageErr:
+				retries++
+				time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
+				continue
+			default:
+				return toObjectErr(err, dataUsageBucket, name)
+			}
+			*d = dataUsageCache{}
+			return nil
 		}
-		*d = dataUsageCache{}
+		if err := d.deserialize(r); err != nil {
+			r.Close()
+			retries++
+			time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
+			continue
+		}
+		r.Close()
 		return nil
 	}
-	defer r.Close()
-	if err := d.deserialize(r); err != nil {
-		*d = dataUsageCache{}
-		logger.LogOnceIf(ctx, err, err.Error())
-	}
+	*d = dataUsageCache{}
 	return nil
 }
 
+// Maximum running concurrent saves on server.
+var maxConcurrentScannerSaves = make(chan struct{}, 4)
+
 // save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
+// Note that no locking is done when saving.
 func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
 	var r io.Reader
-
+	maxConcurrentScannerSaves <- struct{}{}
+	defer func() {
+		<-maxConcurrentScannerSaves
+	}()
 	// If big, do streaming...
 	size := int64(-1)
 	if len(d.Cache) > 10000 {
@@ -909,7 +958,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 		dataUsageBucket,
 		name,
 		NewPutObjReader(hr),
-		ObjectOptions{})
+		ObjectOptions{NoLock: true})
 	if isErrBucketNotFound(err) {
 		return nil
 	}

cmd/data-usage-cache_gen.go

@@ -1540,6 +1540,24 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "vh":
+			var zb0003 uint32
+			zb0003, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "ObjVersions")
+				return
+			}
+			if zb0003 != uint32(dataUsageVersionLen) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
+				return
+			}
+			for za0002 := range z.ObjVersions {
+				z.ObjVersions[za0002], err = dc.ReadUint64()
+				if err != nil {
+					err = msgp.WrapError(err, "ObjVersions", za0002)
+					return
+				}
+			}
 		case "rs":
 			if dc.IsNil() {
 				err = dc.ReadNil()
@@ -1596,16 +1614,16 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
-	zb0001Len := uint32(8)
-	var zb0001Mask uint8 /* 8 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.ReplicationStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x20
+		zb0001Mask |= 0x40
 	}
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x40
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	err = en.Append(0x80 | uint8(zb0001Len))
@@ -1672,7 +1690,24 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	if (zb0001Mask & 0x20) == 0 { // if not empty
+	// write "vh"
+	err = en.Append(0xa2, 0x76, 0x68)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
+	if err != nil {
+		err = msgp.WrapError(err, "ObjVersions")
+		return
+	}
+	for za0002 := range z.ObjVersions {
+		err = en.WriteUint64(z.ObjVersions[za0002])
+		if err != nil {
+			err = msgp.WrapError(err, "ObjVersions", za0002)
+			return
+		}
+	}
+	if (zb0001Mask & 0x40) == 0 { // if not empty
 		// write "rs"
 		err = en.Append(0xa2, 0x72, 0x73)
 		if err != nil {
@@ -1691,7 +1726,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x40) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not empty
 		// write "ats"
 		err = en.Append(0xa3, 0x61, 0x74, 0x73)
 		if err != nil {
@@ -1727,16 +1762,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// omitempty: check for empty values
-	zb0001Len := uint32(8)
-	var zb0001Mask uint8 /* 8 bits */
+	zb0001Len := uint32(9)
+	var zb0001Mask uint16 /* 9 bits */
 	_ = zb0001Mask
 	if z.ReplicationStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x20
+		zb0001Mask |= 0x40
 	}
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x40
+		zb0001Mask |= 0x80
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
@@ -1765,7 +1800,13 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	for za0001 := range z.ObjSizes {
 		o = msgp.AppendUint64(o, z.ObjSizes[za0001])
 	}
-	if (zb0001Mask & 0x20) == 0 { // if not empty
+	// string "vh"
+	o = append(o, 0xa2, 0x76, 0x68)
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
+	for za0002 := range z.ObjVersions {
+		o = msgp.AppendUint64(o, z.ObjVersions[za0002])
+	}
+	if (zb0001Mask & 0x40) == 0 { // if not empty
 		// string "rs"
 		o = append(o, 0xa2, 0x72, 0x73)
 		if z.ReplicationStats == nil {
@@ -1778,7 +1819,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x40) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not empty
 		// string "ats"
 		o = append(o, 0xa3, 0x61, 0x74, 0x73)
 		if z.AllTierStats == nil {
@@ -1857,6 +1898,24 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "vh":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjVersions")
+				return
+			}
+			if zb0003 != uint32(dataUsageVersionLen) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
+				return
+			}
+			for za0002 := range z.ObjVersions {
+				z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "ObjVersions", za0002)
+					return
+				}
+			}
 		case "rs":
 			if msgp.IsNil(bts) {
 				bts, err = msgp.ReadNilBytes(bts)
@@ -1911,7 +1970,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageEntry) Msgsize() (s int) {
-	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3
+	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
 	if z.ReplicationStats == nil {
 		s += msgp.NilSize
 	} else {
@@ -3704,3 +3763,81 @@ func (z tierStats) Msgsize() (s int) {
 	s = 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
 	return
 }
+
+// DecodeMsg implements msgp.Decodable
+func (z *versionsHistogram) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageVersionLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *versionsHistogram) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(dataUsageVersionLen))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for za0001 := range z {
+		err = en.WriteUint64(z[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *versionsHistogram) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen))
+	for za0001 := range z {
+		o = msgp.AppendUint64(o, z[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *versionsHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageVersionLen) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *versionsHistogram) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size))
+	return
+}

cmd/data-usage-cache_gen_test.go

@@ -1196,3 +1196,116 @@ func BenchmarkDecodetierStats(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalversionsHistogram(t *testing.T) {
+	v := versionsHistogram{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeversionsHistogram(t *testing.T) {
+	v := versionsHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeversionsHistogram Msgsize() is inaccurate")
+	}
+
+	vn := versionsHistogram{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeversionsHistogram(b *testing.B) {
+	v := versionsHistogram{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/data-usage-utils.go

@@ -58,11 +58,12 @@ type BucketUsageInfo struct {
 	// Total number of objects that failed replication
 	ReplicationFailedCountV1 uint64 `json:"objectsFailedReplicationCount"`
 
-	ObjectsCount         uint64                           `json:"objectsCount"`
-	ObjectSizesHistogram map[string]uint64                `json:"objectsSizesHistogram"`
-	VersionsCount        uint64                           `json:"versionsCount"`
-	ReplicaSize          uint64                           `json:"objectReplicaTotalSize"`
-	ReplicationInfo      map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
+	ObjectsCount            uint64                           `json:"objectsCount"`
+	ObjectSizesHistogram    map[string]uint64                `json:"objectsSizesHistogram"`
+	ObjectVersionsHistogram map[string]uint64                `json:"objectsVersionsHistogram"`
+	VersionsCount           uint64                           `json:"versionsCount"`
+	ReplicaSize             uint64                           `json:"objectReplicaTotalSize"`
+	ReplicationInfo         map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
 }
 
 // DataUsageInfo represents data usage stats of the underlying Object API

cmd/disk-cache.go

@@ -25,7 +25,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
@@ -35,6 +34,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
+	xnet "github.com/minio/pkg/net"
 	"github.com/minio/pkg/wildcard"
 )
 
@@ -448,7 +448,10 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 			return cachedObjInfo, nil
 		}
 		c.cacheStats.incMiss()
-		return ObjectInfo{}, BackendDown{}
+		if xnet.IsNetworkOrHostDown(err, false) {
+			return ObjectInfo{}, BackendDown{Err: err.Error()}
+		}
+		return ObjectInfo{}, err
 	}
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
@@ -893,8 +896,16 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 				cacheDiskStats[i].UsageSize = info.Used
 				cacheDiskStats[i].TotalCapacity = info.Total
 				cacheDiskStats[i].Dir = dcache.stats.Dir
-				atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&dcache.stats.UsageState))
-				atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&dcache.stats.UsagePercent))
+				if info.Total != 0 {
+					// UsageState
+					gcTriggerPct := dcache.quotaPct * dcache.highWatermark / 100
+					usedPercent := float64(info.Used) * 100 / float64(info.Total)
+					if usedPercent >= float64(gcTriggerPct) {
+						cacheDiskStats[i].UsageState = 1
+					}
+					// UsagePercent
+					cacheDiskStats[i].UsagePercent = uint64(usedPercent)
+				}
 			}
 		}
 		return cacheDiskStats

cmd/encryption-v1.go

@@ -110,7 +110,7 @@ func kmsKeyIDFromMetadata(metadata map[string]string) string {
 //
 // DecryptETags uses a KMS bulk decryption API, if available, which
 // is more efficient than decrypting ETags sequentually.
-func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error {
+func DecryptETags(ctx context.Context, k kms.KMS, objects []ObjectInfo) error {
 	const BatchSize = 250 // We process the objects in batches - 250 is a reasonable default.
 	var (
 		metadata = make([]map[string]string, 0, BatchSize)
@@ -170,7 +170,7 @@ func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error
 		// For all SSE-S3 single-part objects we have to
 		// fetch their decryption keys. We do this using
 		// a Bulk-Decryption API call, if available.
-		keys, err := crypto.S3.UnsealObjectKeys(ctx, KMS, metadata, buckets, names)
+		keys, err := crypto.S3.UnsealObjectKeys(ctx, k, metadata, buckets, names)
 		if err != nil {
 			return err
 		}

cmd/endpoint_test.go

@@ -62,13 +62,14 @@ func TestNewEndpoint(t *testing.T) {
 				err = endpoint.UpdateIsLocal()
 			}
 
-			if test.expectedErr == nil {
+			switch {
+			case test.expectedErr == nil:
 				if err != nil {
 					t.Errorf("error: expected = <nil>, got = %v", err)
 				}
-			} else if err == nil {
+			case err == nil:
 				t.Errorf("error: expected = %v, got = <nil>", test.expectedErr)
-			} else if test.expectedErr.Error() != err.Error() {
+			case test.expectedErr.Error() != err.Error():
 				t.Errorf("error: expected = %v, got = %v", test.expectedErr, err)
 			}
 
@@ -115,13 +116,14 @@ func TestNewEndpoints(t *testing.T) {
 
 	for _, testCase := range testCases {
 		_, err := NewEndpoints(testCase.args...)
-		if testCase.expectedErr == nil {
+		switch {
+		case testCase.expectedErr == nil:
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)
 			}
-		} else if err == nil {
+		case err == nil:
 			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
+		case testCase.expectedErr.Error() != err.Error():
 			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
 		}
 	}

cmd/erasure-healing.go

@@ -294,6 +294,26 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 	return false
 }
 
+const xMinIOHealing = ReservedMetadataPrefix + "healing"
+
+// SetHealing marks object (version) as being healed.
+// Note: this is to be used only from healObject
+func (fi *FileInfo) SetHealing() {
+	if fi.Metadata == nil {
+		fi.Metadata = make(map[string]string)
+	}
+	fi.Metadata[xMinIOHealing] = "true"
+}
+
+// Healing returns true if object is being healed (i.e fi is being passed down
+// from healObject)
+func (fi FileInfo) Healing() bool {
+	if _, ok := fi.Metadata[xMinIOHealing]; ok {
+		return true
+	}
+	return false
+}
+
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
 	dryRun := opts.DryRun
@@ -505,8 +525,34 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	migrateDataDir := mustGetUUID()
 
 	// Reorder so that we have data disks first and parity disks next.
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(availableDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from available disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, availableDisks, bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	latestDisks := shuffleDisks(availableDisks, latestMeta.Erasure.Distribution)
+
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(outDatedDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from outdated disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, outDatedDisks, bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution)
+
+	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(partsMetadata) {
+		err := fmt.Errorf("unexpected file distribution (%v) from metadata entries (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			latestMeta.Erasure.Distribution, len(partsMetadata), bucket, object, versionID)
+		logger.LogIf(ctx, err)
+		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
+			bucket, object, versionID), err
+	}
+
 	partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution)
 
 	copyPartsMetadata := make([]FileInfo, len(partsMetadata))
@@ -638,6 +684,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		partsMetadata[i].Erasure.Index = i + 1
 
 		// Attempt a rename now from healed data to final location.
+		partsMetadata[i].SetHealing()
 		if _, err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
 			logger.LogIf(ctx, err)
 			return result, err
@@ -910,13 +957,14 @@ func isObjectDirDangling(errs []error) (ok bool) {
 	var foundNotEmpty int
 	var otherFound int
 	for _, readErr := range errs {
-		if readErr == nil {
+		switch {
+		case readErr == nil:
 			found++
-		} else if readErr == errFileNotFound || readErr == errVolumeNotFound {
+		case readErr == errFileNotFound || readErr == errVolumeNotFound:
 			notFound++
-		} else if readErr == errVolumeNotEmpty {
+		case readErr == errVolumeNotEmpty:
 			foundNotEmpty++
-		} else {
+		default:
 			otherFound++
 		}
 	}
@@ -938,11 +986,12 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 			diskNotFoundCount int
 		)
 		for _, readErr := range cerrs {
-			if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
+			switch {
+			case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
 				notFoundCount++
-			} else if errors.Is(readErr, errFileCorrupt) {
+			case errors.Is(readErr, errFileCorrupt):
 				corruptedCount++
-			} else if errors.Is(readErr, errDiskNotFound) {
+			case errors.Is(readErr, errDiskNotFound):
 				diskNotFoundCount++
 			}
 		}

cmd/erasure-healing_test.go

@@ -1621,10 +1621,10 @@ func TestHealLastDataShard(t *testing.T) {
 			}
 
 			firstGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
-			defer firstGr.Close()
 			if err != nil {
 				t.Fatal(err)
 			}
+			defer firstGr.Close()
 
 			firstHealedH := sha256.New()
 			_, err = io.Copy(firstHealedH, firstGr)
@@ -1651,10 +1651,10 @@ func TestHealLastDataShard(t *testing.T) {
 			}
 
 			secondGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
-			defer secondGr.Close()
 			if err != nil {
 				t.Fatal(err)
 			}
+			defer secondGr.Close()
 
 			secondHealedH := sha256.New()
 			_, err = io.Copy(secondHealedH, secondGr)

cmd/erasure-object.go

@@ -658,6 +658,13 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		return fi, nil, nil, err
 	}
 
+	if !fi.Deleted && len(fi.Erasure.Distribution) != len(onlineDisks) {
+		err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
+			fi.Erasure.Distribution, onlineDisks, bucket, object, opts.VersionID)
+		logger.LogIf(ctx, err)
+		return fi, nil, nil, toObjectErr(err, bucket, object, opts.VersionID)
+	}
+
 	filterOnlineDisksInplace(fi, metaArr, onlineDisks)
 
 	// if one of the disk is offline, return right here no need
@@ -972,17 +979,14 @@ func healObjectVersionsDisparity(bucket string, entry metaCacheEntry) error {
 
 	fivs, err := entry.fileInfoVersions(bucket)
 	if err != nil {
-		go healObject(bucket, entry.name, "", madmin.HealNormalScan)
+		go healObject(bucket, entry.name, "", madmin.HealDeepScan)
 		return err
 	}
 
-	if len(fivs.Versions) > 2 {
-		return fmt.Errorf("bucket(%s)/object(%s) object with %d versions needs healing, allowing lazy healing via scanner instead here for versions greater than 2",
-			bucket, entry.name, len(fivs.Versions))
-	}
-
-	for _, version := range fivs.Versions {
-		go healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
+	if len(fivs.Versions) <= 2 {
+		for _, version := range fivs.Versions {
+			go healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
+		}
 	}
 
 	return nil
@@ -994,7 +998,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	if opts.CheckPrecondFn != nil {
 		obj, err := er.getObjectInfo(ctx, bucket, object, opts)
-		if err != nil && !isErrVersionNotFound(err) {
+		if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
 			return objInfo, err
 		}
 		if opts.CheckPrecondFn(obj) {
@@ -1474,8 +1478,11 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 
 	// Check failed deletes across multiple objects
 	for i, dobj := range dobjects {
-		// This object errored, no need to attempt a heal.
-		if errs[i] != nil {
+		// This object errored, we should attempt a heal just in case.
+		if errs[i] != nil && !isErrVersionNotFound(errs[i]) && !isErrObjectNotFound(errs[i]) {
+			// all other direct versionId references we should
+			// ensure no dangling file is left over.
+			er.addPartial(bucket, dobj.ObjectName, dobj.VersionID, -1)
 			continue
 		}
 
@@ -1566,8 +1573,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
 	goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
 	if gerr != nil && goi.Name == "" {
-		switch gerr.(type) {
-		case InsufficientReadQuorum:
+		if _, ok := gerr.(InsufficientReadQuorum); ok {
 			return objInfo, InsufficientWriteQuorum{}
 		}
 		// For delete marker replication, versionID being replicated will not exist on disk
@@ -1645,6 +1651,18 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	}
 	fvID := mustGetUUID()
 
+	defer func() {
+		// attempt a heal before returning if there are offline disks
+		// for both del marker and permanent delete situations.
+		for _, disk := range storageDisks {
+			if disk != nil && disk.IsOnline() {
+				continue
+			}
+			er.addPartial(bucket, object, opts.VersionID, -1)
+			break
+		}
+	}()
+
 	if markDelete && (opts.Versioned || opts.VersionSuspended) {
 		if !deleteMarker {
 			// versioning suspended means we add `null` version as
@@ -1692,14 +1710,6 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 
-	for _, disk := range storageDisks {
-		if disk != nil && disk.IsOnline() {
-			continue
-		}
-		er.addPartial(bucket, object, opts.VersionID, -1)
-		break
-	}
-
 	return dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
@@ -1943,6 +1953,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 			return toObjectErr(err, bucket, object)
 		}
 	}
+	traceFn := globalLifecycleSys.trace(fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended))
 
 	destObj, err := genTransitionObjName(bucket)
 	if err != nil {
@@ -1987,9 +1998,10 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		EventName:  eventName,
 		BucketName: bucket,
 		Object:     objInfo,
-		Host:       "Internal: [ILM-Transition]",
+		UserAgent:  "Internal: [ILM-Transition]",
+		Host:       globalLocalNodeName,
 	})
-	auditLogLifecycle(ctx, objInfo, ILMTransition)
+	auditLogLifecycle(ctx, objInfo, ILMTransition, traceFn)
 	return err
 }
 
@@ -2093,3 +2105,46 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 	})
 	return setRestoreHeaderFn(oi, err)
 }
+
+// DecomTieredObject - moves tiered object to another pool during decommissioning.
+func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
+	if opts.UserDefined == nil {
+		opts.UserDefined = make(map[string]string)
+	}
+	// overlay Erasure info for this set of disks
+	storageDisks := er.getDisks()
+	// Get parity and data drive count based on storage class metadata
+	parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass])
+	if parityDrives < 0 {
+		parityDrives = er.defaultParityCount
+	}
+	dataDrives := len(storageDisks) - parityDrives
+
+	// we now know the number of blocks this object needs for data and parity.
+	// writeQuorum is dataBlocks + 1
+	writeQuorum := dataDrives
+	if dataDrives == parityDrives {
+		writeQuorum++
+	}
+
+	// Initialize parts metadata
+	partsMetadata := make([]FileInfo, len(storageDisks))
+
+	fi2 := newFileInfo(pathJoin(bucket, object), dataDrives, parityDrives)
+	fi.Erasure = fi2.Erasure
+	// Initialize erasure metadata.
+	for index := range partsMetadata {
+		partsMetadata[index] = fi
+		partsMetadata[index].Erasure.Index = index + 1
+	}
+
+	// Order disks according to erasure distribution
+	var onlineDisks []StorageAPI
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi)
+
+	if _, err := writeUniqueFileInfo(ctx, onlineDisks, bucket, object, partsMetadata, writeQuorum); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	return nil
+}

cmd/erasure-server-pool-decom.go

@@ -32,7 +32,6 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v2"
-	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
@@ -735,18 +734,18 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			}
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
+
 			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
-			switch evt.Action {
-			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
-				// Skip this entry.
-				return true
-			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-				globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
-				// Skip this entry.
+			switch {
+			case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
+				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt.Action.DeleteRestored())
+				return false
+			case evt.Action.Delete():
+				globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
 				return true
+			default:
+				return false
 			}
-			return false
 		}
 
 		decommissionEntry := func(entry metaCacheEntry) {
@@ -770,12 +769,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 			var decommissionedCount int
 			for _, version := range fivs.Versions {
-				// TODO: Skip transitioned objects for now.
-				if version.IsRemote() {
-					logger.LogIf(ctx, fmt.Errorf("found %s/%s transitioned object, transitioned object won't be decommissioned", bi.Name, version.Name))
-					continue
-				}
-
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bi.Name, version.Name, version) {
 					logger.LogIf(ctx, fmt.Errorf("found %s/%s (%s) expired object based on ILM rules, skipping and scheduled for deletion", bi.Name, version.Name, version.VersionID))
@@ -824,6 +817,22 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				var failure, ignore bool
 				// gr.Close() is ensured by decommissionObject().
 				for try := 0; try < 3; try++ {
+					if version.IsRemote() {
+						stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
+						if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
+							VersionID:   version.VersionID,
+							MTime:       version.ModTime,
+							UserDefined: version.Metadata,
+						}); err != nil {
+							stopFn(err)
+							failure = true
+							logger.LogIf(ctx, err)
+							continue
+						}
+						stopFn(nil)
+						failure = false
+						break
+					}
 					gr, err := set.GetObjectNInfo(ctx,
 						bi.Name,
 						encodeDirObject(version.Name),
@@ -1271,17 +1280,6 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, indices ...i
 		z.HealBucket(ctx, bucket.Name, madmin.HealOpts{})
 	}
 
-	// TODO: Support decommissioning transition tiers.
-	for _, bucket := range decomBuckets {
-		if lc, err := globalLifecycleSys.Get(bucket.Name); err == nil {
-			if lc.HasTransition() {
-				return decomError{
-					Err: fmt.Sprintf("Bucket is part of transitioned tier %s: decommission is not allowed in Tier'd setups", bucket.Name),
-				}
-			}
-		}
-	}
-
 	// Create .minio.sys/conifg, .minio.sys/buckets paths if missing,
 	// this code is present to avoid any missing meta buckets on other
 	// pools.

cmd/erasure-server-pool-rebalance.go

@@ -32,7 +32,6 @@ import (
 
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v2"
-	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
@@ -463,17 +462,13 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 			}
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
-			event := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
-			switch action := event.Action; action {
-			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
-				// Skip this entry.
-				return true
-			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-				globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
-				// Skip this entry.
+
+			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			if evt.Action.Delete() {
+				globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
 				return true
 			}
+
 			return false
 		}
 

cmd/erasure-server-pool.go

@@ -37,7 +37,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -157,7 +156,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		break
 	}
 
+	globalLocalDrivesMu.Lock()
 	globalLocalDrives = localDrives
+	defer globalLocalDrivesMu.Unlock()
 	return z, nil
 }
 
@@ -345,11 +346,17 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 	g.Wait()
 
 	for i, zinfo := range storageInfos {
-		var available uint64
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(zinfo, size) {
+		if zinfo == nil {
 			serverPools[i] = poolAvailableSpace{Index: i}
 			continue
 		}
+		var available uint64
+		if !isMinioMetaBucketName(bucket) {
+			if avail, err := hasSpaceFor(zinfo, size); err != nil && !avail {
+				serverPools[i] = poolAvailableSpace{Index: i}
+				continue
+			}
+		}
 		var maxUsedPct int
 		for _, disk := range zinfo {
 			if disk == nil || disk.Total == 0 {
@@ -421,36 +428,48 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
 		return mtime1.After(mtime2)
 	})
 
+	defPool := PoolObjInfo{Index: -1}
 	for _, pinfo := range poolObjInfos {
 		// skip all objects from suspended pools if asked by the
 		// caller.
-		if z.IsSuspended(pinfo.Index) && opts.SkipDecommissioned {
+		if opts.SkipDecommissioned && z.IsSuspended(pinfo.Index) {
 			continue
 		}
 		// Skip object if it's from pools participating in a rebalance operation.
 		if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) {
 			continue
 		}
-
-		if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
+		if pinfo.Err == nil {
+			// found a pool
+			return pinfo, nil
+		}
+		if isErrReadQuorum(pinfo.Err) {
+			// read quorum is returned when the object is visibly
+			// present but its unreadable, we simply ask the writes to
+			// schedule to this pool instead. If there is no quorum
+			// it will fail anyways, however if there is quorum available
+			// with enough disks online but sufficiently inconsistent to
+			// break parity threshold, allow them to be overwritten
+			// or allow new versions to be added.
+			return pinfo, nil
+		}
+		defPool = pinfo
+		if !isErrObjectNotFound(pinfo.Err) {
 			return pinfo, pinfo.Err
 		}
 
-		if isErrObjectNotFound(pinfo.Err) {
-			// No object exists or its a delete marker,
-			// check objInfo to confirm.
-			if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
-				return pinfo, nil
-			}
-
-			// objInfo is not valid, truly the object doesn't
-			// exist proceed to next pool.
-			continue
+		// No object exists or its a delete marker,
+		// check objInfo to confirm.
+		if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
+			return pinfo, nil
 		}
-
-		return pinfo, nil
 	}
-
+	if opts.ReplicationRequest && opts.DeleteMarker && defPool.Index >= 0 {
+		// If the request is a delete marker replication request, return a default pool
+		// in cases where the object does not exist.
+		// This is to ensure that the delete marker is replicated to the destination.
+		return defPool, nil
+	}
 	return PoolObjInfo{}, toObjectErr(errFileNotFound, bucket, object)
 }
 
@@ -689,9 +708,11 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataU
 	case updateCloser <- ch:
 		<-ch
 	case <-ctx.Done():
+		mu.Lock()
 		if firstErr == nil {
 			firstErr = ctx.Err()
 		}
+		defer mu.Unlock()
 	}
 	return firstErr
 }
@@ -926,8 +947,15 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 	object = encodeDirObject(object)
 
 	if z.SinglePool() {
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), data.Size()) {
-			return ObjectInfo{}, toObjectErr(errDiskFull)
+		if !isMinioMetaBucketName(bucket) {
+			avail, err := hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), data.Size())
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return ObjectInfo{}, toObjectErr(errErasureWriteQuorum)
+			}
+			if !avail {
+				return ObjectInfo{}, toObjectErr(errDiskFull)
+			}
 		}
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
 	}
@@ -985,8 +1013,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	gopts.NoLock = true
 	pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts)
 	if err != nil {
-		switch err.(type) {
-		case InsufficientReadQuorum:
+		if _, ok := err.(InsufficientReadQuorum); ok {
 			return objInfo, InsufficientWriteQuorum{}
 		}
 		return objInfo, err
@@ -1299,11 +1326,12 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		if err == nil {
 			if opts.Lifecycle != nil {
 				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
-				switch evt.Action {
-				case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-					fallthrough
-				case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-					return loi, nil
+				if evt.Action.Delete() {
+					globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
+					if !evt.Action.DeleteRestored() {
+						// Skip entry if ILM action was DeleteVersionAction or DeleteAction
+						return loi, nil
+					}
 				}
 			}
 			loi.Objects = append(loi.Objects, objInfo)
@@ -1391,8 +1419,15 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}
 
 	if z.SinglePool() {
-		if !isMinioMetaBucketName(bucket) && !hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), -1) {
-			return nil, toObjectErr(errDiskFull)
+		if !isMinioMetaBucketName(bucket) {
+			avail, err := hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), -1)
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return nil, toObjectErr(errErasureWriteQuorum)
+			}
+			if !avail {
+				return nil, toObjectErr(errDiskFull)
+			}
 		}
 		return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
 	}
@@ -1452,8 +1487,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 		if err == nil {
 			return pi, nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			// Look for information on the next pool
 			continue
 		}
@@ -1484,8 +1518,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 		if err == nil {
 			return mi, nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			// upload id not found, continue to the next pool.
 			continue
 		}
@@ -1516,8 +1549,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		if err == nil {
 			return result, nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			continue
 		}
 		return ListPartsInfo{}, err
@@ -1547,8 +1579,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		if err == nil {
 			return nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			// upload id not found move to next pool
 			continue
 		}
@@ -1579,8 +1610,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 		if err == nil {
 			return objInfo, nil
 		}
-		switch err.(type) {
-		case InvalidUploadID:
+		if _, ok := err.(InvalidUploadID); ok {
 			// upload id not found move to next pool
 			continue
 		}
@@ -1635,15 +1665,14 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 	}
 
 	err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
-	if err == nil || errors.Is(err, errVolumeNotFound) {
+	if err == nil || isErrBucketNotFound(err) {
 		// If site replication is configured, hold on to deleted bucket state until sites sync
-		switch opts.SRDeleteOp {
-		case MarkDelete:
+		if opts.SRDeleteOp == MarkDelete {
 			z.s3Peer.MakeBucket(context.Background(), pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket), MakeBucketOptions{})
 		}
 	}
 
-	if err != nil && !errors.Is(err, errVolumeNotFound) {
+	if err != nil && !isErrBucketNotFound(err) {
 		if !opts.NoRecreate {
 			z.s3Peer.MakeBucket(ctx, bucket, MakeBucketOptions{})
 		}
@@ -1742,8 +1771,7 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
 	for _, pool := range z.serverPools {
 		result, err := pool.HealBucket(ctx, bucket, opts)
 		if err != nil {
-			switch err.(type) {
-			case BucketNotFound:
+			if _, ok := err.(BucketNotFound); ok {
 				continue
 			}
 			return result, err
@@ -2076,9 +2104,14 @@ func (z *erasureServerPools) getPoolAndSet(id string) (poolIdx, setIdx, diskIdx
 	return -1, -1, -1, fmt.Errorf("DriveID(%s) %w", id, errDiskNotFound)
 }
 
+const (
+	vmware = "VMWare"
+)
+
 // HealthOptions takes input options to return sepcific information
 type HealthOptions struct {
-	Maintenance bool
+	Maintenance    bool
+	DeploymentType string
 }
 
 // HealthResult returns the current state of the system, also
@@ -2164,7 +2197,8 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	}
 
 	var aggHealStateResult madmin.BgHealState
-	if opts.Maintenance {
+	// Check if disks are healing on in-case of VMware vsphere deployments.
+	if opts.Maintenance && opts.DeploymentType == vmware {
 		// check if local disks are being healed, if they are being healed
 		// we need to tell healthy status as 'false' so that this server
 		// is not taken down for maintenance
@@ -2347,3 +2381,27 @@ func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, ob
 	}
 	return nil
 }
+
+// DecomTieredObject - moves tiered object to another pool during decommissioning.
+func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
+	object = encodeDirObject(object)
+	if z.SinglePool() {
+		return fmt.Errorf("error decommissioning %s/%s", bucket, object)
+	}
+	if !opts.NoLock {
+		ns := z.NewNSLock(bucket, object)
+		lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return err
+		}
+		ctx = lkctx.Context()
+		defer ns.Unlock(lkctx)
+		opts.NoLock = true
+	}
+	idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size)
+	if err != nil {
+		return err
+	}
+
+	return z.serverPools[idx].DecomTieredObject(ctx, bucket, object, fi, opts)
+}

cmd/erasure-sets.go

@@ -1215,6 +1215,12 @@ func (s *erasureSets) PutObjectMetadata(ctx context.Context, bucket, object stri
 	return er.PutObjectMetadata(ctx, bucket, object, opts)
 }
 
+// DecomTieredObject - moves tiered object to another pool during decommissioning.
+func (s *erasureSets) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
+	er := s.getHashedSet(object)
+	return er.DecomTieredObject(ctx, bucket, object, fi, opts)
+}
+
 // PutObjectTags - replace or add tags to an existing object
 func (s *erasureSets) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) {
 	er := s.getHashedSet(object)

cmd/erasure.go

@@ -207,6 +207,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				Scanning:       info.Scanning,
 				State:          diskErrToDriveState(err),
 				FreeInodes:     info.FreeInodes,
+				UsedInodes:     info.UsedInodes,
 			}
 			di.PoolIndex, di.SetIndex, di.DiskIndex = disks[index].GetDiskLoc()
 			if info.Healing {
@@ -412,9 +413,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 
 		for {
 			select {
-			case <-ctx.Done():
-				// Return without saving.
-				return
 			case <-t.C:
 				if cache.Info.LastUpdate.Equal(lastSave) {
 					continue
@@ -452,6 +450,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 	// Start one scanner per disk
 	var wg sync.WaitGroup
 	wg.Add(len(disks))
+
 	for i := range disks {
 		go func(i int) {
 			defer wg.Done()
@@ -488,10 +487,13 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				go func(name string) {
 					defer wg.Done()
 					for update := range updates {
-						bucketResults <- dataUsageEntryInfo{
+						select {
+						case <-ctx.Done():
+						case bucketResults <- dataUsageEntryInfo{
 							Name:   name,
 							Parent: dataUsageRoot,
 							Entry:  update,
+						}:
 						}
 					}
 				}(cache.Info.Name)
@@ -518,15 +520,16 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				if r := cache.root(); r != nil {
 					root = cache.flatten(*r)
 				}
-				t := time.Now()
-				bucketResults <- dataUsageEntryInfo{
+				select {
+				case <-ctx.Done():
+					return
+				case bucketResults <- dataUsageEntryInfo{
 					Name:   cache.Info.Name,
 					Parent: dataUsageRoot,
 					Entry:  root,
+				}:
 				}
-				// We want to avoid synchronizing up all writes in case
-				// the results are piled up.
-				time.Sleep(time.Duration(float64(time.Since(t)) * rand.Float64()))
+
 				// Save cache
 				logger.LogIf(ctx, cache.save(ctx, er, cacheName))
 			}

cmd/event-notification.go

@@ -101,7 +101,7 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
 		return errServerNotInitialized
 	}
 
-	if err := evnot.targetList.Add(globalConfigTargetList.Targets()...); err != nil {
+	if err := evnot.targetList.Add(globalNotifyTargetList.Targets()...); err != nil {
 		return err
 	}
 

cmd/generic-handlers.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"net/http"
 	"path"
+	"path/filepath"
 	"runtime/debug"
 	"strings"
 	"sync/atomic"
@@ -35,7 +36,6 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/http/stats"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
 )
@@ -111,6 +111,7 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrUnsupportedMetadata), r.URL)
 			return
 		}
@@ -121,6 +122,7 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsHeader, 1)
 			return
@@ -133,9 +135,8 @@ func setRequestLimitHandler(h http.Handler) http.Handler {
 
 // Reserved bucket.
 const (
-	minioReservedBucket              = "minio"
-	minioReservedBucketPath          = SlashSeparator + minioReservedBucket
-	minioReservedBucketPathWithSlash = SlashSeparator + minioReservedBucket + SlashSeparator
+	minioReservedBucket     = "minio"
+	minioReservedBucketPath = SlashSeparator + minioReservedBucket
 
 	loginPathPrefix = SlashSeparator + "login"
 )
@@ -277,60 +278,6 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
 	return time.Time{}, ErrMissingDateHeader
 }
 
-// splitStr splits a string into n parts, empty strings are added
-// if we are not able to reach n elements
-func splitStr(path, sep string, n int) []string {
-	splits := strings.SplitN(path, sep, n)
-	// Add empty strings if we found elements less than nr
-	for i := n - len(splits); i > 0; i-- {
-		splits = append(splits, "")
-	}
-	return splits
-}
-
-func url2Bucket(p string) (bucket string) {
-	tokens := splitStr(p, SlashSeparator, 3)
-	return tokens[1]
-}
-
-// setHttpStatsHandler sets a http Stats handler to gather HTTP statistics
-func setHTTPStatsHandler(h http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Meters s3 connection stats.
-		meteredRequest := &stats.IncomingTrafficMeter{ReadCloser: r.Body}
-		meteredResponse := &stats.OutgoingTrafficMeter{ResponseWriter: w}
-
-		// Execute the request
-		r.Body = meteredRequest
-		h.ServeHTTP(meteredResponse, r)
-
-		if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
-			strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
-			strings.HasPrefix(r.URL.Path, peerS3Prefix) ||
-			strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
-			globalConnStats.incInputBytes(meteredRequest.BytesRead())
-			globalConnStats.incOutputBytes(meteredResponse.BytesWritten())
-			return
-		}
-		if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
-			globalConnStats.incAdminInputBytes(meteredRequest.BytesRead())
-			globalConnStats.incAdminOutputBytes(meteredResponse.BytesWritten())
-			return
-		}
-
-		globalConnStats.incS3InputBytes(meteredRequest.BytesRead())
-		globalConnStats.incS3OutputBytes(meteredResponse.BytesWritten())
-
-		if r.URL != nil {
-			bucket := url2Bucket(r.URL.Path)
-			if bucket != "" && bucket != minioReservedBucket {
-				globalBucketConnStats.incS3InputBytes(bucket, meteredRequest.BytesRead())
-				globalBucketConnStats.incS3OutputBytes(bucket, meteredResponse.BytesWritten())
-			}
-		}
-	})
-}
-
 // Bad path components to be rejected by the path validity handler.
 const (
 	dotdotComponent = ".."
@@ -349,7 +296,7 @@ func hasBadHost(host string) error {
 // Check if the incoming path has bad path components,
 // such as ".." and "."
 func hasBadPathComponent(path string) bool {
-	path = strings.TrimSpace(path)
+	path = filepath.ToSlash(strings.TrimSpace(path)) // For windows '\' must be converted to '/'
 	for _, p := range strings.Split(path, SlashSeparator) {
 		switch strings.TrimSpace(p) {
 		case dotdotComponent:
@@ -388,6 +335,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (%s)", invalidReq.Description, err)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -402,6 +350,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 			atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 			return
@@ -415,6 +364,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 						tc.ResponseRecorder.LogErrBody = true
 					}
 
+					defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 					writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 					atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1)
 					return
@@ -427,6 +377,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 				tc.ResponseRecorder.LogErrBody = true
 			}
 
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest)
 			invalidReq.Description = fmt.Sprintf("%s (request has multiple authentication types, please use one)", invalidReq.Description)
 			writeErrorResponse(r.Context(), w, invalidReq, r.URL)
@@ -441,6 +392,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 					tc.FuncName = "handler.ValidRequest"
 					tc.ResponseRecorder.LogErrBody = true
 				}
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL)
 				return
 			}
@@ -453,6 +405,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = false
 				}
 
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest))
 			} else {
 				if ok {
@@ -460,6 +413,7 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = true
 				}
 
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL)
 			}
 			return
@@ -510,6 +464,7 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 		}
 		sr, err := globalDNSConfig.Get(bucket)
 		if err != nil {
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 			if err == dns.ErrNoEntriesFound {
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrNoSuchBucket), r.URL)
 			} else {
@@ -554,7 +509,7 @@ func addCustomHeaders(h http.Handler) http.Handler {
 		if globalLocalNodeName != "" {
 			w.Header().Set(xhttp.AmzRequestHostID, globalLocalNodeNameHex)
 		}
-		h.ServeHTTP(xhttp.NewResponseRecorder(w), r)
+		h.ServeHTTP(w, r)
 	})
 }
 
@@ -592,6 +547,7 @@ func setUploadForwardingHandler(h http.Handler) http.Handler {
 			h.ServeHTTP(w, r)
 			return
 		}
+
 		bucket, object := request2BucketObjectName(r)
 		uploadID := r.Form.Get(xhttp.UploadID)
 
@@ -608,6 +564,7 @@ func setUploadForwardingHandler(h http.Handler) http.Handler {
 			}
 			// forward request to peer handling this upload
 			if globalBucketTargetSys.isOffline(remote.EndpointURL) {
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrReplicationRemoteConnectionError), r.URL)
 				return
 			}

cmd/globals.go

@@ -49,6 +49,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	etcd "go.etcd.io/etcd/client/v3"
 
+	levent "github.com/minio/minio/internal/config/lambda/event"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/pkg/certs"
@@ -174,7 +175,8 @@ var (
 	globalMinioConsoleHost = ""
 
 	// Holds the possible host endpoint.
-	globalMinioEndpoint = ""
+	globalMinioEndpoint    = ""
+	globalMinioEndpointURL *xnet.URL
 
 	// globalConfigSys server config system.
 	globalConfigSys *ConfigSys
@@ -182,7 +184,8 @@ var (
 	globalNotificationSys *NotificationSys
 
 	globalEventNotifier    *EventNotifier
-	globalConfigTargetList *event.TargetList
+	globalNotifyTargetList *event.TargetList
+	globalLambdaTargetList *levent.TargetList
 
 	globalBucketMetadataSys *BucketMetadataSys
 	globalBucketMonitor     *bandwidth.Monitor
@@ -350,8 +353,10 @@ var (
 	globalServiceFreezeCnt int32
 	globalServiceFreezeMu  sync.Mutex // Updates.
 
-	// List of local drives to this node, this is only set during server startup.
-	globalLocalDrives []StorageAPI
+	// List of local drives to this node, this is only set during server startup,
+	// and should never be mutated. Hold globalLocalDrivesMu to access.
+	globalLocalDrives   []StorageAPI
+	globalLocalDrivesMu sync.RWMutex
 
 	// Is MINIO_CI_CD set?
 	globalIsCICD bool

cmd/handler-api.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/internal/config/api"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/mcontext"
 )
 
 type apiConfig struct {
@@ -272,6 +273,10 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 
 		globalHTTPStats.addRequestsInQueue(1)
 
+		if tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt); ok {
+			tc.FuncName = "s3.MaxClients"
+		}
+
 		deadlineTimer := time.NewTimer(deadline)
 		defer deadlineTimer.Stop()
 
@@ -288,6 +293,10 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 			globalHTTPStats.addRequestsInQueue(-1)
 			return
 		case <-r.Context().Done():
+			// When the client disconnects before getting the S3 handler
+			// status code response, set the status code to 499 so this request
+			// will be properly audited and traced.
+			w.WriteHeader(499)
 			globalHTTPStats.addRequestsInQueue(-1)
 			return
 		}

cmd/handler-utils.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/mcontext"
 	xnet "github.com/minio/pkg/net"
 )
 
@@ -355,11 +356,45 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 		globalHTTPStats.currentS3Requests.Inc(api)
 		defer globalHTTPStats.currentS3Requests.Dec(api)
 
-		statsWriter := xhttp.NewResponseRecorder(w)
+		f.ServeHTTP(w, r)
 
-		f.ServeHTTP(statsWriter, r)
+		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
+		if !ok {
+			return
+		}
 
-		globalHTTPStats.updateStats(api, r, statsWriter)
+		if tc != nil {
+			if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
+				strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
+				strings.HasPrefix(r.URL.Path, peerS3Prefix) ||
+				strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
+				globalConnStats.incInputBytes(int64(tc.RequestRecorder.Size()))
+				globalConnStats.incOutputBytes(int64(tc.ResponseRecorder.Size()))
+				return
+			}
+
+			if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
+				globalConnStats.incAdminInputBytes(int64(tc.RequestRecorder.Size()))
+				globalConnStats.incAdminOutputBytes(int64(tc.ResponseRecorder.Size()))
+				return
+			}
+
+			globalHTTPStats.updateStats(api, tc.ResponseRecorder)
+			globalConnStats.incS3InputBytes(int64(tc.RequestRecorder.Size()))
+			globalConnStats.incS3OutputBytes(int64(tc.ResponseRecorder.Size()))
+
+			resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
+			if err != nil {
+				logger.LogIf(r.Context(), fmt.Errorf("Unable to get the actual resource in the incoming request: %v", err))
+				return
+			}
+
+			bucket, _ := path2BucketObject(resource)
+			if bucket != "" && bucket != minioReservedBucket {
+				globalBucketConnStats.incS3InputBytes(bucket, int64(tc.RequestRecorder.Size()))
+				globalBucketConnStats.incS3OutputBytes(bucket, int64(tc.ResponseRecorder.Size()))
+			}
+		}
 	}
 }
 

cmd/healthcheck-handler.go

@@ -47,7 +47,10 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
 	defer cancel()
 
-	opts := HealthOptions{Maintenance: r.Form.Get("maintenance") == "true"}
+	opts := HealthOptions{
+		Maintenance:    r.Form.Get("maintenance") == "true",
+		DeploymentType: r.Form.Get("deployment-type"),
+	}
 	result := objLayer.Health(ctx, opts)
 	if result.WriteQuorum > 0 {
 		w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum))

cmd/http-stats.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"net/http"
-	"strings"
 	"sync"
 	"sync/atomic"
 
@@ -299,12 +298,7 @@ func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
 }
 
 // Update statistics from http request and response data
-func (st *HTTPStats) updateStats(api string, r *http.Request, w *xhttp.ResponseRecorder) {
-	// Ignore non S3 requests
-	if strings.HasSuffix(r.URL.Path, minioReservedBucketPathWithSlash) {
-		return
-	}
-
+func (st *HTTPStats) updateStats(api string, w *xhttp.ResponseRecorder) {
 	st.totalS3Requests.Inc(api)
 
 	// Increment the prometheus http request response histogram with appropriate label

cmd/http-tracer.go

@@ -39,7 +39,7 @@ var ldapPwdRegex = regexp.MustCompile("(^.*?)LDAPPassword=([^&]*?)(&(.*?))?$")
 // redact LDAP password if part of string
 func redactLDAPPwd(s string) string {
 	parts := ldapPwdRegex.FindStringSubmatch(s)
-	if len(parts) > 0 {
+	if len(parts) > 3 {
 		return parts[1] + "LDAPPassword=*REDACTED*" + parts[3]
 	}
 	return s
@@ -67,41 +67,37 @@ func getOpName(name string) (op string) {
 // otherwise, generate a trace event with request information but no response.
 func httpTracer(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if globalTrace.NumSubscribers(madmin.TraceS3|madmin.TraceInternal) == 0 {
-			h.ServeHTTP(w, r)
-			return
-		}
+		// Setup a http request response recorder - this is needed for
+		// http stats requests and audit if enabled.
+		respRecorder := xhttp.NewResponseRecorder(w)
+
+		// Setup a http request body recorder
+		reqRecorder := &xhttp.RequestRecorder{Reader: r.Body}
+		r.Body = reqRecorder
 
 		// Create tracing data structure and associate it to the request context
 		tc := mcontext.TraceCtxt{
-			AmzReqID: r.Header.Get(xhttp.AmzRequestID),
+			AmzReqID:         r.Header.Get(xhttp.AmzRequestID),
+			RequestRecorder:  reqRecorder,
+			ResponseRecorder: respRecorder,
 		}
 
-		ctx := context.WithValue(r.Context(), mcontext.ContextTraceKey, &tc)
-		r = r.WithContext(ctx)
-
-		// Setup a http request and response body recorder
-		reqRecorder := &xhttp.RequestRecorder{Reader: r.Body}
-		respRecorder := xhttp.NewResponseRecorder(w)
-
-		tc.RequestRecorder = reqRecorder
-		tc.ResponseRecorder = respRecorder
-
-		// Execute call.
-		r.Body = reqRecorder
+		r = r.WithContext(context.WithValue(r.Context(), mcontext.ContextTraceKey, &tc))
 
 		reqStartTime := time.Now().UTC()
 		h.ServeHTTP(respRecorder, r)
 		reqEndTime := time.Now().UTC()
 
+		if globalTrace.NumSubscribers(madmin.TraceS3|madmin.TraceInternal) == 0 {
+			// no subscribers nothing to trace.
+			return
+		}
+
 		tt := madmin.TraceInternal
 		if strings.HasPrefix(tc.FuncName, "s3.") {
 			tt = madmin.TraceS3
 		}
-		// No need to continue if no subscribers for actual type...
-		if globalTrace.NumSubscribers(tt) == 0 {
-			return
-		}
+
 		// Calculate input body size with headers
 		reqHeaders := r.Header.Clone()
 		reqHeaders.Set("Host", r.Host)
@@ -110,7 +106,7 @@ func httpTracer(h http.Handler) http.Handler {
 		} else {
 			reqHeaders.Set("Transfer-Encoding", strings.Join(r.TransferEncoding, ","))
 		}
-		inputBytes := reqRecorder.BodySize()
+		inputBytes := reqRecorder.Size()
 		for k, v := range reqHeaders {
 			inputBytes += len(k) + len(v)
 		}

cmd/iam-etcd-store.go

@@ -248,6 +248,13 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
+			if u.Credentials.IsTemp() {
+				// We should delete such that the client can re-request
+				// for the expiring credentials.
+				deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType))
+				deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false))
+				return nil
+			}
 			return err
 		}
 		u.Credentials.Claims = jwtClaims.Map()

cmd/iam-object-store.go

@@ -187,7 +187,15 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 	if u.Credentials.SessionToken != "" {
 		jwtClaims, err := extractJWTClaims(u)
 		if err != nil {
+			if u.Credentials.IsTemp() {
+				// We should delete such that the client can re-request
+				// for the expiring credentials.
+				iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
+				iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
+				return nil
+			}
 			return err
+
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}
@@ -349,7 +357,9 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[str
 // Assumes cache is locked by caller.
 func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
 	bootstrapTrace("loading all IAM items")
-
+	if iamOS.objAPI == nil {
+		return errServerNotInitialized
+	}
 	listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
 	if err != nil {
 		return err

cmd/iam-store.go

@@ -937,8 +937,7 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 			}
 
 			if g.Status == statusDisabled {
-				// TODO: return an error?
-				return updatedAt, nil, nil
+				return updatedAt, nil, errGroupDisabled
 			}
 		}
 		mp = cache.iamGroupPolicyMap[name]

cmd/local-locker.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/minio/minio/internal/dsync"
@@ -50,7 +51,7 @@ func isWriteLock(lri []lockRequesterInfo) bool {
 
 // localLocker implements Dsync.NetLocker
 type localLocker struct {
-	mutex   chan struct{}
+	mutex   sync.Mutex
 	lockMap map[string][]lockRequesterInfo
 	lockUID map[string]string // UUID -> resource map.
 }
@@ -69,41 +70,13 @@ func (l *localLocker) canTakeLock(resources ...string) bool {
 	return true
 }
 
-// lockMu locks the "mutex" of the local locker.
-// If "ctx" is canceled before the lock can be obtained false is returned.
-// If "true" is returned unlockMu MUST be called.
-// Behavior is similar to sync.Mutex.
-func (l *localLocker) lockMu(ctx context.Context) (ok bool) {
-	select {
-	case l.mutex <- struct{}{}:
-		return true
-	case <-ctx.Done():
-		return false
-	}
-}
-
-// lockMuBlock will block while getting the mutex.
-// When returned unlockMu *must* be called.
-// Behavior is similar to sync.Mutex.
-func (l *localLocker) lockMuBlock() {
-	l.mutex <- struct{}{}
-}
-
-// unlockMu unlocks an acquired mutex.
-// This may only be called once after successfully getting a mutex.
-func (l *localLocker) unlockMu() {
-	<-l.mutex
-}
-
 func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
 	if len(args.Resources) > maxDeleteList {
 		return false, fmt.Errorf("internal error: localLocker.Lock called with more than %d resources", maxDeleteList)
 	}
 
-	if !l.lockMu(ctx) {
-		return false, ctx.Err()
-	}
-	defer l.unlockMu()
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 
 	if !l.canTakeLock(args.Resources...) {
 		// Not all locks can be taken on resources,
@@ -142,9 +115,8 @@ func (l *localLocker) Unlock(_ context.Context, args dsync.LockArgs) (reply bool
 		return false, fmt.Errorf("internal error: localLocker.Unlock called with more than %d resources", maxDeleteList)
 	}
 
-	l.lockMuBlock()
-	defer l.unlockMu()
-
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 	err = nil
 
 	for _, resource := range args.Resources {
@@ -191,11 +163,8 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 		return false, fmt.Errorf("internal error: localLocker.RLock called with more than one resource")
 	}
 
-	if !l.lockMu(ctx) {
-		return false, ctx.Err()
-	}
-	defer l.unlockMu()
-
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 	resource := args.Resources[0]
 	lrInfo := lockRequesterInfo{
 		Name:            resource,
@@ -227,9 +196,8 @@ func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply boo
 		return false, fmt.Errorf("internal error: localLocker.RUnlock called with more than one resource")
 	}
 
-	l.lockMuBlock()
-	defer l.unlockMu()
-
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 	var lri []lockRequesterInfo
 
 	resource := args.Resources[0]
@@ -245,12 +213,41 @@ func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply boo
 	return reply, nil
 }
 
+type lockStats struct {
+	Total  int
+	Writes int
+	Reads  int
+}
+
+func (l *localLocker) stats() lockStats {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+
+	st := lockStats{Total: len(l.lockMap)}
+	for _, v := range l.lockMap {
+		if len(v) == 0 {
+			continue
+		}
+		entry := v[0]
+		if entry.Writer {
+			st.Writes++
+		} else {
+			st.Reads += len(v)
+		}
+	}
+	return st
+}
+
 func (l *localLocker) DupLockMap() map[string][]lockRequesterInfo {
-	l.lockMuBlock()
-	defer l.unlockMu()
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 
 	lockCopy := make(map[string][]lockRequesterInfo, len(l.lockMap))
 	for k, v := range l.lockMap {
+		if len(v) == 0 {
+			delete(l.lockMap, k)
+			continue
+		}
 		lockCopy[k] = append(make([]lockRequesterInfo, 0, len(v)), v...)
 	}
 	return lockCopy
@@ -271,90 +268,97 @@ func (l *localLocker) IsLocal() bool {
 }
 
 func (l *localLocker) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
-	l.lockMuBlock()
-	defer l.unlockMu()
-
-	if len(args.UID) == 0 {
-		for _, resource := range args.Resources {
-			lris, ok := l.lockMap[resource]
-			if !ok {
-				continue
-			}
-			// Collect uids, so we don't mutate while we delete
-			uids := make([]string, 0, len(lris))
-			for _, lri := range lris {
-				uids = append(uids, lri.UID)
-			}
-
-			// Delete collected uids:
-			for _, uid := range uids {
+	select {
+	case <-ctx.Done():
+		return false, ctx.Err()
+	default:
+		l.mutex.Lock()
+		defer l.mutex.Unlock()
+		if len(args.UID) == 0 {
+			for _, resource := range args.Resources {
 				lris, ok := l.lockMap[resource]
 				if !ok {
-					// Just to be safe, delete uuids.
-					for idx := 0; idx < maxDeleteList; idx++ {
-						mapID := formatUUID(uid, idx)
-						if _, ok := l.lockUID[mapID]; !ok {
-							break
-						}
-						delete(l.lockUID, mapID)
-					}
 					continue
 				}
-				l.removeEntry(resource, dsync.LockArgs{UID: uid}, &lris)
-			}
-		}
-		return true, nil
-	}
+				// Collect uids, so we don't mutate while we delete
+				uids := make([]string, 0, len(lris))
+				for _, lri := range lris {
+					uids = append(uids, lri.UID)
+				}
 
-	idx := 0
-	for {
-		mapID := formatUUID(args.UID, idx)
-		resource, ok := l.lockUID[mapID]
-		if !ok {
-			return idx > 0, nil
+				// Delete collected uids:
+				for _, uid := range uids {
+					lris, ok := l.lockMap[resource]
+					if !ok {
+						// Just to be safe, delete uuids.
+						for idx := 0; idx < maxDeleteList; idx++ {
+							mapID := formatUUID(uid, idx)
+							if _, ok := l.lockUID[mapID]; !ok {
+								break
+							}
+							delete(l.lockUID, mapID)
+						}
+						continue
+					}
+					l.removeEntry(resource, dsync.LockArgs{UID: uid}, &lris)
+				}
+			}
+			return true, nil
 		}
-		lris, ok := l.lockMap[resource]
-		if !ok {
-			// Unexpected  inconsistency, delete.
-			delete(l.lockUID, mapID)
+
+		idx := 0
+		for {
+			mapID := formatUUID(args.UID, idx)
+			resource, ok := l.lockUID[mapID]
+			if !ok {
+				return idx > 0, nil
+			}
+			lris, ok := l.lockMap[resource]
+			if !ok {
+				// Unexpected  inconsistency, delete.
+				delete(l.lockUID, mapID)
+				idx++
+				continue
+			}
+			reply = true
+			l.removeEntry(resource, dsync.LockArgs{UID: args.UID}, &lris)
 			idx++
-			continue
 		}
-		reply = true
-		l.removeEntry(resource, dsync.LockArgs{UID: args.UID}, &lris)
-		idx++
 	}
 }
 
 func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refreshed bool, err error) {
-	if !l.lockMu(ctx) {
+	select {
+	case <-ctx.Done():
 		return false, ctx.Err()
-	}
-	defer l.unlockMu()
+	default:
+		l.mutex.Lock()
+		defer l.mutex.Unlock()
 
-	// Check whether uid is still active.
-	resource, ok := l.lockUID[formatUUID(args.UID, 0)]
-	if !ok {
-		return false, nil
-	}
-	idx := 0
-	for {
-		lris, ok := l.lockMap[resource]
+		// Check whether uid is still active.
+		resource, ok := l.lockUID[formatUUID(args.UID, 0)]
 		if !ok {
-			// Inconsistent. Delete UID.
-			delete(l.lockUID, formatUUID(args.UID, idx))
-			return idx > 0, nil
+			return false, nil
 		}
-		for i := range lris {
-			if lris[i].UID == args.UID {
-				lris[i].TimeLastRefresh = UTCNow()
+		idx := 0
+		for {
+			lris, ok := l.lockMap[resource]
+			if !ok {
+				// Inconsistent. Delete UID.
+				delete(l.lockUID, formatUUID(args.UID, idx))
+				return idx > 0, nil
+			}
+			for i := range lris {
+				if lris[i].UID == args.UID {
+					lris[i].TimeLastRefresh = UTCNow()
+				}
+			}
+			idx++
+			resource, ok = l.lockUID[formatUUID(args.UID, idx)]
+			if !ok {
+				// No more resources for UID, but we did update at least one.
+				return true, nil
 			}
-		}
-		idx++
-		resource, ok = l.lockUID[formatUUID(args.UID, idx)]
-		if !ok {
-			// No more resources for UID, but we did update at least one.
-			return true, nil
 		}
 	}
 }
@@ -362,8 +366,8 @@ func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refresh
 // Similar to removeEntry but only removes an entry only if the lock entry exists in map.
 // Caller must hold 'l.mutex' lock.
 func (l *localLocker) expireOldLocks(interval time.Duration) {
-	l.lockMuBlock()
-	defer l.unlockMu()
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 
 	for k, lris := range l.lockMap {
 		modified := false
@@ -394,7 +398,6 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
 
 func newLocker() *localLocker {
 	return &localLocker{
-		mutex:   make(chan struct{}, 1),
 		lockMap: make(map[string][]lockRequesterInfo, 1000),
 		lockUID: make(map[string]string, 1000),
 	}

cmd/local-locker_test.go

@@ -419,7 +419,7 @@ func Test_localLocker_RUnlock(t *testing.T) {
 					}
 					start = time.Now()
 					for _, lock := range toUnLock {
-						ok, err := l.RUnlock(nil, lock)
+						ok, err := l.RUnlock(context.TODO(), lock)
 						if err != nil || !ok {
 							t.Fatal(err)
 						}

cmd/lock-rest-server-common_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"os"
 	"reflect"
+	"sync"
 	"testing"
 
 	"github.com/minio/minio/internal/dsync"
@@ -37,7 +38,10 @@ func createLockTestServer(ctx context.Context, t *testing.T) (string, *lockRESTS
 	}
 
 	locker := &lockRESTServer{
-		ll: newLocker(),
+		ll: &localLocker{
+			mutex:   sync.Mutex{},
+			lockMap: make(map[string][]lockRequesterInfo),
+		},
 	}
 	creds := globalActiveCred
 	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")

cmd/metacache-entries.go

@@ -816,15 +816,16 @@ func (m *metaCacheEntriesSorted) merge(other metaCacheEntriesSorted, limit int)
 	a := m.entries()
 	b := other.entries()
 	for len(a) > 0 && len(b) > 0 {
-		if a[0].name == b[0].name && bytes.Equal(a[0].metadata, b[0].metadata) {
+		switch {
+		case a[0].name == b[0].name && bytes.Equal(a[0].metadata, b[0].metadata):
 			// Same, discard one.
 			merged = append(merged, a[0])
 			a = a[1:]
 			b = b[1:]
-		} else if a[0].name < b[0].name {
+		case a[0].name < b[0].name:
 			merged = append(merged, a[0])
 			a = a[1:]
-		} else {
+		default:
 			merged = append(merged, b[0])
 			b = b[1:]
 		}

cmd/metacache-server-pool.go

@@ -28,7 +28,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -341,7 +340,6 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
 	defer close(out)
 
-	vcfg, _ := globalBucketVersioningSys.Get(o.Bucket)
 	for {
 		var obj metaCacheEntry
 		var ok bool
@@ -354,37 +352,61 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 			}
 		}
 
-		fi, err := obj.fileInfo(o.Bucket)
+		var skip bool
+
+		versioned := o.Versioning != nil && o.Versioning.Versioned(obj.name)
+
+		// skip latest object from listing only for regular
+		// listObjects calls, versioned based listing cannot
+		// filter out between versions 'obj' cannot be truncated
+		// in such a manner, so look for skipping an object only
+		// for regular ListObjects() call only.
+		if !o.Versioned {
+			fi, err := obj.fileInfo(o.Bucket)
+			if err != nil {
+				continue
+			}
+
+			objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
+			if o.Lifecycle != nil {
+				act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo).Action
+				skip = act.Delete()
+				if act.DeleteRestored() {
+					// do not skip DeleteRestored* actions
+					skip = false
+				}
+			}
+		}
+
+		// Skip entry only if needed via ILM, skipping is never true for versioned listing.
+		if !skip {
+			select {
+			case <-ctx.Done():
+				return
+			case out <- obj:
+			}
+		}
+
+		fiv, err := obj.fileInfoVersions(o.Bucket)
 		if err != nil {
 			continue
 		}
 
-		versioned := vcfg != nil && vcfg.Versioned(obj.name)
+		// Expire all versions if needed, if not attempt to queue for replication.
+		for _, version := range fiv.Versions {
+			objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
 
-		objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
-		if o.Lifecycle != nil {
-			evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
-			switch evt.Action {
-			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-				globalExpiryState.enqueueByDays(objInfo, false, evt.Action == lifecycle.DeleteVersionAction)
-				// Skip this entry.
-				continue
-			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-				globalExpiryState.enqueueByDays(objInfo, true, evt.Action == lifecycle.DeleteRestoredVersionAction)
-				// Skip this entry.
-				continue
-			}
-		}
-		select {
-		case <-ctx.Done():
-			return
-		case out <- obj:
-			if fiv, err := obj.fileInfoVersions(o.Bucket); err == nil {
-				for _, version := range fiv.Versions {
-					objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
-					queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
+			if o.Lifecycle != nil {
+				evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
+				if evt.Action.Delete() {
+					globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
+					if !evt.Action.DeleteRestored() {
+						continue
+					} // queue version for replication upon expired restored copies if needed.
 				}
 			}
+
+			queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
 		}
 	}
 }

cmd/metacache-set.go

@@ -34,6 +34,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
@@ -95,6 +96,10 @@ type listPathOptions struct {
 	// Versioned is this a ListObjectVersions call.
 	Versioned bool
 
+	// Versioning config is used for if the path
+	// has versioning enabled.
+	Versioning *versioning.Versioning
+
 	// Lifecycle performs filtering based on lifecycle.
 	// This will filter out objects if the most recent version should be deleted by lifecycle.
 	// Is not transferred across request calls.
@@ -119,12 +124,14 @@ func init() {
 
 func (o *listPathOptions) setBucketMeta(ctx context.Context) {
 	lc, _ := globalLifecycleSys.Get(o.Bucket)
+	vc, _ := globalBucketVersioningSys.Get(o.Bucket)
 
 	// Check if bucket is object locked.
 	rcfg, _ := globalBucketObjectLockSys.Get(o.Bucket)
 	replCfg, _, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, o.Bucket)
 	tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, o.Bucket)
 	o.Lifecycle = lc
+	o.Versioning = vc
 	o.Replication = replicationConfig{
 		Config:  replCfg,
 		remotes: tgts,
@@ -390,6 +397,11 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 	retries := 0
 	rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
 
+	const (
+		retryDelay    = 50 * time.Millisecond
+		retryDelay250 = 250 * time.Millisecond
+	)
+
 	for {
 		if contextCanceled(ctx) {
 			return entries, ctx.Err()
@@ -404,7 +416,6 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			retries = 1
 		}
 
-		const retryDelay = 250 * time.Millisecond
 		// All operations are performed without locks, so we must be careful and allow for failures.
 		// Read metadata associated with the object from a disk.
 		if retries > 0 {
@@ -418,7 +429,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				_, err := disk.ReadVersion(ctx, minioMetaBucket,
 					o.objectPath(0), "", false)
 				if err != nil {
-					time.Sleep(retryDelay)
+					time.Sleep(retryDelay250)
 					retries++
 					continue
 				}
@@ -433,11 +444,19 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			switch toObjectErr(err, minioMetaBucket, o.objectPath(0)).(type) {
 			case ObjectNotFound:
 				retries++
-				time.Sleep(retryDelay)
+				if retries == 1 {
+					time.Sleep(retryDelay)
+				} else {
+					time.Sleep(retryDelay250)
+				}
 				continue
 			case InsufficientReadQuorum:
 				retries++
-				time.Sleep(retryDelay)
+				if retries == 1 {
+					time.Sleep(retryDelay)
+				} else {
+					time.Sleep(retryDelay250)
+				}
 				continue
 			default:
 				return entries, fmt.Errorf("reading first part metadata: %w", err)
@@ -456,7 +475,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				retries = -1
 			}
 			retries++
-			time.Sleep(retryDelay)
+			time.Sleep(retryDelay250)
 			continue
 		case errors.Is(err, io.EOF):
 			return entries, io.EOF
@@ -490,7 +509,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 						_, err := disk.ReadVersion(ctx, minioMetaBucket,
 							o.objectPath(partN), "", false)
 						if err != nil {
-							time.Sleep(retryDelay)
+							time.Sleep(retryDelay250)
 							retries++
 							continue
 						}
@@ -501,7 +520,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				// Load partN metadata...
 				fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{}, true)
 				if err != nil {
-					time.Sleep(retryDelay)
+					time.Sleep(retryDelay250)
 					retries++
 					continue
 				}
@@ -523,9 +542,9 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			}()
 
 			tmp := newMetacacheReader(pr)
-			defer tmp.Close()
 			e, err := tmp.filter(o)
 			pr.CloseWithError(err)
+			tmp.Close()
 			entries.o = append(entries.o, e.o...)
 			if o.Limit > 0 && entries.len() > o.Limit {
 				entries.truncate(o.Limit)
@@ -539,11 +558,11 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				switch toObjectErr(err, minioMetaBucket, o.objectPath(partN)).(type) {
 				case ObjectNotFound:
 					retries++
-					time.Sleep(retryDelay)
+					time.Sleep(retryDelay250)
 					continue
 				case InsufficientReadQuorum:
 					retries++
-					time.Sleep(retryDelay)
+					time.Sleep(retryDelay250)
 					continue
 				default:
 					logger.LogIf(ctx, err)
@@ -870,8 +889,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 	var fdMu sync.Mutex
 	fds := opts.fallbackDisks
 	fallback := func(err error) StorageAPI {
-		switch err.(type) {
-		case StorageErr:
+		if _, ok := err.(StorageErr); ok {
 			// Attempt to grab a fallback disk
 			fdMu.Lock()
 			defer fdMu.Unlock()

cmd/metacache-stream.go

@@ -562,8 +562,7 @@ func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry
 	}
 	for {
 		if more, err := r.mr.ReadBool(); !more {
-			switch err {
-			case io.EOF:
+			if err == io.EOF {
 				err = io.ErrUnexpectedEOF
 			}
 			r.err = err

cmd/metacache-walk.go

@@ -323,13 +323,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				}
 			case isSysErrNotDir(err):
 				// skip
-			default:
-				// It is totally possible that xl.meta was overwritten
-				// while being concurrently listed at the same time in
-				// such scenarios the 'xl.meta' might get truncated
-				if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) {
-					logger.LogIf(ctx, err)
-				}
 			}
 		}
 

cmd/metrics-v2.go

@@ -77,7 +77,7 @@ func init() {
 		return allMetrics
 	}()
 
-	nodeCollector = newMinioCollectorNode([]*MetricsGroup{
+	nodeGroups := []*MetricsGroup{
 		getNodeHealthMetrics(),
 		getLocalDriveStorageMetrics(),
 		getCacheMetrics(),
@@ -86,7 +86,10 @@ func init() {
 		getMinioVersionMetrics(),
 		getS3TTFBMetric(),
 		getNotificationMetrics(),
-	})
+		getDistLockMetrics(),
+	}
+
+	nodeCollector = newMinioCollectorNode(nodeGroups)
 	clusterCollector = newMinioClusterCollector(allMetricsGroups)
 }
 
@@ -131,6 +134,7 @@ const (
 	iamSubsystem              MetricSubsystem = "iam"
 	kmsSubsystem              MetricSubsystem = "kms"
 	notifySubsystem           MetricSubsystem = "notify"
+	lambdaSubsystem           MetricSubsystem = "lambda"
 	auditSubsystem            MetricSubsystem = "audit"
 )
 
@@ -151,6 +155,7 @@ const (
 	waitingTotal   MetricName = "waiting_total"
 	incomingTotal  MetricName = "incoming_total"
 	objectTotal    MetricName = "object_total"
+	versionTotal   MetricName = "version_total"
 	offlineTotal   MetricName = "offline_total"
 	onlineTotal    MetricName = "online_total"
 	openTotal      MetricName = "open_total"
@@ -182,8 +187,9 @@ const (
 	usageInfo   MetricName = "usage_info"
 	versionInfo MetricName = "version_info"
 
-	sizeDistribution = "size_distribution"
-	ttfbDistribution = "ttfb_seconds_distribution"
+	sizeDistribution    = "size_distribution"
+	versionDistribution = "version_distribution"
+	ttfbDistribution    = "ttfb_seconds_distribution"
 
 	lastActivityTime = "last_activity_nano_seconds"
 	startTime        = "starttime_seconds"
@@ -504,6 +510,16 @@ func getBucketUsageObjectsTotalMD() MetricDescription {
 	}
 }
 
+func getBucketUsageVersionsTotalMD() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: usageSubsystem,
+		Name:      versionTotal,
+		Help:      "Total number of versions",
+		Type:      gaugeMetric,
+	}
+}
+
 func getBucketRepLatencyMD() MetricDescription {
 	return MetricDescription{
 		Namespace: bucketMetricNamespace,
@@ -564,6 +580,16 @@ func getBucketObjectDistributionMD() MetricDescription {
 	}
 }
 
+func getBucketObjectVersionsMD() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: objectsSubsystem,
+		Name:      versionDistribution,
+		Help:      "Distribution of object sizes in the bucket, includes label for the bucket name",
+		Type:      histogramMetric,
+	}
+}
+
 func getInternodeFailedRequests() MetricDescription {
 	return MetricDescription{
 		Namespace: interNodeMetricNamespace,
@@ -1651,13 +1677,60 @@ func getCacheMetrics() *MetricsGroup {
 	return mg
 }
 
+func getDistLockMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: 1 * time.Second,
+	}
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		if !globalIsDistErasure {
+			return []Metric{}
+		}
+
+		st := globalLockServer.stats()
+
+		metrics := make([]Metric, 0, 3)
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: minioNamespace,
+				Subsystem: "locks",
+				Name:      "total",
+				Help:      "Number of current locks on this peer",
+				Type:      gaugeMetric,
+			},
+			Value: float64(st.Total),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: minioNamespace,
+				Subsystem: "locks",
+				Name:      "write_total",
+				Help:      "Number of current WRITE locks on this peer",
+				Type:      gaugeMetric,
+			},
+			Value: float64(st.Writes),
+		})
+		metrics = append(metrics, Metric{
+			Description: MetricDescription{
+				Namespace: minioNamespace,
+				Subsystem: "locks",
+				Name:      "read_total",
+				Help:      "Number of current READ locks on this peer",
+				Type:      gaugeMetric,
+			},
+			Value: float64(st.Reads),
+		})
+		return metrics
+	})
+	return mg
+}
+
 func getNotificationMetrics() *MetricsGroup {
 	mg := &MetricsGroup{
 		cacheInterval: 10 * time.Second,
 	}
 	mg.RegisterRead(func(ctx context.Context) []Metric {
-		stats := globalConfigTargetList.Stats()
-		metrics := make([]Metric, 0, 1+len(stats.TargetStats))
+		nstats := globalNotifyTargetList.Stats()
+		metrics := make([]Metric, 0, 1+len(nstats.TargetStats))
 		metrics = append(metrics, Metric{
 			Description: MetricDescription{
 				Namespace: minioNamespace,
@@ -1666,9 +1739,9 @@ func getNotificationMetrics() *MetricsGroup {
 				Help:      "Number of concurrent async Send calls active to all targets",
 				Type:      gaugeMetric,
 			},
-			Value: float64(stats.CurrentSendCalls),
+			Value: float64(nstats.CurrentSendCalls),
 		})
-		for _, st := range stats.TargetStats {
+		for _, st := range nstats.TargetStats {
 			metrics = append(metrics, Metric{
 				Description: MetricDescription{
 					Namespace: minioNamespace,
@@ -1681,6 +1754,43 @@ func getNotificationMetrics() *MetricsGroup {
 				Value:          float64(st.CurrentQueue),
 			})
 		}
+
+		lstats := globalLambdaTargetList.Stats()
+		for _, st := range lstats.TargetStats {
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: lambdaSubsystem,
+					Name:      "active_requests",
+					Help:      "Number of in progress requests",
+				},
+				VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
+				Value:          float64(st.ActiveRequests),
+			})
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: lambdaSubsystem,
+					Name:      "total_requests",
+					Help:      "Total number of requests sent since start",
+					Type:      counterMetric,
+				},
+				VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
+				Value:          float64(st.TotalRequests),
+			})
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: lambdaSubsystem,
+					Name:      "failed_requests",
+					Help:      "Total number of requests that failed to send since start",
+					Type:      counterMetric,
+				},
+				VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
+				Value:          float64(st.FailedRequests),
+			})
+		}
+
 		// Audit and system:
 		audit := logger.CurrentStats()
 		for id, st := range audit {
@@ -1896,6 +2006,12 @@ func getBucketUsageMetrics() *MetricsGroup {
 				VariableLabels: map[string]string{"bucket": bucket},
 			})
 
+			metrics = append(metrics, Metric{
+				Description:    getBucketUsageVersionsTotalMD(),
+				Value:          float64(usage.VersionsCount),
+				VariableLabels: map[string]string{"bucket": bucket},
+			})
+
 			metrics = append(metrics, Metric{
 				Description:    getBucketRepReceivedBytesMD(),
 				Value:          float64(stats.ReplicaSize),
@@ -1961,6 +2077,12 @@ func getBucketUsageMetrics() *MetricsGroup {
 				HistogramBucketLabel: "range",
 				VariableLabels:       map[string]string{"bucket": bucket},
 			})
+			metrics = append(metrics, Metric{
+				Description:          getBucketObjectVersionsMD(),
+				Histogram:            usage.ObjectVersionsHistogram,
+				HistogramBucketLabel: "range",
+				VariableLabels:       map[string]string{"bucket": bucket},
+			})
 		}
 		return
 	})
@@ -2322,8 +2444,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 				continue
 			}
 			metricType := prometheus.GaugeValue
-			switch metric.Description.Type {
-			case counterMetric:
+			if metric.Description.Type == counterMetric {
 				metricType = prometheus.CounterValue
 			}
 			toPost := prometheus.MustNewConstMetric(
@@ -2431,8 +2552,7 @@ func (c *minioNodeCollector) Collect(ch chan<- prometheus.Metric) {
 		}
 
 		metricType := prometheus.GaugeValue
-		switch metric.Description.Type {
-		case counterMetric:
+		if metric.Description.Type == counterMetric {
 			metricType = prometheus.CounterValue
 		}
 		ch <- prometheus.MustNewConstMetric(

cmd/metrics.go

@@ -441,6 +441,18 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
 				k,
 			)
 		}
+		for k, v := range usageInfo.ObjectVersionsHistogram {
+			ch <- prometheus.MustNewConstMetric(
+				prometheus.NewDesc(
+					prometheus.BuildFQName(bucketNamespace, "objects", "histogram"),
+					"Total number of versions of objects in a bucket",
+					[]string{"bucket", "object_versions"}, nil),
+				prometheus.GaugeValue,
+				float64(v),
+				bucket,
+				k,
+			)
+		}
 	}
 }