fix: return HTTP 503 when multipart scan capacity is exhausted

Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/api-errors.go

@@ -452,6 +452,7 @@ const (
 	ErrIAMNotInitialized
 	ErrMultipartListingLegacy
 	ErrMultipartListingIdentity
+	ErrSlowDown
 
 	apiErrCodeEnd // This is used only for the testing code
 )
@@ -1348,6 +1349,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Multipart upload metadata is inconsistent. Run the multipart preflight check to locate the affected storage set.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrSlowDown: {
+		Code:           "SlowDown",
+		Description:    "Please reduce your request rate",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
 	ErrBucketMetadataNotInitialized: {
 		Code:           "XMinioBucketMetadataNotInitialized",
 		Description:    "Bucket metadata not initialized yet, please try again.",
@@ -2319,6 +2325,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	}
 
 	switch err.(type) {
+	case SlowDown:
+		apiErr = ErrSlowDown
 	case StorageFull:
 		apiErr = ErrStorageFull
 	case hash.BadDigest:

cmd/erasure-multipart-listing_test.go

@@ -333,6 +333,18 @@ func TestMultipartPreflightAdminHTTP(t *testing.T) {
 	if err = json.Unmarshal(rec.Body.Bytes(), &report); err != nil || !report.Ready || !report.Complete {
 		t.Fatalf("preflight: %+v %v", report, err)
 	}
+	for range cap(multipartScanSlots) {
+		scan, err := startMultipartScan(t.Context(), false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer scan.close()
+	}
+	rec = httptest.NewRecorder()
+	bed.router.ServeHTTP(rec, req)
+	if rec.Code != http.StatusServiceUnavailable {
+		t.Fatalf("busy admin preflight: %d %s", rec.Code, rec.Body.String())
+	}
 }
 
 type multipartLateCreateDisk struct {
@@ -641,6 +653,9 @@ func TestMultipartListingBudgetAndAdmission(t *testing.T) {
 	if !errors.As(err, &limited) {
 		t.Fatalf("budget: %v", err)
 	}
+	if apiErr := toAPIError(t.Context(), err); apiErr.HTTPStatusCode != http.StatusServiceUnavailable || apiErr.Code != "SlowDown" {
+		t.Fatalf("budget error mapping: %+v", apiErr)
+	}
 	second, err := startMultipartScan(t.Context(), false)
 	if err != nil {
 		t.Fatal(err)
@@ -652,6 +667,36 @@ func TestMultipartListingBudgetAndAdmission(t *testing.T) {
 	}
 }
 
+func TestMultipartListingAdmissionHTTP(t *testing.T) {
+	z, _, _ := multipartListingFixture(t)
+	bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for range cap(multipartScanSlots) {
+		scan, err := startMultipartScan(t.Context(), false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer scan.close()
+	}
+	req, err := newTestSignedRequestV4(http.MethodGet,
+		getListMultipartUploadsURLWithParams("", bucket, "", "", "", "", "1"),
+		0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	rec := httptest.NewRecorder()
+	router.ServeHTTP(rec, req)
+	var response APIErrorResponse
+	if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
+		t.Fatal(err)
+	}
+	if rec.Code != http.StatusServiceUnavailable || response.Code != "SlowDown" {
+		t.Fatalf("admission returned %d %s", rec.Code, rec.Body.String())
+	}
+}
+
 func TestMultipartListingPreflightMinorityLegacy(t *testing.T) {
 	z, set, bucket := multipartListingFixture(t)
 	mp, err := z.NewMultipartUpload(t.Context(), bucket, "old", ObjectOptions{})