fix(pools): lock single-pool object writes in the object's own erasure set

In a single pool, DELETE in every form, bulk delete, the CopyObject
destination, SelectObjectContent, batch key rotation and the upload-ID
guards of complete, abort, get-info and list-parts locked through the
first erasure set, while PUT, multipart commit, PutObjectPart, tag,
retention, legal-hold and metadata updates, restore, transition and heal
lock in the set the object hashes to. When the sets of a pool use
different hosts, as every automatic layout of 18 or more hosts does, the
two locks ask different lockers and do not exclude each other, and
concurrent read-modify-writes of xl.meta can lose acknowledged versions
and delete markers or leave an object on a minority of drives.

Route these locks through NewObjectNSLock, which returns the lock of
the object's own set in a single pool and the pools lock otherwise.
Upload guards choose the set by their object and keep the name
object/uploadID. True prefix deletion and the locks of names that are
not objects keep their domain.

Bulk delete groups its names by set and takes one set's lock at a time,
in ascending set index; a failed group fails alone. A single node, whose
sets share one local lock table, keeps its one bulk lock.

The COPY handler takes its source and destination locks before it opens
the source, sorted, and transactionally: bounded waits while holding a
lock, release in reverse order and retry with new lock instances, one
deadline. It reads the source under them, passes NoLock to every nested
call, and runs that work in the locks' context. This also removes a
deadlock of copies in opposite directions wherever both locks were in
one domain, such as a single node or several pools.

Lock names and RPCs are unchanged, but a pool is protected only when
every node runs this change: affected pools must be upgraded and rolled
back with a drained restart.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/batch-rotate.go

@@ -180,7 +180,7 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
 	versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
 
-	lock := api.NewNSLock(r.Bucket, objInfo.Name)
+	lock := api.NewObjectNSLock(r.Bucket, objInfo.Name)
 	lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return err

cmd/erasure-server-pool-locks.go

@@ -0,0 +1,322 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"math/rand/v2"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/set"
+)
+
+// Object lock domains.
+//
+// Every erasure set has its own lockers, one per host among its drives, and a
+// distributed lock is granted by a majority of them. Two locks of one name
+// exclude each other only when they ask the same lockers. In a single pool the
+// writers (PutObject, multipart commit, PutObjectPart, tags, retention, legal
+// hold, metadata, restore, transition, heal) lock in the set an object hashes
+// to. The pools layer locks an object, and its upload IDs, in the same set, so
+// that a pool whose sets use different hosts still has one lock per object.
+// With several pools every object write takes the pools lock instead, which
+// these helpers then return unchanged.
+//
+// Generic NewNSLock keeps one stable domain for names that are not objects:
+// bucket and IAM metadata locks, format and leader coordination, and the
+// /[replicate]/ serializer.
+
+// NewObjectNSLock returns the lock of one object in the lock domain that its
+// writes use: in a single pool the object's own erasure set, otherwise the
+// pools lock. Directory objects are named as storage names them.
+func (z *erasureServerPools) NewObjectNSLock(bucket, object string) RWLocker {
+	if z.SinglePool() {
+		object = encodeDirObject(object)
+	}
+	return z.objectNSLock(bucket, object, object)
+}
+
+// uploadIDNSLock returns the guard of one upload ID. It is chosen by the
+// upload's object, as PutObjectPart locks it in that object's set; the name
+// locked stays object/uploadID.
+func (z *erasureServerPools) uploadIDNSLock(bucket, object, uploadID string) RWLocker {
+	return z.objectNSLock(bucket, object, pathJoin(object, uploadID))
+}
+
+// objectNSLock returns the lock of resource, a name owned by object, in the
+// lock domain of object.
+func (z *erasureServerPools) objectNSLock(bucket, object, resource string) RWLocker {
+	if z.SinglePool() {
+		return z.serverPools[0].getHashedSet(encodeDirObject(object)).NewNSLock(bucket, resource)
+	}
+	return z.NewNSLock(bucket, resource)
+}
+
+// objectLockRequest is one object lock that lockObjects takes.
+type objectLockRequest struct {
+	bucket, object string
+	write          bool
+}
+
+// lockObjects takes the locks of several objects, such as the source and
+// destination of a copy, before any of them is read. A name that appears twice
+// gets one lock, a write lock if either asks for one. The locks are taken in
+// one order, by lock domain and then by bucket and object, and transactionally
+// (see getNSLocks), so that two copies in opposite directions, or an operation
+// that orders its locks differently, cannot hold each other forever. The
+// returned context ends when any of the locks is lost; the returned function
+// releases them all.
+func (z *erasureServerPools) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) {
+	type entry struct {
+		objectLockRequest
+		domain int
+	}
+	locks := make([]entry, 0, len(reqs))
+	for _, r := range reqs {
+		r.object = encodeDirObject(r.object)
+		i := slices.IndexFunc(locks, func(e entry) bool { return e.bucket == r.bucket && e.object == r.object })
+		if i >= 0 {
+			locks[i].write = locks[i].write || r.write
+			continue
+		}
+		domain := 0
+		if z.SinglePool() {
+			domain = z.serverPools[0].getHashedSetIndex(r.object)
+		}
+		locks = append(locks, entry{objectLockRequest: r, domain: domain})
+	}
+	slices.SortFunc(locks, func(a, b entry) int {
+		if a.domain != b.domain {
+			return a.domain - b.domain
+		}
+		return strings.Compare(pathJoin(a.bucket, a.object), pathJoin(b.bucket, b.object))
+	})
+	nsLocks := make([]nsLockRequest, len(locks))
+	for i, l := range locks {
+		nsLocks[i] = nsLockRequest{
+			newLock: func() RWLocker { return z.NewObjectNSLock(l.bucket, l.object) },
+			write:   l.write,
+			atomic:  true,
+		}
+	}
+	return getNSLocks(ctx, nsLocks, globalOperationTimeout)
+}
+
+// nsLockRequest is one lock of a transactional acquisition.
+type nsLockRequest struct {
+	// newLock returns a new instance of the lock for each attempt. An
+	// instance names its holder, and a distributed unlock retries in the
+	// background; a later attempt with the same instance could be released
+	// by such a retry of the attempt before.
+	newLock func() RWLocker
+	write   bool
+	// atomic: the lock is granted as a whole or not at all. A distributed
+	// lock is, also over several names. The local backend takes several
+	// names one at a time, holding the first while it waits for the next;
+	// its lock context is the context it was given.
+	atomic bool
+}
+
+const (
+	// nsLockTryTimeout bounds each wait for a lock while holding another.
+	nsLockTryTimeout = 2 * time.Second
+	// nsLockRetryWait is the longest random pause before a new attempt.
+	nsLockRetryWait = 250 * time.Millisecond
+)
+
+// getNSLocks acquires every lock of locks, in their order, or none. While it
+// holds a lock it waits for the next one only up to nsLockTryTimeout; then it
+// releases what it holds in reverse order, pauses, and tries again with a
+// fresh context, until the timeout of all attempts. A wait that holds nothing
+// can last for the rest of the timeout. Each lock is taken in the context of
+// the one before, so the returned context ends when any lease is lost.
+func getNSLocks(ctx context.Context, locks []nsLockRequest, timeout *dynamicTimeout) (LockContext, func(), error) {
+	type held struct {
+		lock  RWLocker
+		write bool
+		lkctx LockContext
+	}
+	release := func(hs []held) {
+		for i := len(hs) - 1; i >= 0; i-- {
+			if hs[i].write {
+				hs[i].lock.Unlock(hs[i].lkctx)
+			} else {
+				hs[i].lock.RUnlock(hs[i].lkctx)
+			}
+		}
+	}
+	// Fail as a single lock does: a canceled request with its own error,
+	// anything else as a timeout.
+	failed := func() error {
+		if err := ctx.Err(); errors.Is(err, context.Canceled) {
+			return err
+		}
+		return OperationTimedOut{}
+	}
+	start := time.Now()
+	deadline := start.Add(timeout.Timeout())
+	for {
+		hs := make([]held, 0, len(locks))
+		lctx := ctx
+		var err error
+		for _, l := range locks {
+			wait := time.Until(deadline)
+			if len(hs) > 0 || !l.atomic {
+				wait = min(wait, nsLockTryTimeout)
+			}
+			if wait <= 0 {
+				err = OperationTimedOut{}
+				break
+			}
+			lk := l.newLock()
+			try := newDynamicTimeout(wait, wait)
+			// The local backend waits up to the timeout for each name: bound
+			// the whole attempt by its context instead.
+			tctx, cancel := lctx, context.CancelFunc(func() {})
+			if !l.atomic {
+				tctx, cancel = context.WithTimeout(lctx, wait)
+			}
+			var lkctx LockContext
+			if l.write {
+				lkctx, err = lk.GetLock(tctx, try)
+			} else {
+				lkctx, err = lk.GetRLock(tctx, try)
+			}
+			cancel()
+			if err != nil {
+				break
+			}
+			if !l.atomic {
+				lkctx.ctx = lctx
+			}
+			hs = append(hs, held{lock: lk, write: l.write, lkctx: lkctx})
+			lctx = lkctx.Context()
+		}
+		if err == nil {
+			timeout.LogSuccess(time.Since(start))
+			if len(hs) == 0 {
+				return LockContext{ctx: ctx, cancel: func() {}}, func() {}, nil
+			}
+			return LockContext{ctx: lctx, cancel: func() {}}, func() { release(hs) }, nil
+		}
+		release(hs)
+		// A lost lease cancels the context of a later wait; only the
+		// request's own context ends the attempts.
+		if ctx.Err() != nil || (!errors.As(err, &OperationTimedOut{}) && !errors.Is(err, context.Canceled)) {
+			return LockContext{}, nil, err
+		}
+		pause := rand.N(nsLockRetryWait) + time.Millisecond
+		if time.Until(deadline) <= pause {
+			timeout.LogFailure()
+			return LockContext{}, nil, OperationTimedOut{}
+		}
+		t := time.NewTimer(pause)
+		select {
+		case <-ctx.Done():
+			t.Stop()
+			return LockContext{}, nil, failed()
+		case <-t.C:
+		}
+	}
+}
+
+// sharesLocalNSLock reports whether every set of the pool uses one local lock
+// table, as a single node does: there all sets are one lock domain.
+func (s *erasureSets) sharesLocalNSLock() bool {
+	first := s.sets[0].nsMutex
+	if first.isDistErasure {
+		return false
+	}
+	for _, set := range s.sets[1:] {
+		if set.nsMutex != first {
+			return false
+		}
+	}
+	return true
+}
+
+// deleteObjectsInSets is the bulk delete of a single pool whose sets do not
+// share one lock table. The names are grouped by the set they hash to, the
+// lock domain of their writers. The groups run one after another in ascending
+// set index; each takes its set's lock over all of its names in one
+// acquisition, deletes its sub-batch, and releases the lock before the next
+// group starts. No two sets' locks are held at once.
+//
+// Each group's lock is taken in the request's context. A group whose lock
+// cannot be taken fails as a whole and the next group still runs; once the
+// request is canceled the remaining groups fail without being attempted. The
+// results keep their positions in objects. Replication, events and auditing
+// stay with the callers, on the merged results.
+func (z *erasureServerPools) deleteObjectsInSets(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions, dobjects []DeletedObject, derrs []error, timeout *dynamicTimeout) {
+	pool := z.serverPools[0]
+	groups := make(map[int][]int)
+	for i, obj := range objects {
+		idx := pool.getHashedSetIndex(obj.ObjectName)
+		groups[idx] = append(groups[idx], i)
+	}
+	order := make([]int, 0, len(groups))
+	for idx := range groups {
+		order = append(order, idx)
+	}
+	slices.Sort(order)
+
+	for _, idx := range order {
+		members := groups[idx]
+		fail := func(err error) {
+			for _, i := range members {
+				derrs[i] = err
+			}
+		}
+		if err := ctx.Err(); err != nil {
+			fail(err)
+			continue
+		}
+		er := pool.sets[idx]
+		names := set.NewStringSet()
+		batch := make([]ObjectToDelete, len(members))
+		for j, i := range members {
+			names.Add(objects[i].ObjectName)
+			batch[j] = objects[i]
+		}
+		resources := names.ToSlice()
+		lkctx, unlock, err := getNSLocks(ctx, []nsLockRequest{{
+			newLock: func() RWLocker { return er.NewNSLock(bucket, resources...) },
+			write:   true,
+			atomic:  er.nsMutex.isDistErasure || len(resources) == 1,
+		}}, timeout)
+		if err != nil {
+			fail(err)
+			continue
+		}
+		deleted, errs := er.DeleteObjects(lkctx.Context(), bucket, batch, opts)
+		unlock()
+		// Merge as the pools loop of DeleteObjects does for one pool.
+		for j, i := range members {
+			if errs[j] == nil && deleted[j].found {
+				dobjects[i], derrs[i] = deleted[j], nil
+				continue
+			}
+			if derrs[i] == nil {
+				dobjects[i], derrs[i] = deleted[j], errs[j]
+			}
+		}
+	}
+}

cmd/erasure-server-pool-locks_test.go

@@ -0,0 +1,555 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/dsync"
+)
+
+// ldTry reports whether lk can be taken now, and releases it again.
+func ldTry(t *testing.T, lk RWLocker, write bool) bool {
+	t.Helper()
+	dt := newDynamicTimeout(100*time.Millisecond, 100*time.Millisecond)
+	if write {
+		lc, err := lk.GetLock(t.Context(), dt)
+		if err == nil {
+			lk.Unlock(lc)
+		}
+		return err == nil
+	}
+	lc, err := lk.GetRLock(t.Context(), dt)
+	if err == nil {
+		lk.RUnlock(lc)
+	}
+	return err == nil
+}
+
+// ldFree reports whether lk can be taken within a few seconds: a distributed
+// unlock completes in the background.
+func ldFree(t *testing.T, lk RWLocker, write bool) bool {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for !ldTry(t, lk, write) {
+		if time.Now().After(deadline) {
+			return false
+		}
+	}
+	return true
+}
+
+// ldTaken waits until someone else holds lk.
+func ldTaken(t *testing.T, lk RWLocker) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for ldTry(t, lk, false) {
+		if time.Now().After(deadline) {
+			t.Fatal("lock never taken")
+		}
+		time.Sleep(time.Millisecond)
+	}
+}
+
+func ldHoldLock(t *testing.T, lk RWLocker) func() {
+	t.Helper()
+	lc, err := lk.GetLock(t.Context(), globalOperationTimeout)
+	if err != nil {
+		t.Fatal(err)
+	}
+	done := false
+	return func() {
+		if !done {
+			done = true
+			lk.Unlock(lc)
+		}
+	}
+}
+
+// Object locks, and upload guards by their object, are in the set the
+// object's writes lock in; with several pools they stay the pools lock.
+func TestObjectNSLockRouting(t *testing.T) {
+	for _, mode := range ldModes {
+		t.Run(mode, func(t *testing.T) {
+			z, bucket := ldPool(t, mode)
+			pool := z.serverPools[0]
+			key := ldKey(t, z, "route", 1)
+			dir := ldKey(t, z, "route-dir", 0) + "/"
+			for _, name := range []string{key, dir} {
+				release := ldHoldLock(t, z.NewObjectNSLock(bucket, name))
+				encoded := encodeDirObject(name)
+				if ldTry(t, pool.getHashedSet(encoded).NewNSLock(bucket, encoded), false) {
+					t.Errorf("%s: a writer of the object's set ignores the object lock", name)
+				}
+				release()
+			}
+			id := mustGetUUID()
+			release := ldHoldLock(t, z.uploadIDNSLock(bucket, key, id))
+			if ldTry(t, pool.getHashedSet(key).NewNSLock(bucket, pathJoin(key, id)), false) {
+				t.Error("PutObjectPart's guard ignores the upload guard")
+			}
+			release()
+			if mode != ldShared && pool.sharesLocalNSLock() {
+				t.Error("split sets reported as one local table")
+			}
+			if mode == ldShared && !pool.sharesLocalNSLock() {
+				t.Error("one local table not recognized")
+			}
+		})
+	}
+	t.Run("pools", func(t *testing.T) {
+		z, bucket := bulkDeletePools(t, 2)
+		release := ldHoldLock(t, z.NewNSLock(bucket, "pooled"))
+		defer release()
+		if ldTry(t, z.NewObjectNSLock(bucket, "pooled"), false) || ldTry(t, z.uploadIDNSLock(bucket, "pooled", ""), false) {
+			t.Error("multi-pool object locks left the pools lock")
+		}
+	})
+}
+
+// lockObjects merges a name asked twice into one write lock and takes source
+// and destination in their own sets.
+func TestLockObjects(t *testing.T) {
+	for _, mode := range ldModes {
+		t.Run(mode, func(t *testing.T) {
+			z, bucket := ldPool(t, mode)
+			pool := z.serverPools[0]
+			src, dst := ldKey(t, z, "lock-src", 1), ldKey(t, z, "lock-dst", 0)
+			lkctx, unlock, err := z.lockObjects(t.Context(), objectLockRequest{bucket: bucket, object: src}, objectLockRequest{bucket: bucket, object: dst, write: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if lkctx.Context().Err() != nil {
+				t.Fatal("lock context ended")
+			}
+			if !ldTry(t, pool.getHashedSet(src).NewNSLock(bucket, src), false) {
+				t.Error("source not shared with readers")
+			}
+			if ldTry(t, pool.getHashedSet(src).NewNSLock(bucket, src), true) {
+				t.Error("source not locked against writers")
+			}
+			if ldTry(t, pool.getHashedSet(dst).NewNSLock(bucket, dst), false) {
+				t.Error("destination not locked")
+			}
+			unlock()
+			if !ldFree(t, pool.getHashedSet(dst).NewNSLock(bucket, dst), true) || !ldFree(t, pool.getHashedSet(src).NewNSLock(bucket, src), true) {
+				t.Error("locks left behind")
+			}
+
+			// A copy onto itself takes one write lock; a directory object is
+			// locked by its stored name.
+			self := ldKey(t, z, "lock-self", 1) + "/"
+			_, unlock, err = z.lockObjects(t.Context(), objectLockRequest{bucket: bucket, object: self}, objectLockRequest{bucket: bucket, object: self, write: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			encoded := encodeDirObject(self)
+			if ldTry(t, pool.getHashedSet(encoded).NewNSLock(bucket, encoded), false) {
+				t.Error("self copy not write locked")
+			}
+			unlock()
+		})
+	}
+}
+
+// ldLeaseLock is a lock whose lease the test ends.
+type ldLeaseLock struct {
+	RWLocker
+	lost chan context.CancelFunc
+}
+
+func (l ldLeaseLock) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
+	lc, err := l.RWLocker.GetLock(ctx, timeout)
+	if err != nil {
+		return lc, err
+	}
+	lease, cancel := context.WithCancel(lc.Context())
+	select {
+	case l.lost <- cancel:
+	default:
+		panic("too many lock attempts")
+	}
+	return LockContext{ctx: lease, cancel: cancel}, nil
+}
+
+// getNSLocks never holds one lock while it waits for another for long: it
+// releases what it holds and retries, also after a lost lease, until its
+// timeout or the request ends, and leaves nothing behind.
+func TestGetNSLocksTransactional(t *testing.T) {
+	ns := newNSLock(false)
+	lock := func(name string) RWLocker { return ns.NewNSLock(nil, "bucket", name) }
+	newLock := func(name string) func() RWLocker { return func() RWLocker { return lock(name) } }
+	reqs := func() []nsLockRequest {
+		return []nsLockRequest{{newLock: newLock("a"), write: true, atomic: true}, {newLock: newLock("b"), write: true, atomic: true}}
+	}
+
+	t.Run("partial", func(t *testing.T) {
+		releaseB := ldHoldLock(t, lock("b"))
+		defer releaseB()
+		type result struct {
+			unlock func()
+			err    error
+		}
+		done := make(chan result, 1)
+		go func() {
+			_, unlock, err := getNSLocks(t.Context(), reqs(), globalOperationTimeout)
+			done <- result{unlock, err}
+		}()
+		// "a" is taken while "b" is waited for, and free again between
+		// attempts.
+		ldTaken(t, lock("a"))
+		deadline := time.Now().Add(3 * nsLockTryTimeout)
+		for !ldTry(t, lock("a"), true) {
+			if time.Now().After(deadline) {
+				t.Fatal("a stays held while b is waited for")
+			}
+		}
+		releaseB()
+		res := <-done
+		if res.err != nil {
+			t.Fatal(res.err)
+		}
+		if ldTry(t, lock("a"), false) || ldTry(t, lock("b"), false) {
+			t.Error("not both locks held")
+		}
+		res.unlock()
+		if !ldTry(t, lock("a"), true) || !ldTry(t, lock("b"), true) {
+			t.Error("locks left behind")
+		}
+	})
+
+	t.Run("timeout", func(t *testing.T) {
+		releaseB := ldHoldLock(t, lock("b"))
+		defer releaseB()
+		start := time.Now()
+		_, _, err := getNSLocks(t.Context(), reqs(), newDynamicTimeout(3*time.Second, 3*time.Second))
+		if !errors.As(err, &OperationTimedOut{}) {
+			t.Fatalf("got %v, want a timeout", err)
+		}
+		if took := time.Since(start); took > 6*time.Second {
+			t.Errorf("timed out after %v", took)
+		}
+		if !ldTry(t, lock("a"), true) {
+			t.Error("a left behind")
+		}
+	})
+
+	t.Run("cancel", func(t *testing.T) {
+		releaseB := ldHoldLock(t, lock("b"))
+		defer releaseB()
+		ctx, cancel := context.WithCancel(t.Context())
+		time.AfterFunc(300*time.Millisecond, cancel)
+		_, _, err := getNSLocks(ctx, reqs(), globalOperationTimeout)
+		if !errors.Is(err, context.Canceled) {
+			t.Fatalf("got %v, want canceled", err)
+		}
+		if !ldTry(t, lock("a"), true) {
+			t.Error("a left behind")
+		}
+	})
+
+	t.Run("lease", func(t *testing.T) {
+		lost := make(chan context.CancelFunc, 64)
+		leased := []nsLockRequest{{newLock: func() RWLocker { return ldLeaseLock{lock("a"), lost} }, write: true, atomic: true}, {newLock: newLock("b"), write: true, atomic: true}}
+		// Losing the first lease while the second is waited for retries
+		// with a fresh context.
+		releaseB := ldHoldLock(t, lock("b"))
+		type result struct {
+			lkctx  LockContext
+			unlock func()
+			err    error
+		}
+		done := make(chan result, 1)
+		go func() {
+			lkctx, unlock, err := getNSLocks(t.Context(), leased, globalOperationTimeout)
+			done <- result{lkctx, unlock, err}
+		}()
+		(<-lost)()
+		releaseB()
+		res := <-done
+		if res.err != nil {
+			t.Fatal(res.err)
+		}
+		if res.lkctx.Context().Err() != nil {
+			t.Fatal("returned context ended before any lease was lost")
+		}
+		// Losing a held lease ends the returned context.
+		var cancel context.CancelFunc
+		for len(lost) > 0 {
+			cancel = <-lost
+		}
+		cancel()
+		select {
+		case <-res.lkctx.Context().Done():
+		case <-time.After(time.Second):
+			t.Error("lost lease did not end the locks' context")
+		}
+		res.unlock()
+		if !ldTry(t, lock("a"), true) || !ldTry(t, lock("b"), true) {
+			t.Error("locks left behind")
+		}
+	})
+}
+
+// A group whose lock times out fails alone and the later groups still run.
+func TestDeleteObjectsInSetsLaterGroupsRun(t *testing.T) {
+	for _, mode := range ldModes[1:] {
+		t.Run(mode, func(t *testing.T) {
+			z, bucket := ldPool(t, mode)
+			k0, k1 := ldKey(t, z, "later", 0), ldKey(t, z, "later", 1)
+			ldMustPut(t, z, bucket, k0, "zero")
+			ldMustPut(t, z, bucket, k1, "one")
+			release := ldHoldLock(t, z.serverPools[0].getHashedSet(k0).NewNSLock(bucket, k0))
+			defer release()
+			objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: k0}}, {ObjectV: ObjectV{ObjectName: k1}}}
+			dobjects := []DeletedObject{{ObjectName: k0}, {ObjectName: k1}}
+			derrs := make([]error, 2)
+			z.deleteObjectsInSets(t.Context(), bucket, objects, ObjectOptions{}, dobjects, derrs, newDynamicTimeout(time.Second, time.Second))
+			if !errors.As(derrs[0], &OperationTimedOut{}) || derrs[1] != nil {
+				t.Fatalf("results %v, want a timeout for k0 only", derrs)
+			}
+			release()
+			if body, _ := ldBody(t, z, bucket, k0); body != "zero" {
+				t.Errorf("k0 = %q", body)
+			}
+			ldAbsent(t, z, bucket, k1)
+		})
+	}
+}
+
+// The local lock takes the names of a group one by one. A group that waits
+// for a name must not keep the names it holds: a part upload holds its part
+// lock and then reads its upload ID, which the batch names too.
+func TestDeleteObjectsInSetsLocalGroupReleases(t *testing.T) {
+	z, bucket := ldPool(t, ldLocal)
+	pool := z.serverPools[0]
+	var upload, part string
+	for i := range 1000 {
+		upload = ldKey(t, z, "local-upload", 1) + fmt.Sprintf("/u%d", i)
+		part = upload + "/1"
+		if pool.getHashedSetIndex(upload) == 1 && pool.getHashedSetIndex(part) == 1 {
+			break
+		}
+	}
+	er := pool.getHashedSet(upload)
+	releasePart := ldHoldLock(t, er.NewNSLock(bucket, part))
+	defer releasePart()
+	done := make(chan []error, 1)
+	go func() {
+		_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: upload}}, {ObjectV: ObjectV{ObjectName: part}}}, ObjectOptions{})
+		done <- errs
+	}()
+	// The batch takes the upload name, waits for the part, and lets the
+	// part upload's read of its upload ID through.
+	ldTaken(t, er.NewNSLock(bucket, upload))
+	read := er.NewNSLock(bucket, upload)
+	rctx, err := read.GetRLock(t.Context(), newDynamicTimeout(4*nsLockTryTimeout, 4*nsLockTryTimeout))
+	if err != nil {
+		t.Fatalf("batch kept a name while waiting: %v", err)
+	}
+	read.RUnlock(rctx)
+	releasePart()
+	select {
+	case errs := <-done:
+		if errors.Join(errs...) != nil {
+			t.Fatal(errs)
+		}
+	case <-time.After(30 * time.Second):
+		t.Fatal("bulk delete did not finish")
+	}
+}
+
+// ldLostUnlockReply applies its first unlock but loses the reply until gate
+// opens, so dsync retries that unlock in the background.
+type ldLostUnlockReply struct {
+	dsync.NetLocker
+	gate             <-chan struct{}
+	applied, retried chan<- struct{}
+	calls            atomic.Int32
+}
+
+func (l *ldLostUnlockReply) Unlock(ctx context.Context, args dsync.LockArgs) (bool, error) {
+	n := l.calls.Add(1)
+	ok, err := l.NetLocker.Unlock(ctx, args)
+	switch n {
+	case 1:
+		l.applied <- struct{}{}
+		<-l.gate
+		return false, errors.New("unlock reply lost")
+	case 2:
+		l.retried <- struct{}{}
+	}
+	return ok, err
+}
+
+// ldFailFirst fails the first acquisition of any of its instances.
+type ldFailFirst struct {
+	RWLocker
+	calls *atomic.Int32
+}
+
+func (l ldFailFirst) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
+	if l.calls.Add(1) == 1 {
+		return LockContext{}, OperationTimedOut{}
+	}
+	return l.RWLocker.GetLock(ctx, timeout)
+}
+
+// A distributed unlock of a failed attempt that is retried late must not
+// release the lock the next attempt took.
+func TestGetNSLocksFreshLockPerAttempt(t *testing.T) {
+	gate := make(chan struct{})
+	applied, retried := make(chan struct{}, 3), make(chan struct{}, 3)
+	lockers := make([]dsync.NetLocker, 3)
+	for i := range lockers {
+		lockers[i] = &ldLostUnlockReply{NetLocker: newLocker(), gate: gate, applied: applied, retried: retried}
+	}
+	ns := newNSLock(true)
+	dst := func() RWLocker {
+		return ns.NewNSLock(func() ([]dsync.NetLocker, string) { return lockers, "test" }, "bucket", "destination")
+	}
+	local := newNSLock(false)
+	var calls atomic.Int32
+	src := func() RWLocker { return ldFailFirst{local.NewNSLock(nil, "bucket", "source"), &calls} }
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second)
+	defer cancel()
+	opened := false
+	open := func() {
+		if !opened {
+			opened = true
+			close(gate)
+		}
+	}
+	defer open()
+	_, unlock, err := getNSLocks(ctx, []nsLockRequest{{newLock: dst, write: true, atomic: true}, {newLock: src, write: true, atomic: true}}, newDynamicTimeout(10*time.Second, 10*time.Second))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer unlock()
+	for range lockers {
+		select {
+		case <-applied:
+		case <-ctx.Done():
+			t.Fatal("the first attempt's unlock never reached the lockers")
+		}
+	}
+	open()
+	for range lockers {
+		select {
+		case <-retried:
+		case <-ctx.Done():
+			t.Fatal("the lost unlock was not retried")
+		}
+	}
+	competitor := dst()
+	lc, err := competitor.GetLock(ctx, newDynamicTimeout(300*time.Millisecond, 300*time.Millisecond))
+	if err == nil {
+		competitor.Unlock(lc)
+		t.Fatal("a late unlock retry released the held lock")
+	}
+}
+
+// One attempt at a local lock over several names is bounded as a whole, and so
+// is the acquisition.
+func TestGetNSLocksLocalAttemptBound(t *testing.T) {
+	ns := newNSLock(false)
+	a, b := ns.NewNSLock(nil, "bucket", "a"), ns.NewNSLock(nil, "bucket", "b")
+	releaseA, releaseB := ldHoldLock(t, a), ldHoldLock(t, b)
+	defer releaseA()
+	defer releaseB()
+	time.AfterFunc(900*time.Millisecond, releaseA)
+	time.AfterFunc(1800*time.Millisecond, releaseB)
+	start := time.Now()
+	_, unlock, err := getNSLocks(t.Context(), []nsLockRequest{{
+		newLock: func() RWLocker { return ns.NewNSLock(nil, "bucket", "a", "b") }, write: true,
+	}}, newDynamicTimeout(1500*time.Millisecond, 1500*time.Millisecond))
+	if err == nil {
+		unlock()
+		t.Fatalf("acquired after %v with a deadline of 1.5s", time.Since(start))
+	}
+	if !errors.As(err, &OperationTimedOut{}) {
+		t.Fatalf("got %v, want a timeout", err)
+	}
+}
+
+// ldLeaseLayer gives the COPY handler locks whose lease the test ends, and
+// holds its destination lookup until the lookup's context ends.
+type ldLeaseLayer struct {
+	ObjectLayer
+	destination string
+	lost        chan context.CancelFunc
+	lookup      chan context.Context
+}
+
+func (l *ldLeaseLayer) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) {
+	lc, unlock, err := l.ObjectLayer.lockObjects(ctx, reqs...)
+	if err != nil {
+		return lc, unlock, err
+	}
+	lease, cancel := context.WithCancel(lc.Context())
+	l.lost <- cancel
+	return LockContext{ctx: lease, cancel: cancel}, func() { cancel(); unlock() }, nil
+}
+
+func (l *ldLeaseLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	if object == l.destination && opts.NoLock {
+		l.lookup <- ctx
+		<-ctx.Done()
+		return ObjectInfo{}, ctx.Err()
+	}
+	return l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+}
+
+// The COPY handler's work under its locks ends when a lease is lost, the
+// destination lookups included.
+func TestLockDomainCopyLookupEndsWithLease(t *testing.T) {
+	z, bucket, _ := ldAPI(t, ldShared)
+	ldMustPut(t, z, bucket, "source", "")
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	globalTierConfigMgr = NewTierConfigMgr()
+	globalTierConfigMgr.Tiers["WARM"] = madmin.TierConfig{Version: madmin.TierConfigVer, Type: madmin.S3, Name: "WARM", S3: &madmin.TierS3{Bucket: "warm"}}
+	layer := &ldLeaseLayer{ObjectLayer: z, destination: "destination", lost: make(chan context.CancelFunc, 1), lookup: make(chan context.Context, 1)}
+	router := initTestAPIEndPoints(layer, []string{"CopyObject"})
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer cancel()
+	code := make(chan int, 1)
+	go func() { code <- ldCopyRequest(ctx, t, router, bucket, "source", "destination", nil).Code }()
+	var lookup context.Context
+	select {
+	case lookup = <-layer.lookup:
+	case <-ctx.Done():
+		t.Fatal("COPY did not look up its destination")
+	}
+	(<-layer.lost)()
+	select {
+	case <-lookup.Done():
+	case <-time.After(5 * time.Second):
+		t.Fatal("the destination lookup outlived the lost lease")
+	}
+	if c := <-code; c == http.StatusOK {
+		t.Error("COPY succeeded after losing its lock")
+	}
+}

cmd/erasure-server-pool.go

@@ -1239,8 +1239,26 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		object = encodeDirObject(object)
 	}
 
-	// Serialize logical deletion with pooled writes and metadata updates.
-	lk := z.NewNSLock(bucket, object)
+	// Serialize logical deletion with the writes and metadata updates of the
+	// object: take the lock they take (see NewObjectNSLock). With one pool
+	// that is the object's own erasure set, which is where PutObject,
+	// multipart commit and the metadata updates lock, so a pool whose sets
+	// use different hosts still excludes them. With several pools it is the
+	// pools lock, as before. Exact-key deletes that use prefix deletion
+	// (DeletePrefixObject) are deletes of one object. A true prefix deletion
+	// keeps the pools lock on the prefix name, which never covered the
+	// objects below it.
+	//
+	// Nodes that still run a release before this routing lock single-pool
+	// deletes and copies in the first set. In a pool whose sets use different
+	// hosts they do not exclude the deletes and copies of updated nodes, nor
+	// their writes, so such a pool must not run mixed versions.
+	var lk RWLocker
+	if opts.DeletePrefix && !opts.DeletePrefixObject {
+		lk = z.NewNSLock(bucket, object)
+	} else {
+		lk = z.NewObjectNSLock(bucket, object)
+	}
 	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -1322,10 +1340,10 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 
 	// Evaluate the conditional (If-Match) precondition while the write lock
 	// acquired above is held, before the delete-marker short-circuit and before
-	// any version is removed, so the object cannot change between the check and
-	// the delete. This is scoped to a single erasure set (see the note at the
-	// lock above): only there do the delete lock and the write path share the
-	// same lock namespace, making the check-then-delete atomic.
+	// any version is removed. The object's writers take the same lock (see the
+	// note at the lock above), so the object cannot change between the check
+	// and the delete. Multi-pool conditional deletes return above, through
+	// deleteObjectReconciled, under the same pools lock.
 	if opts.CheckPrecondFn != nil {
 		// pinfo.ObjInfo is the current latest version. getPoolInfoExistingWithOpts
 		// intentionally strips VersionID, so for a version-scoped delete read the
@@ -1509,6 +1527,18 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		objSets.Add(objects[i].ObjectName)
 	}
 
+	// A single pool locks each name in the set its writers lock it in, one
+	// set at a time. A single node keeps one lock table for all sets, so it
+	// keeps the one bulk lock below: its local lock takes the names one by
+	// one and holds the earlier names while it waits.
+	if z.SinglePool() && !z.serverPools[0].sharesLocalNSLock() {
+		z.deleteObjectsInSets(ctx, bucket, objects, opts, dobjects, derrs, globalOperationTimeout)
+		for i := range dobjects {
+			dobjects[i].ObjectName = decodeDirObject(dobjects[i].ObjectName)
+		}
+		return dobjects, derrs
+	}
+
 	// Acquire a bulk write lock across 'objects'
 	multiDeleteLock := z.NewNSLock(bucket, objSets.ToSlice()...)
 	lkctx, err := multiDeleteLock.GetLock(ctx, globalOperationTimeout)
@@ -1633,8 +1663,10 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 
+	// A caller that also locks the source takes both with lockObjects, before
+	// it reads the source, and passes NoLock.
 	if !dstOpts.NoLock {
-		ns := z.NewNSLock(dstBucket, dstObject)
+		ns := z.NewObjectNSLock(dstBucket, dstObject)
 		lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
 			return ObjectInfo{}, err
@@ -2231,7 +2263,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 		return MultipartInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID)
 	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return MultipartInfo{}, err
@@ -2272,7 +2304,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		return ListPartsInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID)
 	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return ListPartsInfo{}, err
@@ -2323,7 +2355,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		}
 	}(ctx)
 
-	lk := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	lk := z.uploadIDNSLock(bucket, object, uploadID)
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return err
@@ -2363,12 +2395,10 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 
 	// Keep object -> upload lock ordering for single-pool completion too:
 	// native restore holds the object lock while publishing upload parts.
+	// Both locks are in the object's lock domain, the set-local one of
+	// single-pool writes.
 	if !opts.NoLock {
-		lk := z.NewNSLock(bucket, encodeDirObject(object))
-		if z.SinglePool() {
-			// Preserve the set-local lock domain used by single-pool writes.
-			lk = z.serverPools[0].getHashedSet(object).NewNSLock(bucket, encodeDirObject(object))
-		}
+		lk := z.NewObjectNSLock(bucket, encodeDirObject(object))
 		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
 			return objInfo, err
@@ -2433,7 +2463,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 	// An empty upload ID aliases the object lock already held above.
 	// Let the storage layer return its normal missing bucket/upload error.
 	if uploadID != "" {
-		uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+		uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID)
 		wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
 			return objInfo, err

cmd/metacache-server-pool.go

@@ -265,9 +265,12 @@ func (z *erasureServerPools) listPathShortcutEOF(ctx context.Context, bucket str
 }
 
 // listingRLock covers the lock domains used by normal same-key mutations.
-// Multi-pool writes use the pools lock. In a single pool, DELETE/COPY use the
-// first set while PUT and metadata updates use the destination set, so recovery
-// takes both in that order. No writer routing or wire protocol changes here.
+// Multi-pool writes use the pools lock. In a single pool every object write,
+// DELETE and COPY included, locks the object's own set (see NewObjectNSLock),
+// which is the source set here. Releases before that routing locked
+// single-pool DELETE and COPY in the first set, so recovery still takes the
+// first set too, in that order, while nodes of those releases may run. It can
+// drop that lock only behind a version boundary that keeps such nodes out.
 func (z *erasureServerPools) listingRLock(ctx context.Context, bucket, object string, source *erasureObjects) (LockContext, func(), error) {
 	firstSet := z.serverPools[0].sets[0]
 	first := z.NewNSLock(bucket, object)

cmd/object-api-interface.go

@@ -262,7 +262,13 @@ func (o *ObjectOptions) SetEvalRetentionBypassFn(f EvalRetentionBypassFn) {
 // ObjectLayer implements primitives for object API layer.
 type ObjectLayer interface {
 	// Locking operations on object.
+	// NewNSLock locks names in one stable domain. It is for names that are
+	// not objects, such as bucket and IAM metadata locks.
 	NewNSLock(bucket string, objects ...string) RWLocker
+	// NewObjectNSLock locks one object in the lock domain of its writes.
+	NewObjectNSLock(bucket, object string) RWLocker
+	// lockObjects locks several objects together, see erasureServerPools.
+	lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error)
 
 	// Storage operations.
 	Shutdown(context.Context) error

cmd/object-handlers.go

@@ -34,6 +34,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -183,7 +184,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 
 	// Take read lock on object, here so subsequent lower-level
 	// calls do not need to.
-	lock := objectAPI.NewNSLock(bucket, object)
+	lock := objectAPI.NewObjectNSLock(bucket, object)
 	lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1470,12 +1471,35 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return checkCopyObjectPreconditions(ctx, w, r, o)
 	}
 	getOpts.CheckPrecondFn = checkCopyPrecondFn
-	if cpSrcDstSame {
-		getOpts.NoLock = true
+
+	// Lock the source for reading and a local destination for writing before
+	// the source is opened, both in the lock domain of their writers. Taking
+	// them together in one order keeps copies in opposite directions from
+	// holding each other; a copy onto itself takes one write lock. The source
+	// is then read, preconditions and inline data included, and the
+	// destination looked up and written, all under these locks and without
+	// taking them again. That work runs in lockCtx, which ends when either
+	// lock is lost and so aborts the copy; errors are still reported in the
+	// request's context. Both locks are released once the copy has been
+	// written: a synchronous replication below reads the destination with its
+	// own lock.
+	locks := []objectLockRequest{{bucket: srcBucket, object: srcObject}}
+	if !remoteCallRequired {
+		locks = append(locks, objectLockRequest{bucket: dstBucket, object: dstObject, write: true})
 	}
+	lkctx, unlock, err := objectAPI.lockObjects(ctx, locks...)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	unlockCopy := sync.OnceFunc(unlock)
+	defer unlockCopy()
+	lockCtx := lkctx.Context()
+	getOpts.NoLock = true
+	dstOpts.NoLock = true
 
 	var rs *HTTPRangeSpec
-	gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, getOpts)
+	gr, err := getObjectNInfo(lockCtx, srcBucket, srcObject, rs, r.Header, getOpts)
 	if err != nil {
 		if isErrPreconditionFailed(err) {
 			return
@@ -1542,7 +1566,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	length := actualSize
 
 	if !cpSrcDstSame {
-		if err := enforceBucketQuotaHard(ctx, dstBucket, actualSize); err != nil {
+		if err := enforceBucketQuotaHard(lockCtx, dstBucket, actualSize); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -1560,12 +1584,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 		compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
 
-		reader = etag.NewReader(ctx, reader, nil, nil)
+		reader = etag.NewReader(lockCtx, reader, nil, nil)
 	} else {
 		reader = gr
 	}
 
-	srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualSize)
+	srcInfo.Reader, err = hash.NewReader(lockCtx, reader, length, "", "", actualSize)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1660,7 +1684,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			}
 		}
 
-		if err = rotateKey(ctx, oldKey, newKeyID, newKey, srcBucket, srcObject, encMetadata, kmsCtx); err != nil {
+		if err = rotateKey(lockCtx, oldKey, newKeyID, newKey, srcBucket, srcObject, encMetadata, kmsCtx); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -1745,7 +1769,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			dstOpts.IndexCB = cb
 			defer s2c.Close()
 			reader = etag.Wrap(s2c, checksumReader)
-			srcInfo.Reader, err = hash.NewReader(ctx, reader, -1, "", "", actualSize)
+			srcInfo.Reader, err = hash.NewReader(lockCtx, reader, -1, "", "", actualSize)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -1758,7 +1782,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		if isTargetEncrypted && !remoteCallRequired {
 			var encReader io.Reader
 			kind, _ := crypto.IsRequested(r.Header)
-			encReader, objEncKey, err = newEncryptReader(ctx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx)
+			encReader, objEncKey, err = newEncryptReader(lockCtx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -1773,7 +1797,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 
 		// do not try to verify encrypted content
-		srcInfo.Reader, err = hash.NewReader(ctx, reader, targetSize, "", "", actualSize)
+		srcInfo.Reader, err = hash.NewReader(lockCtx, reader, targetSize, "", "", actualSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -1845,7 +1869,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
 	retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectRetentionAction)
 	holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectLegalHoldAction)
-	getObjectInfo := objectAPI.GetObjectInfo
+	// The destination is locked above: look it up without its lock, in the
+	// locks' context.
+	getObjectInfo := func(_ context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+		opts.NoLock = true
+		return objectAPI.GetObjectInfo(lockCtx, bucket, object, opts)
+	}
 
 	// apply default bucket configuration/governance headers for dest side.
 	retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms, replicaTrusted)
@@ -2031,9 +2060,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		// For an ordinary federated copy, actualSize is the logical plaintext
 		// length to declare. srcInfo.Size is not a reliable wire length because
 		// the read path may already have adjusted it.
-		writeCtx, modified := withFederatedWriteTime(ctx)
+		writeCtx, modified := withFederatedWriteTime(lockCtx)
 		remoteObjInfo, rerr := core.PutObject(writeCtx, dstBucket, dstObject, srcInfo.Reader,
 			actualSize, "", "", opts)
+		gr.Close()
+		unlockCopy()
 		if rerr != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL)
 			return
@@ -2080,7 +2111,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 		// Copy source object to destination, if source and destination
 		// object is same then only metadata is updated.
-		objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts)
+		objInfo, err = copyObjectFn(lockCtx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts)
+		gr.Close()
+		unlockCopy()
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return

cmd/object-retention-authz_test.go

@@ -544,6 +544,18 @@ func (p *hookBeforeWrite) CompleteMultipartUpload(ctx context.Context, bucket, k
 	return p.ObjectLayer.CompleteMultipartUpload(ctx, bucket, key, uploadID, parts, opts)
 }
 
+// The COPY handler checks Object Lock under its destination's write lock, so
+// no writer can land between that check and the copy. The latest a writer can
+// land is just before the handler takes its locks.
+func (p *hookBeforeWrite) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) {
+	for _, r := range reqs {
+		if r.write {
+			p.fire(r.object)
+		}
+	}
+	return p.ObjectLayer.lockObjects(ctx, reqs...)
+}
+
 func TestDirectoryObjectLockAfterPreflight(t *testing.T) {
 	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
 		t:                 t,