mirror of
https://github.com/pgsty/minio.git
synced 2026-07-23 14:10:25 +03:00
Add support for timeouts for locks (#4377)
This commit is contained in:
Frank Wessels
+62
-259
@@ -17,8 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -27,9 +25,9 @@ import (
|
||||
func TestNamespaceLockTest(t *testing.T) {
|
||||
// List of test cases.
|
||||
testCases := []struct {
|
||||
lk func(s1, s2, s3 string)
|
||||
lk func(s1, s2, s3 string, t time.Duration) bool
|
||||
unlk func(s1, s2, s3 string)
|
||||
rlk func(s1, s2, s3 string)
|
||||
rlk func(s1, s2, s3 string, t time.Duration) bool
|
||||
runlk func(s1, s2, s3 string)
|
||||
lkCount int
|
||||
lockedRefCount uint
|
||||
@@ -63,7 +61,9 @@ func TestNamespaceLockTest(t *testing.T) {
|
||||
|
||||
// Write lock tests.
|
||||
testCase := testCases[0]
|
||||
testCase.lk("a", "b", "c") // lock once.
|
||||
if !testCase.lk("a", "b", "c", 60*time.Second) { // lock once.
|
||||
t.Fatalf("Failed to acquire lock")
|
||||
}
|
||||
nsLk, ok := globalNSMutex.lockMap[nsParam{"a", "b"}]
|
||||
if !ok && testCase.shouldPass {
|
||||
t.Errorf("Lock in map missing.")
|
||||
@@ -83,10 +83,18 @@ func TestNamespaceLockTest(t *testing.T) {
|
||||
|
||||
// Read lock tests.
|
||||
testCase = testCases[1]
|
||||
testCase.rlk("a", "b", "c") // lock once.
|
||||
testCase.rlk("a", "b", "c") // lock second time.
|
||||
testCase.rlk("a", "b", "c") // lock third time.
|
||||
testCase.rlk("a", "b", "c") // lock fourth time.
|
||||
if !testCase.rlk("a", "b", "c", 60*time.Second) { // lock once.
|
||||
t.Fatalf("Failed to acquire first read lock")
|
||||
}
|
||||
if !testCase.rlk("a", "b", "c", 60*time.Second) { // lock second time.
|
||||
t.Fatalf("Failed to acquire second read lock")
|
||||
}
|
||||
if !testCase.rlk("a", "b", "c", 60*time.Second) { // lock third time.
|
||||
t.Fatalf("Failed to acquire third read lock")
|
||||
}
|
||||
if !testCase.rlk("a", "b", "c", 60*time.Second) { // lock fourth time.
|
||||
t.Fatalf("Failed to acquire fourth read lock")
|
||||
}
|
||||
nsLk, ok = globalNSMutex.lockMap[nsParam{"a", "b"}]
|
||||
if !ok && testCase.shouldPass {
|
||||
t.Errorf("Lock in map missing.")
|
||||
@@ -108,7 +116,9 @@ func TestNamespaceLockTest(t *testing.T) {
|
||||
|
||||
// Read lock 0 ref count.
|
||||
testCase = testCases[2]
|
||||
testCase.rlk("a", "c", "d") // lock once.
|
||||
if !testCase.rlk("a", "c", "d", 60*time.Second) { // lock once.
|
||||
t.Fatalf("Failed to acquire read lock")
|
||||
}
|
||||
|
||||
nsLk, ok = globalNSMutex.lockMap[nsParam{"a", "c"}]
|
||||
if !ok && testCase.shouldPass {
|
||||
@@ -128,258 +138,47 @@ func TestNamespaceLockTest(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockStats(t *testing.T) {
|
||||
func TestNamespaceLockTimedOut(t *testing.T) {
|
||||
|
||||
expectedResult := []lockStateCase{
|
||||
// Test case - 1.
|
||||
// Case where 10 read locks are held.
|
||||
// Entry for any of the 10 reads locks has to be found.
|
||||
// Since they held in a loop, Lock origin for first 10 read locks (opsID 0-9) should be the same.
|
||||
{
|
||||
|
||||
volume: "my-bucket",
|
||||
path: "my-object",
|
||||
opsID: "0",
|
||||
readLock: true,
|
||||
lockSource: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]",
|
||||
// expected metrics.
|
||||
expectedErr: nil,
|
||||
expectedLockStatus: "Running",
|
||||
|
||||
expectedGlobalLockCount: 10,
|
||||
expectedRunningLockCount: 10,
|
||||
expectedBlockedLockCount: 0,
|
||||
|
||||
expectedVolPathLockCount: 10,
|
||||
expectedVolPathRunningCount: 10,
|
||||
expectedVolPathBlockCount: 0,
|
||||
},
|
||||
// Test case - 2.
|
||||
// Case where the first 5 read locks are released.
|
||||
// Entry for any of the 6-10th "Running" reads lock has to be found.
|
||||
{
|
||||
volume: "my-bucket",
|
||||
path: "my-object",
|
||||
opsID: "6",
|
||||
readLock: true,
|
||||
lockSource: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]",
|
||||
// expected metrics.
|
||||
expectedErr: nil,
|
||||
expectedLockStatus: "Running",
|
||||
|
||||
expectedGlobalLockCount: 5,
|
||||
expectedRunningLockCount: 5,
|
||||
expectedBlockedLockCount: 0,
|
||||
|
||||
expectedVolPathLockCount: 5,
|
||||
expectedVolPathRunningCount: 5,
|
||||
expectedVolPathBlockCount: 0,
|
||||
},
|
||||
// Test case - 3.
|
||||
{
|
||||
|
||||
volume: "my-bucket",
|
||||
path: "my-object",
|
||||
opsID: "10",
|
||||
readLock: false,
|
||||
lockSource: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]",
|
||||
// expected metrics.
|
||||
expectedErr: nil,
|
||||
expectedLockStatus: "Running",
|
||||
|
||||
expectedGlobalLockCount: 2,
|
||||
expectedRunningLockCount: 1,
|
||||
expectedBlockedLockCount: 1,
|
||||
|
||||
expectedVolPathLockCount: 2,
|
||||
expectedVolPathRunningCount: 1,
|
||||
expectedVolPathBlockCount: 1,
|
||||
},
|
||||
// Test case - 4.
|
||||
{
|
||||
|
||||
volume: "my-bucket",
|
||||
path: "my-object",
|
||||
// expected metrics.
|
||||
expectedErr: nil,
|
||||
expectedLockStatus: "Blocked",
|
||||
|
||||
expectedGlobalLockCount: 1,
|
||||
expectedRunningLockCount: 0,
|
||||
expectedBlockedLockCount: 1,
|
||||
|
||||
expectedVolPathLockCount: 1,
|
||||
expectedVolPathRunningCount: 0,
|
||||
expectedVolPathBlockCount: 1,
|
||||
},
|
||||
// Test case - 5.
|
||||
{
|
||||
|
||||
volume: "my-bucket",
|
||||
path: "my-object",
|
||||
opsID: "11",
|
||||
readLock: false,
|
||||
lockSource: "[lock held] in github.com/minio/minio/cmd.TestLockStats[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:298]",
|
||||
// expected metrics.
|
||||
expectedErr: nil,
|
||||
expectedLockStatus: "Running",
|
||||
|
||||
expectedGlobalLockCount: 1,
|
||||
expectedRunningLockCount: 1,
|
||||
expectedBlockedLockCount: 0,
|
||||
|
||||
expectedVolPathLockCount: 1,
|
||||
expectedVolPathRunningCount: 1,
|
||||
expectedVolPathBlockCount: 0,
|
||||
},
|
||||
// Test case - 6.
|
||||
// Case where in the first 5 read locks are released, but 2 write locks are
|
||||
// blocked waiting for the remaining 5 read locks locks to be released (10 read locks were held initially).
|
||||
// We check the entry for the first blocked write call here.
|
||||
{
|
||||
|
||||
volume: "my-bucket",
|
||||
path: "my-object",
|
||||
opsID: "10",
|
||||
readLock: false,
|
||||
// write lock is held at line 318.
|
||||
// this confirms that we are looking the right write lock.
|
||||
lockSource: "[lock held] in github.com/minio/minio/cmd.TestLockStats.func2[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:318]",
|
||||
// expected metrics.
|
||||
expectedErr: nil,
|
||||
expectedLockStatus: "Blocked",
|
||||
|
||||
// count of held(running) + blocked locks.
|
||||
expectedGlobalLockCount: 7,
|
||||
// count of acquired locks.
|
||||
expectedRunningLockCount: 5,
|
||||
// 2 write calls are blocked, waiting for the remaining 5 read locks.
|
||||
expectedBlockedLockCount: 2,
|
||||
|
||||
expectedVolPathLockCount: 7,
|
||||
expectedVolPathRunningCount: 5,
|
||||
expectedVolPathBlockCount: 2,
|
||||
},
|
||||
// Test case - 7.
|
||||
// Case where in 9 out of 10 read locks are released.
|
||||
// Since there's one more pending read lock, the 2 write locks are still blocked.
|
||||
// Testing the entry for the last read lock.
|
||||
{volume: "my-bucket",
|
||||
path: "my-object",
|
||||
opsID: "9",
|
||||
readLock: true,
|
||||
lockSource: "[lock held] in github.com/minio/minio/cmd.TestLockStats.func2[/Users/hackintoshrao/mycode/go/src/github.com/minio/minio/cmd/namespace-lock_test.go:318]",
|
||||
// expected metrics.
|
||||
expectedErr: nil,
|
||||
expectedLockStatus: "Running",
|
||||
|
||||
// Total running + blocked locks.
|
||||
// 2 blocked write lock.
|
||||
expectedGlobalLockCount: 3,
|
||||
expectedRunningLockCount: 1,
|
||||
expectedBlockedLockCount: 2,
|
||||
|
||||
expectedVolPathLockCount: 3,
|
||||
expectedVolPathRunningCount: 1,
|
||||
expectedVolPathBlockCount: 2,
|
||||
},
|
||||
// Test case - 8.
|
||||
{
|
||||
|
||||
volume: "my-bucket",
|
||||
path: "my-object",
|
||||
// expected metrics.
|
||||
expectedErr: nil,
|
||||
expectedLockStatus: "Blocked",
|
||||
|
||||
expectedGlobalLockCount: 0,
|
||||
expectedRunningLockCount: 0,
|
||||
expectedBlockedLockCount: 0,
|
||||
},
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
// initializing the locks.
|
||||
initNSLock(false)
|
||||
|
||||
// hold 10 read locks.
|
||||
for i := 0; i < 10; i++ {
|
||||
globalNSMutex.RLock("my-bucket", "my-object", strconv.Itoa(i))
|
||||
}
|
||||
// expected lock info.
|
||||
expectedLockStats := expectedResult[0]
|
||||
// verify the actual lock info with the expected one.
|
||||
verifyLockState(expectedLockStats, t, 1)
|
||||
// unlock 5 readlock.
|
||||
for i := 0; i < 5; i++ {
|
||||
globalNSMutex.RUnlock("my-bucket", "my-object", strconv.Itoa(i))
|
||||
// Get write lock
|
||||
if !globalNSMutex.Lock("my-bucket", "my-object", "abc", 60*time.Second) {
|
||||
t.Fatalf("Failed to acquire lock")
|
||||
}
|
||||
|
||||
expectedLockStats = expectedResult[1]
|
||||
// verify the actual lock info with the expected one.
|
||||
verifyLockState(expectedLockStats, t, 2)
|
||||
|
||||
syncChan := make(chan struct{}, 1)
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
// blocks till all read locks are released.
|
||||
globalNSMutex.Lock("my-bucket", "my-object", strconv.Itoa(10))
|
||||
// Once the above attempt to lock is unblocked/acquired, we verify the stats and release the lock.
|
||||
expectedWLockStats := expectedResult[2]
|
||||
// Since the write lock acquired here, the number of blocked locks should reduce by 1 and
|
||||
// count of running locks should increase by 1.
|
||||
verifyLockState(expectedWLockStats, t, 3)
|
||||
// release the write lock.
|
||||
globalNSMutex.Unlock("my-bucket", "my-object", strconv.Itoa(10))
|
||||
// The number of running locks should decrease by 1.
|
||||
// expectedWLockStats = expectedResult[3]
|
||||
// verifyLockState(expectedWLockStats, t, 4)
|
||||
// Take the lock stats after the first write lock is unlocked.
|
||||
// Only then unlock then second write lock.
|
||||
syncChan <- struct{}{}
|
||||
}()
|
||||
// waiting so that the write locks in the above go routines are held.
|
||||
// sleeping so that we can predict the order of the write locks held.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// since there are 5 more readlocks still held on <"my-bucket","my-object">,
|
||||
// an attempt to hold write locks blocks. So its run in a new go routine.
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
// blocks till all read locks are released.
|
||||
globalNSMutex.Lock("my-bucket", "my-object", strconv.Itoa(11))
|
||||
// Once the above attempt to lock is unblocked/acquired, we release the lock.
|
||||
// Unlock the second write lock only after lock stats for first write lock release is taken.
|
||||
<-syncChan
|
||||
// The number of running locks should decrease by 1.
|
||||
expectedWLockStats := expectedResult[4]
|
||||
verifyLockState(expectedWLockStats, t, 5)
|
||||
globalNSMutex.Unlock("my-bucket", "my-object", strconv.Itoa(11))
|
||||
}()
|
||||
|
||||
expectedLockStats = expectedResult[5]
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
// verify the actual lock info with the expected one.
|
||||
verifyLockState(expectedLockStats, t, 6)
|
||||
|
||||
// unlock 4 out of remaining 5 read locks.
|
||||
for i := 0; i < 4; i++ {
|
||||
globalNSMutex.RUnlock("my-bucket", "my-object", strconv.Itoa(i+5))
|
||||
// Second attempt for write lock on same resource should time out
|
||||
locked := globalNSMutex.Lock("my-bucket", "my-object", "def", 1*time.Second)
|
||||
if locked {
|
||||
t.Fatalf("Should not have acquired lock")
|
||||
}
|
||||
|
||||
// verify the entry for one remaining read lock and count of blocked write locks.
|
||||
expectedLockStats = expectedResult[6]
|
||||
// verify the actual lock info with the expected one.
|
||||
verifyLockState(expectedLockStats, t, 7)
|
||||
// Read lock on same resource should also time out
|
||||
locked = globalNSMutex.RLock("my-bucket", "my-object", "def", 1*time.Second)
|
||||
if locked {
|
||||
t.Fatalf("Should not have acquired read lock while write lock is active")
|
||||
}
|
||||
|
||||
// Releasing the last read lock.
|
||||
globalNSMutex.RUnlock("my-bucket", "my-object", strconv.Itoa(9))
|
||||
wg.Wait()
|
||||
expectedLockStats = expectedResult[7]
|
||||
// verify the actual lock info with the expected one.
|
||||
verifyGlobalLockStats(expectedLockStats, t, 8)
|
||||
// Release write lock
|
||||
globalNSMutex.Unlock("my-bucket", "my-object", "abc")
|
||||
|
||||
// Get read lock
|
||||
if !globalNSMutex.RLock("my-bucket", "my-object", "ghi", 60*time.Second) {
|
||||
t.Fatalf("Failed to acquire read lock")
|
||||
}
|
||||
|
||||
// Write lock on same resource should time out
|
||||
locked = globalNSMutex.Lock("my-bucket", "my-object", "klm", 1*time.Second)
|
||||
if locked {
|
||||
t.Fatalf("Should not have acquired lock")
|
||||
}
|
||||
|
||||
// 2nd read lock should be just fine
|
||||
if !globalNSMutex.RLock("my-bucket", "my-object", "nop", 60*time.Second) {
|
||||
t.Fatalf("Failed to acquire second read lock")
|
||||
}
|
||||
|
||||
// Release both read locks
|
||||
globalNSMutex.RUnlock("my-bucket", "my-object", "ghi")
|
||||
globalNSMutex.RUnlock("my-bucket", "my-object", "nop")
|
||||
}
|
||||
|
||||
// Tests functionality to forcefully unlock locks.
|
||||
@@ -387,7 +186,9 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
|
||||
|
||||
// Create lock.
|
||||
lock := globalNSMutex.NewNSLock("bucket", "object")
|
||||
lock.Lock()
|
||||
if lock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
|
||||
t.Fatalf("Failed to get lock")
|
||||
}
|
||||
// Forcefully unlock lock.
|
||||
globalNSMutex.ForceUnlock("bucket", "object")
|
||||
|
||||
@@ -396,8 +197,10 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
|
||||
go func() {
|
||||
// Try to claim lock again.
|
||||
anotherLock := globalNSMutex.NewNSLock("bucket", "object")
|
||||
anotherLock.Lock()
|
||||
// And signal succes.
|
||||
if anotherLock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
|
||||
t.Fatalf("Failed to get lock")
|
||||
}
|
||||
// And signal success.
|
||||
ch <- struct{}{}
|
||||
}()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user