fix: threadwalk lockup under high load (#9992)

Main issue is that `t.pool[params]` should be `t.pool[oldest]`.

We add a bit more safety features for the code.

* Make writes to the endTimerCh non-blocking in all cases
   so multiple releases cannot lock up.
* Double check expectations.
* Shift down deletes with copy instead of truncating slice.
* Actually delete the oldest if we are above total limit.
* Actually delete the oldest found and not the current.
* Unexport the mutex so nobody from the outside can meddle with it.

cmd/tree-walk-pool_test.go

@@ -80,22 +80,22 @@ func TestManyWalksSameParam(t *testing.T) {
 			tw.Set(params, resultCh, endWalkCh)
 		}
 
-		tw.Lock()
+		tw.mu.Lock()
 		if walks, ok := tw.pool[params]; ok {
 			if len(walks) != treeWalkSameEntryLimit {
 				t.Error("There aren't as many walks as were Set")
 			}
 		}
-		tw.Unlock()
+		tw.mu.Unlock()
 		for i := 0; i < treeWalkSameEntryLimit; i++ {
-			tw.Lock()
+			tw.mu.Lock()
 			if walks, ok := tw.pool[params]; ok {
 				// Before ith Release we should have n-i treeWalk go-routines.
 				if treeWalkSameEntryLimit-i != len(walks) {
 					t.Error("There aren't as many walks as were Set")
 				}
 			}
-			tw.Unlock()
+			tw.mu.Unlock()
 			tw.Release(params)
 		}
 	}
@@ -125,22 +125,22 @@ func TestManyWalksSameParamPrune(t *testing.T) {
 			tw.Set(params, resultCh, endWalkCh)
 		}
 
-		tw.Lock()
+		tw.mu.Lock()
 		if walks, ok := tw.pool[params]; ok {
 			if len(walks) != treeWalkSameEntryLimit {
 				t.Error("There aren't as many walks as were Set")
 			}
 		}
-		tw.Unlock()
+		tw.mu.Unlock()
 		for i := 0; i < treeWalkSameEntryLimit; i++ {
-			tw.Lock()
+			tw.mu.Lock()
 			if walks, ok := tw.pool[params]; ok {
 				// Before ith Release we should have n-i treeWalk go-routines.
 				if treeWalkSameEntryLimit-i != len(walks) {
 					t.Error("There aren't as many walks as were Set")
 				}
 			}
-			tw.Unlock()
+			tw.mu.Unlock()
 			tw.Release(params)
 		}
 	}