diff --git a/libs/rand/sampling.go b/libs/rand/sampling.go deleted file mode 100644 index 6dd730601..000000000 --- a/libs/rand/sampling.go +++ /dev/null @@ -1,121 +0,0 @@ -package rand - -import ( - "fmt" - "math/big" - s "sort" -) - -// Interface for performing weighted deterministic random selection. -type Candidate interface { - Priority() uint64 - LessThan(other Candidate) bool -} - -// Select a specified number of candidates randomly from the candidate set based on each priority. This function is -// deterministic and will produce the same result for the same input. -// -// Inputs: -// seed - 64bit integer used for random selection. -// candidates - A set of candidates. You get different results depending on the order. -// sampleSize - The number of candidates to select at random. -// totalPriority - The exact sum of the priorities of each candidate. -// -// Returns: -// samples - A randomly selected candidate from a set of candidates. NOTE that the same candidate may have been -// selected in duplicate. -func RandomSamplingWithPriority( - seed uint64, candidates []Candidate, sampleSize int, totalPriority uint64) (samples []Candidate) { - - // This step is performed if and only if the parameter is invalid. The reasons are as stated in the message: - err := checkInvalidPriority(candidates, totalPriority) - if err != nil { - panic(err) - } - - // generates a random selection threshold for candidates' cumulative priority - thresholds := make([]uint64, sampleSize) - for i := 0; i < sampleSize; i++ { - // calculating [gross weights] × [(0,1] random number] - thresholds[i] = RandomThreshold(&seed, totalPriority) - } - s.Slice(thresholds, func(i, j int) bool { return thresholds[i] < thresholds[j] }) - - // extract candidates with a cumulative priority threshold - samples = make([]Candidate, sampleSize) - cumulativePriority := uint64(0) - undrawn := 0 - for _, candidate := range candidates { - for thresholds[undrawn] < cumulativePriority+candidate.Priority() { - samples[undrawn] = candidate - undrawn++ - if undrawn == len(samples) { - return - } - } - cumulativePriority += candidate.Priority() - } - - // We're assuming you never get to this code - panic(fmt.Sprintf("Cannot select samples; "+ - "totalPriority=%d, seed=%d, sampleSize=%d, undrawn=%d, threshold[%d]=%d, len(candidates)=%d", - totalPriority, seed, sampleSize, undrawn, undrawn, thresholds[undrawn], len(candidates))) -} - -const uint64Mask = uint64(0x7FFFFFFFFFFFFFFF) - -var divider *big.Int - -func init() { - divider = big.NewInt(int64(uint64Mask)) - divider.Add(divider, big.NewInt(1)) -} - -func RandomThreshold(seed *uint64, total uint64) uint64 { - totalBig := new(big.Int).SetUint64(total) - a := new(big.Int).SetUint64(nextRandom(seed) & uint64Mask) - a.Mul(a, totalBig) - a.Div(a, divider) - return a.Uint64() -} - -// SplitMix64 -// http://xoshiro.di.unimi.it/splitmix64.c -// -// The PRNG used for this random selection: -// 1. must be deterministic. -// 2. should easily portable, independent of language or library -// 3. is not necessary to keep a long period like MT, since there aren't many random numbers to generate and -// we expect a certain amount of randomness in the seed. -func nextRandom(rand *uint64) uint64 { - *rand += uint64(0x9e3779b97f4a7c15) - var z = *rand - z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9 - z = (z ^ (z >> 27)) * 0x94d049bb133111eb - return z ^ (z >> 31) -} - -func checkInvalidPriority(candidates []Candidate, totalPriority uint64) error { - actualTotalPriority := uint64(0) - for i := 0; i < len(candidates); i++ { - actualTotalPriority += candidates[i].Priority() - } - - if len(candidates) == 0 { - return fmt.Errorf("candidates is empty; "+ - "totalPriority=%d, actualTotalPriority=%d, len(candidates)=%d", - totalPriority, actualTotalPriority, len(candidates)) - - } else if totalPriority == 0 || actualTotalPriority == 0 { - return fmt.Errorf("either total priority or actual priority is zero; "+ - "totalPriority=%d, actualTotalPriority=%d, len(candidates)=%d", - totalPriority, actualTotalPriority, len(candidates)) - - } else if actualTotalPriority != totalPriority { - return fmt.Errorf("total priority not equal to actual priority; "+ - "totalPriority=%d, actualTotalPriority=%d, len(candidates)=%d", - totalPriority, actualTotalPriority, len(candidates)) - - } - return nil -} diff --git a/libs/rand/sampling_test.go b/libs/rand/sampling_test.go deleted file mode 100644 index 3fb6ffd2d..000000000 --- a/libs/rand/sampling_test.go +++ /dev/null @@ -1,209 +0,0 @@ -package rand - -import ( - "fmt" - "math" - "math/rand" - s "sort" - "testing" - - "github.com/stretchr/testify/assert" -) - -type Element struct { - id uint32 - winPoint float64 - weight uint64 - votingPower uint64 -} - -func (e *Element) Priority() uint64 { - return e.weight -} - -func (e *Element) LessThan(other Candidate) bool { - o, ok := other.(*Element) - if !ok { - panic("incompatible type") - } - return e.id < o.id -} - -func (e *Element) SetVotingPower(votingPower uint64) { - e.votingPower = votingPower -} -func (e *Element) WinPoint() float64 { return e.winPoint } -func (e *Element) VotingPower() uint64 { return e.votingPower } - -func TestRandomSamplingWithPriority(t *testing.T) { - candidates := newCandidates(100, func(i int) uint64 { return uint64(i) }) - - elected := RandomSamplingWithPriority(0, candidates, 10, calculateTotalPriority(candidates)) - if len(elected) != 10 { - t.Errorf(fmt.Sprintf("unexpected sample size: %d", len(elected))) - } - - // ---- - // The same result can be obtained for the same input. - others := newCandidates(100, func(i int) uint64 { return uint64(i) }) - secondTimeElected := RandomSamplingWithPriority(0, others, 10, calculateTotalPriority(others)) - if len(elected) != len(secondTimeElected) || !sameCandidates(elected, secondTimeElected) { - t.Errorf(fmt.Sprintf("undeterministic: %+v != %+v", elected, others)) - } - - // ---- - // Make sure the winning frequency will be even - candidates = newCandidates(100, func(i int) uint64 { return 1 }) - counts := make([]int, len(candidates)) - for i := 0; i < 100000; i++ { - elected = RandomSamplingWithPriority(uint64(i), candidates, 10, calculateTotalPriority(candidates)) - for _, e := range elected { - counts[e.(*Element).id]++ - } - } - expected := float64(1) / float64(100) - mean, variance, z := calculateZ(expected, counts) - if z >= 1e-15 || math.Abs(mean-expected) >= 1e-15 || variance >= 1e-5 { - t.Errorf("winning frequency is uneven: mean=%f, variance=%e, z=%e", mean, variance, z) - } -} - -func TestRandomSamplingPanicCase(t *testing.T) { - type Case struct { - Candidates []Candidate - TotalPriority uint64 - } - - cases := [...]*Case{ - // empty candidate set - {newCandidates(0, func(i int) uint64 { return 0 }), 0}, - // actual total priority is zero - {newCandidates(100, func(i int) uint64 { return 0 }), 100}, - // specified total priority is less than actual one - {newCandidates(2, func(i int) uint64 { return 1 }), 1000}, - } - - for i, c := range cases { - func() { - defer func() { - if recover() == nil { - t.Errorf("expected panic didn't happen in case %d", i+1) - } - }() - RandomSamplingWithPriority(0, c.Candidates, 10, c.TotalPriority) - }() - } -} - -func TestDivider(t *testing.T) { - assert.True(t, divider.Uint64() == uint64Mask+1) -} - -func TestRandomThreshold(t *testing.T) { - loopCount := 100000 - - // RandomThreshold() should not return a value greater than total. - for i := 0; i < loopCount; i++ { - seed := rand.Uint64() - total := rand.Int63() - random := RandomThreshold(&seed, uint64(total)) - assert.True(t, random < uint64(total)) - } - - // test randomness - total := math.MaxInt64 - bitHit := make([]int, 63) - for i := 0; i < loopCount; i++ { - seed := rand.Uint64() - random := RandomThreshold(&seed, uint64(total)) - for j := 0; j < 63; j++ { - if random&(1< 0 { - bitHit[j]++ - } - } - } - // all bit hit count should be near at loopCount/2 - for i := 0; i < len(bitHit); i++ { - assert.True(t, math.Abs(float64(bitHit[i])-float64(loopCount/2))/float64(loopCount/2) < 0.01) - } - - // verify idempotence - expect := [][]uint64{ - {7070836379803831726, 3176749709313725329, 6607573645926202312, 3491641484182981082, 3795411888399561855}, - {1227844342346046656, 2900311180284727168, 8193302169476290588, 2343329048962716018, 6435608444680946564}, - {1682153688901572301, 5713119979229610871, 1690050691353843586, 6615539178087966730, 965357176598405746}, - {2092789425003139052, 7803713333738082738, 391680292209432075, 3242280302033391430, 2071067388247806529}, - {7958955049054603977, 5770386275058218277, 6648532499409218539, 5505026356475271777, 3466385424369377032}} - for i := 0; i < len(expect); i++ { - seed := uint64(i) - for j := 0; j < len(expect[i]); j++ { - seed = RandomThreshold(&seed, uint64(total)) - assert.True(t, seed == expect[i][j]) - } - } -} - -func sameCandidates(c1 []Candidate, c2 []Candidate) bool { - if len(c1) != len(c2) { - return false - } - s.Slice(c1, func(i, j int) bool { return c1[i].LessThan(c1[j]) }) - s.Slice(c2, func(i, j int) bool { return c2[i].LessThan(c2[j]) }) - for i := 0; i < len(c1); i++ { - if c1[i].(*Element).id != c2[i].(*Element).id { - return false - } - if c1[i].(*Element).winPoint != c2[i].(*Element).winPoint { - return false - } - } - return true -} - -func newCandidates(length int, prio func(int) uint64) (candidates []Candidate) { - candidates = make([]Candidate, length) - for i := 0; i < length; i++ { - candidates[i] = &Element{uint32(i), 0, prio(i), 0} - } - return -} - -// The cumulative VotingPowers should follow a normal distribution with a mean as the expected value. -// A risk factor will be able to acquire from the value using a standard normal distribution table by -// applying the transformation to normalize to the expected value. -func calculateZ(expected float64, values []int) (mean, variance, z float64) { - sum := 0.0 - for i := 0; i < len(values); i++ { - sum += float64(values[i]) - } - actuals := make([]float64, len(values)) - for i := 0; i < len(values); i++ { - actuals[i] = float64(values[i]) / sum - } - mean, variance = calculateMeanAndVariance(actuals) - z = (mean - expected) / math.Sqrt(variance/float64(len(values))) - return -} - -func calculateMeanAndVariance(values []float64) (mean float64, variance float64) { - sum := 0.0 - for _, x := range values { - sum += x - } - mean = sum / float64(len(values)) - sum2 := 0.0 - for _, x := range values { - dx := x - mean - sum2 += dx * dx - } - variance = sum2 / float64(len(values)) - return -} - -func calculateTotalPriority(candidates []Candidate) uint64 { - totalPriority := uint64(0) - for _, candidate := range candidates { - totalPriority += candidate.Priority() - } - return totalPriority -} diff --git a/types/validator_set.go b/types/validator_set.go index 6bfbf4a85..c5e19da6f 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -15,7 +15,6 @@ import ( "github.com/line/ostracon/crypto/merkle" "github.com/line/ostracon/crypto/tmhash" tmmath "github.com/line/ostracon/libs/math" - tmrand "github.com/line/ostracon/libs/rand" ) const ( @@ -60,24 +59,6 @@ type ValidatorSet struct { totalVotingPower int64 } -type candidate struct { - priority uint64 - val *Validator -} - -// for implement Candidate of rand package -func (c *candidate) Priority() uint64 { - return c.priority -} - -func (c *candidate) LessThan(other tmrand.Candidate) bool { - o, ok := other.(*candidate) - if !ok { - panic("incompatible type") - } - return bytes.Compare(c.val.Address, o.val.Address) < 0 -} - // NewValidatorSet initializes a ValidatorSet by copying over the values from // `valz`, a list of Validators. If valz is nil or empty, the new ValidatorSet // will have an empty list of Validators. @@ -822,15 +803,59 @@ func (vals *ValidatorSet) SelectProposer(proofHash []byte, height int64, round i panic("empty validator set") } seed := hashToSeed(MakeRoundHash(proofHash, height, round)) - candidates := make([]tmrand.Candidate, len(vals.Validators)) - for i, val := range vals.Validators { - candidates[i] = &candidate{ - priority: uint64(val.VotingPower), - val: val, // don't need to assign the copy + random := nextRandom(&seed) + totalVotingPower := vals.TotalVotingPower() + thresholdVotingPower := dividePoint(random, totalVotingPower) + threshold := thresholdVotingPower + for _, val := range vals.Validators { + if threshold < uint64(val.VotingPower) { + return val } + threshold -= uint64(val.VotingPower) } - samples := tmrand.RandomSamplingWithPriority(seed, candidates, 1, uint64(vals.TotalVotingPower())) - return samples[0].(*candidate).val + + // This code will never be reached except in the following circumstances: + // 1) The totalVotingPower is not equal to the actual total VotingPower. + // 2) The length of vals.Validators is zero (but checked above). + // Both are due to unexpected state irregularities and can be identified by the output error message. + panic(fmt.Sprintf("Cannot select samples; r=%d, thresholdVotingPower=%d, totalVotingPower=%d: %+v", + random, thresholdVotingPower, totalVotingPower, vals)) +} + +var divider *big.Int + +func init() { + divider = new(big.Int).SetUint64(math.MaxInt64) + divider.Add(divider, big.NewInt(1)) +} + +// dividePoint computes x÷(MaxUint64+1)×y without overflow for uint64. it returns a value in the [0, y) range when x≠0. +// Otherwise returns 0. +func dividePoint(x uint64, y int64) uint64 { + totalBig := big.NewInt(y) + a := new(big.Int).SetUint64(x & math.MaxInt64) + a.Mul(a, totalBig) + a.Div(a, divider) + return a.Uint64() +} + +// nextRandom implements SplitMix64 (based on http://xoshiro.di.unimi.it/splitmix64.c) +// +// The PRNG used for this random selection: +// 1. must be deterministic. +// 2. should easily portable, independent of language or library +// 3. is not necessary to keep a long period like MT, since there aren't many random numbers to generate and +// we expect a certain amount of randomness in the seed. +// +// The shift-register type pRNG fits these requirements well, but there are too many variants. So we adopted SplitMix64, +// which is used in Java's SplittableStream. +// https://github.com/openjdk/jdk/blob/jdk-17+35/src/java.base/share/classes/java/util/SplittableRandom.java#L193-L197 +func nextRandom(rand *uint64) uint64 { + *rand += uint64(0x9e3779b97f4a7c15) + var z = *rand + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9 + z = (z ^ (z >> 27)) * 0x94d049bb133111eb + return z ^ (z >> 31) } //----------------- diff --git a/types/validator_set_test.go b/types/validator_set_test.go index ef199f12b..5278ab626 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -197,17 +197,11 @@ func verifyWinningRate(t *testing.T, vals *ValidatorSet, tries int, error float6 } } } - actual := make([]float64, len(vals.Validators)) - for i := 0; i < len(selected); i++ { - actual[i] = float64(selected[i]) / float64(tries) - } - for i := 0; i < len(actual); i++ { - expected := float64(vals.Validators[i].VotingPower) / float64(vals.TotalVotingPower()) - if math.Abs(expected-actual[i]) > expected*error { - t.Errorf("The winning rate is too far off from expected: %f ∉ %f±%f", - actual[i], expected, expected*error) - } + for i := 0; i < len(selected); i++ { + expected := float64(vals.Validators[i].VotingPower) * float64(tries) / float64(vals.TotalVotingPower()) + assert.InEpsilonf(t, expected, selected[i], error, + "The winning time %d is too far off from expected: %f", selected[i], expected) } } @@ -244,21 +238,19 @@ func TestProposerSelection2(t *testing.T) { expected := []int{0, 1, 0, 0, 2, 2, 0, 2, 1, 2, 2, 1, 2, 2, 2} for i := 0; i < len(valList)*5; i++ { prop := vals.SelectProposer([]byte{}, int64(i), 0) - if bytesToInt(prop.Address) != expected[i] { - t.Fatalf("(%d): Expected %d. Got %d", i, expected[i], bytesToInt(prop.Address)) - } + assert.Equal(t, expected[i], bytesToInt(prop.Address), i) } - verifyWinningRate(t, vals, 10000, 0.01) + verifyWinningRate(t, vals, 1000000, 0.01) // One validator has more than the others *val2 = *newValidator(addr2, 400) vals = NewValidatorSet(valList) - verifyWinningRate(t, vals, 10000, 0.01) + verifyWinningRate(t, vals, 1000000, 0.01) // One validator has more than the others *val2 = *newValidator(addr2, 401) vals = NewValidatorSet(valList) - verifyWinningRate(t, vals, 10000, 0.01) + verifyWinningRate(t, vals, 1000000, 0.01) // each validator should be the proposer a proportional number of times val0, val1, val2 = newValidator(addr0, 4), newValidator(addr1, 5), newValidator(addr2, 3) @@ -270,35 +262,13 @@ func TestProposerSelection2(t *testing.T) { prop := vals.SelectProposer([]byte{}, int64(i), 0) propCount[bytesToInt(prop.Address)]++ } - fmt.Printf("%v\n", propCount) - - if propCount[0] != 40257 { - t.Fatalf( - "Expected prop count for validator with 4/12 of voting power to be %d/%d. Got %d/%d", - 40038, - 10000*N, - propCount[0], - 10000*N, - ) - } - if propCount[1] != 50017 { - t.Fatalf( - "Expected prop count for validator with 5/12 of voting power to be %d/%d. Got %d/%d", - 50077, - 10000*N, - propCount[1], - 10000*N, - ) - } - if propCount[2] != 29726 { - t.Fatalf( - "Expected prop count for validator with 3/12 of voting power to be %d/%d. Got %d/%d", - 29885, - 10000*N, - propCount[2], - 10000*N, - ) - } + + assert.InEpsilon(t, 40000, propCount[0], 0.01, + "the number of elected times validator with 4/12 differs from expected") + assert.InEpsilon(t, 50000, propCount[1], 0.01, + "the number of elected times validator with 5/12 differs from expected") + assert.InEpsilon(t, 30000, propCount[2], 0.01, + "the number of elected times validator with 3/12 differs from expected") } func TestProposerSelection3(t *testing.T) { @@ -1641,6 +1611,19 @@ func TestValidatorSetProtoBuf(t *testing.T) { } } +func TestDividePoint(t *testing.T) { + assert.Equal(t, uint64(0), dividePoint(0, 0)) + assert.Equal(t, uint64(0), dividePoint(math.MaxUint64, 0)) + + for _, total := range [...]int64{ + 1, math.MaxUint32, math.MaxInt64 - 1, math.MaxInt64, + } { + assert.Equalf(t, uint64(0), dividePoint(0, total), "total=0x%X", total) + assert.Equalf(t, uint64(total/2), dividePoint(math.MaxInt64/2+1, total), "total=0x%X", total) + assert.Equalf(t, uint64(total-1), dividePoint(math.MaxInt64, total), "total=0x%X", total) + } +} + // --------------------- // Sort validators by priority and address type validatorsByPriority []*Validator