From 3a133ba9fb94be62c4376ed28a0b6f73781cd5da Mon Sep 17 00:00:00 2001 From: Rafi Shamim Date: Fri, 22 Dec 2023 00:11:15 -0500 Subject: [PATCH] ttljob: fix job to handle composite PKs Certain types, like decimals and collated strings, store their contents in the value of the encoded KeyValue. The ttljob code that decodes the span bounds into datums for the purpose of creating SQL query bounds previously did not take this into account. Release note (bug fix): Fixed a bug in the row-level TTL job that would cause it to skip expired rows if the primary key of the table included columns of the collated string or decimal type. --- .../replicationtestutils/encoding.go | 31 ++- pkg/sql/randgen/datum.go | 4 + pkg/sql/rowenc/BUILD.bazel | 1 + pkg/sql/rowenc/index_encoding.go | 71 ++++- pkg/sql/ttl/ttljob/ttljob_processor.go | 26 +- pkg/sql/ttl/ttljob/ttljob_processor_test.go | 251 +++++++++++++++++- pkg/sql/ttl/ttljob/ttljob_test.go | 48 +++- 7 files changed, 392 insertions(+), 40 deletions(-) diff --git a/pkg/ccl/streamingccl/replicationtestutils/encoding.go b/pkg/ccl/streamingccl/replicationtestutils/encoding.go index c4813593c3fb..cb9822df6e57 100644 --- a/pkg/ccl/streamingccl/replicationtestutils/encoding.go +++ b/pkg/ccl/streamingccl/replicationtestutils/encoding.go @@ -26,9 +26,29 @@ func EncodeKV( t *testing.T, codec keys.SQLCodec, descr catalog.TableDescriptor, pkeyVals ...interface{}, ) roachpb.KeyValue { require.Equal(t, 1, descr.NumFamilies(), "there can be only one") - primary := descr.GetPrimaryIndex() - require.LessOrEqual(t, primary.NumKeyColumns(), len(pkeyVals)) + indexEntries := encodeKVImpl(t, codec, descr, pkeyVals...) + require.Equal(t, 1, len(indexEntries)) + return roachpb.KeyValue{Key: indexEntries[0].Key, Value: indexEntries[0].Value} +} +// EncodeKVs is similar to EncodeKV, but can be used for a table with multiple +// column families, in which case up to one KV is returned per family. +func EncodeKVs( + t *testing.T, codec keys.SQLCodec, descr catalog.TableDescriptor, pkeyVals ...interface{}, +) []roachpb.KeyValue { + indexEntries := encodeKVImpl(t, codec, descr, pkeyVals...) + require.GreaterOrEqual(t, len(indexEntries), 1) + kvs := make([]roachpb.KeyValue, len(indexEntries)) + for i := range indexEntries { + kvs[i] = roachpb.KeyValue{Key: indexEntries[i].Key, Value: indexEntries[i].Value} + } + return kvs +} + +func encodeKVImpl( + t *testing.T, codec keys.SQLCodec, descr catalog.TableDescriptor, pkeyVals ...interface{}, +) []rowenc.IndexEntry { + primary := descr.GetPrimaryIndex() var datums tree.Datums var colMap catalog.TableColMap for i, val := range pkeyVals { @@ -42,9 +62,10 @@ func EncodeKV( indexEntries, err := rowenc.EncodePrimaryIndex(codec, descr, primary, colMap, datums, includeEmpty) require.NoError(t, err) - require.Equal(t, 1, len(indexEntries)) - indexEntries[0].Value.InitChecksum(indexEntries[0].Key) - return roachpb.KeyValue{Key: indexEntries[0].Key, Value: indexEntries[0].Value} + for i := range indexEntries { + indexEntries[i].Value.InitChecksum(indexEntries[i].Key) + } + return indexEntries } func nativeToDatum(t *testing.T, native interface{}) tree.Datum { diff --git a/pkg/sql/randgen/datum.go b/pkg/sql/randgen/datum.go index 742ea68b7d0c..818350f14ad6 100644 --- a/pkg/sql/randgen/datum.go +++ b/pkg/sql/randgen/datum.go @@ -553,6 +553,7 @@ var ( }, types.FloatFamily: { tree.NewDFloat(tree.DFloat(0)), + tree.NewDFloat(tree.DFloat(math.Copysign(0, -1))), // -0 tree.NewDFloat(tree.DFloat(1)), tree.NewDFloat(tree.DFloat(-1)), tree.NewDFloat(tree.DFloat(math.SmallestNonzeroFloat32)), @@ -566,9 +567,12 @@ var ( types.DecimalFamily: func() []tree.Datum { var res []tree.Datum for _, s := range []string{ + "-0", "0", "1", + "1.0", "-1", + "-1.0", "Inf", "-Inf", "NaN", diff --git a/pkg/sql/rowenc/BUILD.bazel b/pkg/sql/rowenc/BUILD.bazel index 13b7b0547b48..ecf48682638f 100644 --- a/pkg/sql/rowenc/BUILD.bazel +++ b/pkg/sql/rowenc/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "//pkg/geo/geoindex", "//pkg/geo/geopb", "//pkg/keys", + "//pkg/kv", "//pkg/roachpb", "//pkg/sql/catalog", "//pkg/sql/catalog/catalogkeys", diff --git a/pkg/sql/rowenc/index_encoding.go b/pkg/sql/rowenc/index_encoding.go index 6386bd1d6648..f1426126bb2d 100644 --- a/pkg/sql/rowenc/index_encoding.go +++ b/pkg/sql/rowenc/index_encoding.go @@ -11,6 +11,7 @@ package rowenc import ( + "bytes" "context" "sort" "unsafe" @@ -18,6 +19,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/geo/geoindex" "github.com/cockroachdb/cockroach/pkg/geo/geopb" "github.com/cockroachdb/cockroach/pkg/keys" + "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/sql/catalog" "github.com/cockroachdb/cockroach/pkg/sql/catalog/catalogkeys" @@ -425,7 +427,8 @@ func DecodeIndexKeyPrefix( } // DecodeIndexKey decodes the values that are a part of the specified index -// key (setting vals). +// key (setting vals). This function does not handle types that have composite +// encoding. See DecodeIndexKeyToDatums for a function that does. // numVals returns the number of vals populated - this can be less than // len(vals) if key ran out of bytes while populating vals. func DecodeIndexKey( @@ -445,18 +448,80 @@ func DecodeIndexKey( // DecodeIndexKeyToDatums decodes a key to tree.Datums. It is similar to // DecodeIndexKey, but eagerly decodes the []EncDatum to tree.Datums. +// Also, unlike DecodeIndexKey, this function is able to handle types +// with composite encoding. func DecodeIndexKeyToDatums( codec keys.SQLCodec, + colIDs catalog.TableColMap, types []*types.T, colDirs []catenumpb.IndexColumn_Direction, - key []byte, + keyValues []kv.KeyValue, a *tree.DatumAlloc, ) (tree.Datums, error) { + if len(keyValues) == 0 { + return nil, errors.AssertionFailedf("no key values to decode") + } vals := make([]EncDatum, len(types)) - numVals, err := DecodeIndexKey(codec, vals, colDirs, key) + numVals, err := DecodeIndexKey(codec, vals, colDirs, keyValues[0].Key) + if err != nil { + return nil, err + } + prefixLen, err := keys.GetRowPrefixLength(keyValues[0].Key) if err != nil { return nil, err } + rowPrefix := keyValues[0].Key[:prefixLen] + + // Types that have a composite encoding can their data stored in the value. + // See docs/tech-notes/encoding.md#composite-encoding for details. + for _, keyValue := range keyValues { + kvVal := keyValue.Value + + if !bytes.HasPrefix(keyValue.Key, rowPrefix) { + // This KV is not part of the same row as the start primary key. Sometimes + // a KV is omitted if all the columns in its column family are NULL. This + // could cause us to scan more KVs than needed to decode the primary index + // columns, so the slice we're iterating through might contain KVs from a + // different row at the end. + break + } + + // The composite encoding for primary index keys is always a tuple, so we + // can ignore anything else. + if kvVal == nil || kvVal.GetTag() != roachpb.ValueType_TUPLE { + continue + } + valueBytes, err := kvVal.GetTuple() + if err != nil { + return nil, err + } + + var lastColID descpb.ColumnID = 0 + for len(valueBytes) > 0 { + typeOffset, _, colIDDiff, _, err := encoding.DecodeValueTag(valueBytes) + if err != nil { + return nil, err + } + colID := lastColID + descpb.ColumnID(colIDDiff) + lastColID = colID + colOrdinal, ok := colIDs.Get(colID) + if !ok { + // This is for a column that is not in the index. We still need to + // consume the data. + _, encLen, err := encoding.PeekValueLength(valueBytes[typeOffset:]) + if err != nil { + return nil, err + } + valueBytes = valueBytes[typeOffset+encLen:] + continue + } + vals[colOrdinal], valueBytes, err = EncDatumFromBuffer(catenumpb.DatumEncoding_VALUE, valueBytes[typeOffset:]) + if err != nil { + return nil, err + } + } + } + datums := make(tree.Datums, 0, numVals) for i, encDatum := range vals[:numVals] { if err := encDatum.EnsureDecoded(types[i], a); err != nil { diff --git a/pkg/sql/ttl/ttljob/ttljob_processor.go b/pkg/sql/ttl/ttljob/ttljob_processor.go index 59c87dc19ac0..914120a8ba3b 100644 --- a/pkg/sql/ttl/ttljob/ttljob_processor.go +++ b/pkg/sql/ttl/ttljob/ttljob_processor.go @@ -98,9 +98,11 @@ func (t *ttlProcessor) work(ctx context.Context) error { var ( relationName string + pkColIDs catalog.TableColMap pkColNames []string pkColTypes []*types.T pkColDirs []catenumpb.IndexColumn_Direction + numFamilies int labelMetrics bool processorRowCount int64 ) @@ -110,6 +112,7 @@ func (t *ttlProcessor) work(ctx context.Context) error { return err } + numFamilies = desc.NumFamilies() var buf bytes.Buffer primaryIndexDesc := desc.GetPrimaryIndex().IndexDesc() pkColNames = make([]string, 0, len(primaryIndexDesc.KeyColumnNames)) @@ -123,6 +126,10 @@ func (t *ttlProcessor) work(ctx context.Context) error { return err } pkColDirs = primaryIndexDesc.KeyColumnDirections + pkColIDs = catalog.TableColMap{} + for i, id := range primaryIndexDesc.KeyColumnIDs { + pkColIDs.Set(id, i) + } if !desc.HasRowLevelTTL() { return errors.Newf("unable to find TTL on table %s", desc.GetName()) @@ -215,8 +222,10 @@ func (t *ttlProcessor) work(ctx context.Context) error { ctx, kvDB, codec, + pkColIDs, pkColTypes, pkColDirs, + numFamilies, span, &alloc, ); err != nil { @@ -400,15 +409,16 @@ func SpanToQueryBounds( ctx context.Context, kvDB *kv.DB, codec keys.SQLCodec, + pkColIDs catalog.TableColMap, pkColTypes []*types.T, pkColDirs []catenumpb.IndexColumn_Direction, + numFamilies int, span roachpb.Span, alloc *tree.DatumAlloc, ) (bounds QueryBounds, hasRows bool, _ error) { - const maxRows = 1 partialStartKey := span.Key partialEndKey := span.EndKey - startKeyValues, err := kvDB.Scan(ctx, partialStartKey, partialEndKey, maxRows) + startKeyValues, err := kvDB.Scan(ctx, partialStartKey, partialEndKey, int64(numFamilies)) if err != nil { return bounds, false, errors.Wrapf(err, "scan error startKey=%x endKey=%x", []byte(partialStartKey), []byte(partialEndKey)) } @@ -416,7 +426,7 @@ func SpanToQueryBounds( if len(startKeyValues) == 0 { return bounds, false, nil } - endKeyValues, err := kvDB.ReverseScan(ctx, partialStartKey, partialEndKey, maxRows) + endKeyValues, err := kvDB.ReverseScan(ctx, partialStartKey, partialEndKey, int64(numFamilies)) if err != nil { return bounds, false, errors.Wrapf(err, "reverse scan error startKey=%x endKey=%x", []byte(partialStartKey), []byte(partialEndKey)) } @@ -426,15 +436,13 @@ func SpanToQueryBounds( if len(endKeyValues) == 0 { return bounds, false, nil } - startKey := startKeyValues[0].Key - bounds.Start, err = rowenc.DecodeIndexKeyToDatums(codec, pkColTypes, pkColDirs, startKey, alloc) + bounds.Start, err = rowenc.DecodeIndexKeyToDatums(codec, pkColIDs, pkColTypes, pkColDirs, startKeyValues, alloc) if err != nil { - return bounds, false, errors.Wrapf(err, "decode startKey error key=%x", []byte(startKey)) + return bounds, false, errors.Wrapf(err, "decode startKeyValues error on %+v", startKeyValues) } - endKey := endKeyValues[0].Key - bounds.End, err = rowenc.DecodeIndexKeyToDatums(codec, pkColTypes, pkColDirs, endKey, alloc) + bounds.End, err = rowenc.DecodeIndexKeyToDatums(codec, pkColIDs, pkColTypes, pkColDirs, endKeyValues, alloc) if err != nil { - return bounds, false, errors.Wrapf(err, "decode endKey error key=%x", []byte(endKey)) + return bounds, false, errors.Wrapf(err, "decode endKeyValues error on %+v", endKeyValues) } return bounds, true, nil } diff --git a/pkg/sql/ttl/ttljob/ttljob_processor_test.go b/pkg/sql/ttl/ttljob/ttljob_processor_test.go index 7a6931344a7a..cb9f27cef53a 100644 --- a/pkg/sql/ttl/ttljob/ttljob_processor_test.go +++ b/pkg/sql/ttl/ttljob/ttljob_processor_test.go @@ -12,11 +12,14 @@ package ttljob_test import ( "context" "fmt" + "strings" "testing" "github.com/cockroachdb/cockroach/pkg/base" "github.com/cockroachdb/cockroach/pkg/ccl/streamingccl/replicationtestutils" + "github.com/cockroachdb/cockroach/pkg/kv" "github.com/cockroachdb/cockroach/pkg/roachpb" + "github.com/cockroachdb/cockroach/pkg/sql/catalog" "github.com/cockroachdb/cockroach/pkg/sql/catalog/desctestutils" "github.com/cockroachdb/cockroach/pkg/sql/rowenc" "github.com/cockroachdb/cockroach/pkg/sql/sem/tree" @@ -167,6 +170,10 @@ func TestSpanToQueryBounds(t *testing.T) { tableName, ) primaryIndexDesc := tableDesc.GetPrimaryIndex().IndexDesc() + pkColIDs := catalog.TableColMap{} + for i, id := range primaryIndexDesc.KeyColumnIDs { + pkColIDs.Set(id, i) + } pkColTypes, err := ttljob.GetPKColumnTypes(tableDesc, primaryIndexDesc) require.NoError(t, err) pkColDirs := primaryIndexDesc.KeyColumnDirections @@ -182,8 +189,9 @@ func TestSpanToQueryBounds(t *testing.T) { key := keyValue.Key if truncateKey { key = key[:len(key)-3] + kvKeyValues := []kv.KeyValue{{Key: key, Value: &keyValue.Value}} // Ensure truncated key cannot be decoded. - _, err = rowenc.DecodeIndexKeyToDatums(codec, pkColTypes, pkColDirs, key, &alloc) + _, err = rowenc.DecodeIndexKeyToDatums(codec, pkColIDs, pkColTypes, pkColDirs, kvKeyValues, &alloc) require.ErrorContainsf(t, err, "did not find terminator 0x0 in buffer", "pkValue=%s", pkValue) } return key @@ -194,18 +202,10 @@ func TestSpanToQueryBounds(t *testing.T) { endKey := createKey(tc.endPKValue, tc.truncateEndPKValue, primaryIndexSpan.EndKey) // Run test function. - actualBounds, actualHasRows, err := ttljob.SpanToQueryBounds( - ctx, - kvDB, - codec, - pkColTypes, - pkColDirs, - roachpb.Span{ - Key: startKey, - EndKey: endKey, - }, - &alloc, - ) + actualBounds, actualHasRows, err := ttljob.SpanToQueryBounds(ctx, kvDB, codec, pkColIDs, pkColTypes, pkColDirs, 1, roachpb.Span{ + Key: startKey, + EndKey: endKey, + }, &alloc) // Verify results. require.NoError(t, err) @@ -219,3 +219,228 @@ func TestSpanToQueryBounds(t *testing.T) { }) } } + +func TestSpanToQueryBoundsCompositeKeys(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + testCases := []struct { + desc string + // tablePKValues are PK values initially inserted into the table. + tablePKValues [][]string + // startPKValue is the PK value used to create the span start key. + startPKValue []string + // truncateStartPKValue removes end bytes from startPKValue to cause a + // decoding error. + truncateStartPKValue bool + // endPKValue is the PK value used to create the span end key. + endPKValue []string + // truncateEndPKValue removes end bytes from endPKValue to cause a + // decoding error. + truncateEndPKValue bool + expectedHasRows bool + expectedBoundsStart []string + expectedBoundsEnd []string + }{ + { + desc: "empty table", + tablePKValues: [][]string{}, + expectedHasRows: false, + }, + { + desc: "start key < table value", + tablePKValues: [][]string{{"B", "2"}}, + startPKValue: []string{"A", "1"}, + expectedHasRows: true, + expectedBoundsStart: []string{"B", "2"}, + expectedBoundsEnd: []string{"B", "2"}, + }, + { + desc: "start key = table value", + tablePKValues: [][]string{{"A", "1"}}, + startPKValue: []string{"A", "1"}, + expectedHasRows: true, + expectedBoundsStart: []string{"A", "1"}, + expectedBoundsEnd: []string{"A", "1"}, + }, + { + desc: "start key > table value", + tablePKValues: [][]string{{"A", "1"}}, + startPKValue: []string{"B", "2"}, + expectedHasRows: false, + }, + { + desc: "end key < table value", + tablePKValues: [][]string{{"B", "2"}}, + endPKValue: []string{"A", "1"}, + expectedHasRows: false, + }, + { + desc: "end key = table value", + tablePKValues: [][]string{{"A", "1"}}, + endPKValue: []string{"A", "1"}, + expectedHasRows: false, + }, + { + desc: "end key > table value", + tablePKValues: [][]string{{"A", "1"}}, + endPKValue: []string{"B", "2"}, + expectedHasRows: true, + expectedBoundsStart: []string{"A", "1"}, + expectedBoundsEnd: []string{"A", "1"}, + }, + { + desc: "start key between values", + tablePKValues: [][]string{{"A", "1"}, {"B", "2"}, {"D", "4"}, {"E", "5"}}, + startPKValue: []string{"C", "3"}, + expectedHasRows: true, + expectedBoundsStart: []string{"D", "4"}, + expectedBoundsEnd: []string{"E", "5"}, + }, + { + desc: "end key between values", + tablePKValues: [][]string{{"A", "1"}, {"B", "2"}, {"D", "4"}, {"E", "5"}}, + endPKValue: []string{"C", "3"}, + expectedHasRows: true, + expectedBoundsStart: []string{"A", "1"}, + expectedBoundsEnd: []string{"B", "2"}, + }, + { + desc: "truncated start key", + tablePKValues: [][]string{{"A", "1"}, {"B", "2"}, {"C", "3"}}, + startPKValue: []string{"B", "2"}, + truncateStartPKValue: true, + expectedHasRows: true, + expectedBoundsStart: []string{"B", "2"}, + expectedBoundsEnd: []string{"C", "3"}, + }, + { + desc: "truncated end key", + tablePKValues: [][]string{{"A", "1"}, {"B", "2"}, {"C", "3"}}, + endPKValue: []string{"B", "2"}, + truncateEndPKValue: true, + expectedHasRows: true, + expectedBoundsStart: []string{"A", "1"}, + expectedBoundsEnd: []string{"A", "1"}, + }, + } + + // Test with different column families, since this affects how the primary + // key gets encoded. + familyClauses := []string{ + "", + "FAMILY (a, b), FAMILY (c),", + "FAMILY (c), FAMILY (a, b),", + "FAMILY (a), FAMILY (b), FAMILY (c),", + } + + for _, tc := range testCases { + for _, families := range familyClauses { + t.Run(tc.desc, func(t *testing.T) { + + const tableName = "tbl" + ctx := context.Background() + srv, sqlDB, kvDB := serverutils.StartServer(t, base.TestServerArgs{}) + defer srv.Stopper().Stop(ctx) + codec := srv.ApplicationLayer().Codec() + + sqlRunner := sqlutils.MakeSQLRunner(sqlDB) + + // Create table. + sqlRunner.Exec(t, fmt.Sprintf(` + CREATE TABLE %s ( + a string, + b string COLLATE en_US_u_ks_level2, + c STRING, + %s + PRIMARY KEY(a,b) + )`, tableName, families)) + + // Insert tablePKValues into table. + if len(tc.tablePKValues) > 0 { + insertValues := "" + for i, val := range tc.tablePKValues { + if i > 0 { + insertValues += ", " + } + insertValues += "('" + strings.Join(val, "','") + "')" + } + sqlRunner.Exec(t, fmt.Sprintf("INSERT INTO %s VALUES %s", tableName, insertValues)) + } + + // Get table descriptor. + tableDesc := desctestutils.TestingGetPublicTableDescriptor( + kvDB, + codec, + "defaultdb", /* database */ + tableName, + ) + primaryIndexDesc := tableDesc.GetPrimaryIndex().IndexDesc() + pkColIDs := catalog.TableColMap{} + for i, id := range primaryIndexDesc.KeyColumnIDs { + pkColIDs.Set(id, i) + } + pkColTypes, err := ttljob.GetPKColumnTypes(tableDesc, primaryIndexDesc) + require.NoError(t, err) + pkColDirs := primaryIndexDesc.KeyColumnDirections + + var alloc tree.DatumAlloc + primaryIndexSpan := tableDesc.PrimaryIndexSpan(codec) + + createKey := func(pkValue []string, truncateKey bool, defaultKey roachpb.Key) roachpb.Key { + if len(pkValue) == 0 { + return defaultKey + } + require.Equal(t, 2, len(pkValue)) + dString := tree.NewDString(pkValue[0]) + dCollatedString, err := alloc.NewDCollatedString(pkValue[1], "en_US_u_ks_level2") + require.NoError(t, err) + + keyValues := replicationtestutils.EncodeKVs(t, codec, tableDesc, dString, dCollatedString) + key := keyValues[0].Key + if truncateKey { + key = key[:len(key)-3] + kvKeyValues := make([]kv.KeyValue, len(keyValues)) + for i := range keyValues { + kvKeyValues[i] = kv.KeyValue{Key: key, Value: &keyValues[i].Value} + } + // Ensure truncated key cannot be decoded. + _, err = rowenc.DecodeIndexKeyToDatums(codec, pkColIDs, pkColTypes, pkColDirs, kvKeyValues, &alloc) + require.ErrorContainsf(t, err, "did not find terminator 0x0 in buffer", "pkValue=%s", pkValue) + } + return key + } + + // Create keys for test. + startKey := createKey(tc.startPKValue, tc.truncateStartPKValue, primaryIndexSpan.Key) + endKey := createKey(tc.endPKValue, tc.truncateEndPKValue, primaryIndexSpan.EndKey) + + // Run test function. + actualBounds, actualHasRows, err := ttljob.SpanToQueryBounds( + ctx, kvDB, codec, pkColIDs, pkColTypes, pkColDirs, tableDesc.NumFamilies(), + roachpb.Span{ + Key: startKey, + EndKey: endKey, + }, + &alloc, + ) + + // Verify results. + require.NoError(t, err) + require.Equal(t, tc.expectedHasRows, actualHasRows) + if actualHasRows { + actualBoundsStart := []string{ + string(*actualBounds.Start[0].(*tree.DString)), + actualBounds.Start[1].(*tree.DCollatedString).Contents, + } + require.Equalf(t, tc.expectedBoundsStart, actualBoundsStart, "start") + actualBoundsEnd := []string{ + string(*actualBounds.End[0].(*tree.DString)), + actualBounds.End[1].(*tree.DCollatedString).Contents, + } + require.Equalf(t, tc.expectedBoundsEnd, actualBoundsEnd, "end") + } + }) + } + } +} diff --git a/pkg/sql/ttl/ttljob/ttljob_test.go b/pkg/sql/ttl/ttljob/ttljob_test.go index 0c862bcfec03..610c12382ba6 100644 --- a/pkg/sql/ttl/ttljob/ttljob_test.go +++ b/pkg/sql/ttl/ttljob/ttljob_test.go @@ -572,14 +572,13 @@ func TestRowLevelTTLJobRandomEntries(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) - // This test is very slow. - skip.UnderDeadlock(t) - skip.UnderRace(t) + skip.UnderDuress(t, "this test is very slow") rng, _ := randutil.NewTestRand() + collatedStringType := types.MakeCollatedString(types.String, "en" /* locale */) var indexableTyps []*types.T - for _, typ := range types.Scalar { + for _, typ := range append(types.Scalar, collatedStringType) { // TODO(#76419): DateFamily has a broken `-infinity` case. // TODO(#99432): JsonFamily has broken cases. This is because the test is wrapping JSON // objects in multiple single quotes which causes parsing errors. @@ -742,8 +741,29 @@ func TestRowLevelTTLJobRandomEntries(t *testing.T) { }, }, } - // Also randomly generate random PKs. + // Also randomly generate random PKs and families. + generateFamilyClauses := func(colNames []string) string { + familyClauses := strings.Builder{} + numFamilies := rng.Intn(len(colNames)) + for fam := 0; fam < numFamilies && len(colNames) > 0; fam++ { + rng.Shuffle(len(colNames), func(i, j int) { + colNames[i], colNames[j] = colNames[j], colNames[i] + }) + familySize := 1 + rng.Intn(len(colNames)) + familyClauses.WriteString(fmt.Sprintf("FAMILY fam%d (", fam)) + for col := 0; col < familySize; col++ { + if col > 0 { + familyClauses.WriteString(", ") + } + familyClauses.WriteString(colNames[col]) + } + colNames = colNames[familySize:] + familyClauses.WriteString("), ") + } + return familyClauses.String() + } for i := 0; i < 5; i++ { + familyClauses := generateFamilyClauses([]string{"id", "rand_col_1", "rand_col_2", "t", "i"}) testCases = append( testCases, testCase{ @@ -753,11 +773,14 @@ func TestRowLevelTTLJobRandomEntries(t *testing.T) { id UUID DEFAULT gen_random_uuid(), rand_col_1 %s, rand_col_2 %s, - text TEXT, + t TEXT NULL, + i INT8 NULL, + %s PRIMARY KEY (id, rand_col_1, rand_col_2) ) WITH (ttl_expire_after = '30 days', ttl_select_batch_size = %d, ttl_delete_batch_size = %d)`, randgen.RandTypeFromSlice(rng, indexableTyps).SQLString(), randgen.RandTypeFromSlice(rng, indexableTyps).SQLString(), + familyClauses, 1+rng.Intn(100), 1+rng.Intn(100), ), @@ -783,10 +806,15 @@ func TestRowLevelTTLJobRandomEntries(t *testing.T) { lexbase.EncodeRestrictedSQLIdent(&b, string(def.Name), lexbase.EncNoFlags) insertColumns = append(insertColumns, b.String()) - d := randgen.RandDatum(rng, def.Type.(*types.T), false /* nullOk */) - f := tree.NewFmtCtx(tree.FmtBareStrings) - d.Format(f) - values = append(values, f.CloseAndGetString()) + nullOK := def.Nullable.Nullability == tree.Null + d := randgen.RandDatum(rng, def.Type.(*types.T), nullOK) + if d == tree.DNull { + values = append(values, nil) + } else { + f := tree.NewFmtCtx(tree.FmtBareStrings) + d.Format(f) + values = append(values, f.CloseAndGetString()) + } } }