add limit for how many parallel schema changes we can have on a parti…

…tion

Signed-off-by: Dorin Hogea <dhogea@bloomberg.net>

db/comdb2.h

@@ -1863,6 +1863,7 @@ extern int gbl_dohsql_max_queued_kb_highwm;
 extern int gbl_dohsql_full_queue_poll_msec;
 extern int gbl_dohsql_max_threads;
 extern int gbl_dohsql_pool_thr_slack;
+extern int gbl_dohsql_sc_max_threads;
 extern int gbl_sockbplog;
 extern int gbl_sockbplog_sockpool;
 

db/db_tunables.h

@@ -1802,6 +1802,11 @@ REGISTER_TUNABLE(
     TUNABLE_INTEGER, &gbl_dohsql_full_queue_poll_msec, 0, NULL, NULL, NULL,
     NULL);
 
+REGISTER_TUNABLE(
+    "dohsql_sc_max_threads",
+    "If the partition has more shards than this, we run one shard at a time.",
+    TUNABLE_INTEGER, &gbl_dohsql_sc_max_threads, 8, NULL, NULL, NULL, NULL);
+
 REGISTER_TUNABLE("random_fail_client_write_lock",
                  "Force a random client write-lock failure 1/this many times.  "
                  "(Default: 0)",

db/dohsql.c

@@ -32,8 +32,9 @@ int gbl_dohsql_disable = 0;
 int gbl_dohsql_verbose = 0;
 int gbl_dohsql_max_queued_kb_highwm = 10000;    /* 10 MB */
 int gbl_dohsql_full_queue_poll_msec = 10;       /* 10msec */
-int gbl_dohsql_max_threads = 8; /* do not run more than 8 threads */
+int gbl_dohsql_max_threads = 8; /* do not run more than 8 parallel shards */
 int gbl_dohsql_pool_thr_slack = 24; /* half default sqlengine pool maxthds */
+int gbl_dohsql_sc_max_threads = 8; /* do not run more than 8 parallel sc-s */
 /* for now we keep this tunning "private */
 static int gbl_dohsql_track_stats = 1;
 static int gbl_dohsql_que_free_highwm = 10;

db/osqlcomm.c

@@ -5901,6 +5901,7 @@ static int _process_partitioned_table_merge(struct ireq *iq)
     timepart_sc_arg_t arg = {0};
     arg.s = sc;
     arg.s->iq = iq;
+    /* note: we have already set nothrevent depending on the number of shards */
     rc = timepart_foreach_shard(
         sc->tablename, start_schema_change_tran_wrapper_merge, &arg, 0);
     return rc;
@@ -5988,6 +5989,8 @@ static int _process_single_table_sc_partitioning(struct ireq *iq)
         arg.s->kind = SC_ADDTABLE;
         arg.indx = 1; /* first shard is already there */
     }
+    /* should we serialize ? */
+    arg.s->nothrevent = sc->partition.u.tpt.retention > gbl_dohsql_sc_max_threads;
     rc = timepart_foreach_shard_lockless(
             sc->newpartition, start_schema_change_tran_wrapper, &arg);
 
@@ -6053,6 +6056,19 @@ static int _process_partition_alter_and_drop(struct ireq *iq)
         goto out;
     }
 
+    int nshards = timepart_get_num_shards(sc->tablename);
+    if (nshards <= 0) {
+        /*somehow the time partition got away from us */
+        logmsg(LOGMSG_ERROR, "Failed to retrieve nshards in sc for %s\n",
+               sc->tablename);
+        sc_errf(sc, "Failed to retrieve nshards in sc for %s",
+               sc->tablename);
+        return ERR_SC;
+    }
+
+    /* should we serialize ? */
+    sc->nothrevent = nshards > gbl_dohsql_sc_max_threads;
+
     if (sc->partition.type == PARTITION_MERGE) {
         return _process_partitioned_table_merge(iq);
     }

docs/pages/config/config_files.md

@@ -108,6 +108,7 @@ Settings:
 |dohsql_max_queued_kb_highwm | 10000 | Maximum shard queue size, in KB; throttles amount of cached rows by each parallel component
 |dohsql_max_threads | 8 | Allow only up to 8 parallel components. If more are required, statement runs sequential
 |dohsql_pool_thread_slack | 1 | Reserve a number of sql engines to run only non-parallel load (including parallel components).  
+|dohsql_sc_max_threads | 8 | Allow only up to 8 parallel schema changes. If more are required, they runs sequential
 
 
 ### Networks

tests/tunables.test/t00_all_tunables.expected

@@ -247,6 +247,7 @@
 (name='dohsql_max_queued_kb_highwm', description='Maximum shard queue size, in KB; shard sqlite will pause once queued bytes limit is reached.', type='INTEGER', value='10000', read_only='N')
 (name='dohsql_max_threads', description='Maximum number of parallel threads, otherwise run sequential.', type='INTEGER', value='8', read_only='N')
 (name='dohsql_pool_thread_slack', description='Forbid parallel sql coordinators from running on this many sql engines (if 0, defaults to 24).', type='INTEGER', value='24', read_only='N')
+(name='dohsql_sc_max_threads', description='If the partition has more shards than this, we run one shard at a time.', type='INTEGER', value='8', read_only='N')
 (name='dohsql_verbose', description='Run distributed queries in verbose/debug mode', type='BOOLEAN', value='OFF', read_only='N')
 (name='dont_abort_on_in_use_rqid', description='Disable 'abort_on_in_use_rqid'', type='BOOLEAN', value='OFF', read_only='Y')
 (name='dont_block_delete_files_thread', description='Ignore files that would block delete-files thread.  (Default: off)', type='BOOLEAN', value='OFF', read_only='N')