DDL Ingest fails due to missing tmp_ddl-4000 directory

[dveeden]

Bug Report

1. Minimal reproduce step (Required)

Run DDL

2. What did you expect to see? (Required)

Functioning DDL

3. What did you see instead (Required)

{
  "level": "ERROR",
  "time": "2023/07/18 09:33:51.968 +00:00",
  "caller": "ddl_worker.go:942",
  "message": "[ddl] run DDL job error",
  "worker": "worker 5, tp add index",
  "error": "cannot get disk capacity at /tmp/tidb/tmp_ddl-4000: no such file or directory",
  "errorVerbose": "no such file or directory\ncannot get disk capacity at /tmp/tidb/tmp_ddl-3306\ngithub.com/pingcap/tidb/br/pkg/lightning/common.GetStorageSize\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/br/pkg/lightning/common/storage_unix.go:40\ngithub.com/pingcap/tidb/ddl/ingest.(*diskRootImpl).PreCheckUsage\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/ingest/disk_root.go:110\ngithub.com/pingcap/tidb/ddl/ingest.(*litBackendCtxMgr).CheckAvailable\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/ingest/backend_mgr.go:73\ngithub.com/pingcap/tidb/ddl.pickBackfillType\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/index.go:726\ngithub.com/pingcap/tidb/ddl.(*worker).onCreateIndex\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/index.go:627\ngithub.com/pingcap/tidb/ddl.(*worker).runDDLJob\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/ddl_worker.go:1051\ngithub.com/pingcap/tidb/ddl.(*worker).HandleDDLJobTable\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/ddl_worker.go:772\ngithub.com/pingcap/tidb/ddl.(*ddl).delivery2worker.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/job_table.go:414\ngithub.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1\n\t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_arm64.s:1172"
}

Formatted errorVerbose:

no such file or directory
cannot get disk capacity at /tmp/tidb/tmp_ddl-3306
github.com/pingcap/tidb/br/pkg/lightning/common.GetStorageSize
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/br/pkg/lightning/common/storage_unix.go:40
github.com/pingcap/tidb/ddl/ingest.(*diskRootImpl).PreCheckUsage
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/ingest/disk_root.go:110
github.com/pingcap/tidb/ddl/ingest.(*litBackendCtxMgr).CheckAvailable
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/ingest/backend_mgr.go:73
github.com/pingcap/tidb/ddl.pickBackfillType
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/index.go:726
github.com/pingcap/tidb/ddl.(*worker).onCreateIndex
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/index.go:627
github.com/pingcap/tidb/ddl.(*worker).runDDLJob
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/ddl_worker.go:1051
github.com/pingcap/tidb/ddl.(*worker).HandleDDLJobTable
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/ddl_worker.go:772
github.com/pingcap/tidb/ddl.(*ddl).delivery2worker.func1
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/ddl/job_table.go:414
github.com/pingcap/tidb/util.(*WaitGroupWrapper).Run.func1
	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/util/wait_group_wrapper.go:154
runtime.goexit
	/usr/local/go/src/runtime/asm_arm64.s:1172

4. What is your TiDB version? (Required)

v7.1.0

[dveeden]

tidb/ddl/ingest/env.go

Lines 78 to 101 in dfa03cd

	// Generate lightning local store dir in TiDB data dir.
	// it will append -port to be tmp_ddl suffix.
	func genLightningDataDir() (string, error) {
	sortPath := ConfigSortPath()
	if _, err := os.Stat(sortPath); err != nil {
	if !os.IsNotExist(err) {
	logutil.BgLogger().Error(LitErrStatDirFail,
	zap.String("category", "ddl-ingest"),
	zap.String("sort path", sortPath), zap.Error(err))
	return "", err
	}
	}
	err := os.MkdirAll(sortPath, 0o700)
	if err != nil {
	logutil.BgLogger().Error(LitErrCreateDirFail,
	zap.String("category", "ddl-ingest"),
	zap.String("sort path", sortPath), zap.Error(err))
	return "", err
	}
	logutil.BgLogger().Info(LitInfoSortDir,
	zap.String("category", "ddl-ingest"),
	zap.String("data path:", sortPath))
	return sortPath, nil
	}

So genLightningDataDir() should create this directory.. but that doesn't seem to work somehow.

[tangenta]

Close by #45457.