Enhance Makefile to avoid "Argument list too long"

define a macro for split long variable and redefine variable in batch. see details in `apps/Make.defs` `SPLITVARIABLE`. replace the variable reference that caused the error. Signed-off-by: xuxin19 <xuxin19@xiaomi.com>
zouboan · Aug 10, 2023 · 8e71885 · 8e71885
1 parent 6bea926
commit 8e71885
Show file tree

Hide file tree

Showing 4 changed files with 71 additions and 41 deletions.
diff --git a/Application.mk b/Application.mk
@@ -266,9 +266,12 @@ register::
 endif
 
 .depend: Makefile $(wildcard $(foreach SRC, $(SRCS), $(addsuffix /$(SRC), $(subst :, ,$(VPATH))))) $(DEPCONFIG)
-	$(Q) $(MKDEP) $(DEPPATH) --obj-suffix .c$(SUFFIX)$(OBJEXT) "$(CC)" -- $(CFLAGS) -- $(filter %.c,$^) >Make.dep
-	$(Q) $(MKDEP) $(DEPPATH) --obj-suffix .S$(SUFFIX)$(OBJEXT) "$(CC)" -- $(CFLAGS) -- $(filter %.S,$^) >>Make.dep
-	$(Q) $(MKDEP) $(DEPPATH) --obj-suffix $(CXXEXT)$(SUFFIX)$(OBJEXT) "$(CXX)" -- $(CXXFLAGS) -- $(filter %$(CXXEXT),$^) >>Make.dep
+	$(call SPLITVARIABLE,ALL_DEP_OBJS,$^,100)
+	$(foreach BATCH, $(ALL_DEP_OBJS_TOTAL), \
+	  $(shell $(MKDEP) $(DEPPATH) --obj-suffix .c$(SUFFIX)$(OBJEXT) "$(CC)" -- $(CFLAGS) -- $(filter %.c,$(ALL_DEP_OBJS_$(BATCH))) >Make.dep) \
+	  $(shell $(MKDEP) $(DEPPATH) --obj-suffix .S$(SUFFIX)$(OBJEXT) "$(CC)" -- $(CFLAGS) -- $(filter %.S,$(ALL_DEP_OBJS_$(BATCH))) >>Make.dep) \
+	  $(shell $(MKDEP) $(DEPPATH) --obj-suffix $(CXXEXT)$(SUFFIX)$(OBJEXT) "$(CXX)" -- $(CXXFLAGS) -- $(filter %$(CXXEXT),$(ALL_DEP_OBJS_$(BATCH))) >>Make.dep) \
+	) 
 	$(Q) touch $@
 
 depend:: .depend

diff --git a/Make.defs b/Make.defs
@@ -122,3 +122,29 @@ define GETINDEX
 	)
 	$(i)
 endef
+
+# SPLITVARIABLE - Split long variables into specified batches
+# Usage: $(call SPLITVARIABLE, variable-def, variable-ref, batch-size)
+#
+#	Example: VAR:= a b c x y z foo bar
+#			 $(call SPLITVARIABLE, VAR, $(VAR), 3)
+#			 $(foreach, num, $(VAR_TOTAL), $(shell echo $(VAR_$(num))))
+#	Return new variable definition:
+#		$(variable-def)_TOTAL : total split sequence , start with 1.
+#		$(variable-def)_$(num) : newly defined variable , ended with batch num.
+#	In the case above:
+#			$(VAR_TOTAL) = 1 2 3
+#			$(VAR_1) = a b c
+#			$(VAR_2) = x y z
+#			$(VAR_3) = foo bar
+
+define SPLITVARIABLE
+	$(eval PREFIX = $(1))
+	$(eval BATCH_SIZE = $(3))
+	$(eval TOTAL_BATCH = $(shell expr $(words $(2)) / $(BATCH_SIZE) + 1))
+	$(eval $(PREFIX)_TOTAL = $(shell seq 1 $(TOTAL_BATCH)))
+	$(foreach idx, $($(PREFIX)_TOTAL), \
+		$(eval FROMINDEX=$(shell expr 1 + $(idx) \* $(BATCH_SIZE) - $(BATCH_SIZE))) \
+		$(eval $(PREFIX)_$(idx)=$(wordlist $(FROMINDEX), $(shell expr $(FROMINDEX) + $(BATCH_SIZE) - 1), $(2))) \
+	)
+endef
diff --git a/builtin/Makefile b/builtin/Makefile
@@ -38,20 +38,32 @@ builtin_list.c: builtin_list.h builtin_proto.h
 registry$(DELIM).updated:
 	$(Q) touch registry$(DELIM).updated
 
+define CONFILE
+    if [ -n "$(strip $(2))" ]; then cat $(2) >> $1; fi
+endef
+
 builtin_list.h: registry$(DELIM).updated
 ifeq ($(BDATLIST),)
 	$(call DELFILE, builtin_list.h)
 	$(Q) touch builtin_list.h
 else
-	$(call CATFILE, builtin_list.h, $(BDATLIST))
+	$(shell echo '' > builtin_list.h)
+	$(call SPLITVARIABLE,BDA,$(BDATLIST),50) \
+	$(foreach BATCH, $(BDA_TOTAL), \
+	  	$(shell $(call CONFILE, builtin_list.h, $(BDA_$(BATCH)))) \
+	)
 endif
 
 builtin_proto.h: registry$(DELIM).updated
 ifeq ($(PDATLIST),)
 	$(call DELFILE, builtin_proto.h)
 	$(Q) touch builtin_proto.h
 else
-	$(call CATFILE, builtin_proto.h, $(PDATLIST))
+	$(shell echo '' > builtin_proto.h)
+	$(call SPLITVARIABLE,PDA,$(PDATLIST),50) \
+	$(foreach BATCH, $(PDA_TOTAL), \
+	  	$(shell $(call CONFILE, builtin_proto.h, $(PDA_$(BATCH)))) \
+	)
 endif
 
 depend:: builtin_list.h builtin_proto.h

diff --git a/testing/ltp/Makefile b/testing/ltp/Makefile
@@ -220,51 +220,40 @@ BLACKWORDS += shmget
 BLACKWORDS += shmat
 endif
 
-MAINWORDS    += "main("
+MAINWORDS += "main("
 
-LTP_ORIGS_1  := $(shell find $(TESTDIR) -name *.c | head -n 500)
-LTP_ORIGS_2  := $(shell find $(TESTDIR) -name *.c | head -n 1000|tail -n +501)
-LTP_ORIGS_3  := $(shell find $(TESTDIR) -name *.c | head -n 1500|tail -n +1001)
-LTP_ORIGS_4  := $(shell find $(TESTDIR) -name *.c | head -n 2000|tail -n +1501)
-
-$(foreach word, $(BLACKWORDS), $(eval BLACKLIST+=$(shell grep -lr $(word) $(LTP_ORIGS_1))))
-$(foreach src, $(BLACKSRCS), $(eval BLACKLIST+=$(filter %$(src),$(LTP_ORIGS_1))))
-
-$(foreach word, $(BLACKWORDS), $(eval BLACKLIST+=$(shell grep -lr $(word) $(LTP_ORIGS_2))))
-$(foreach src, $(BLACKSRCS), $(eval BLACKLIST+=$(filter %$(src),$(LTP_ORIGS_2))))
-
-$(foreach word, $(BLACKWORDS), $(eval BLACKLIST+=$(shell grep -lr $(word) $(LTP_ORIGS_3))))
-$(foreach src, $(BLACKSRCS), $(eval BLACKLIST+=$(filter %$(src),$(LTP_ORIGS_3))))
-
-$(foreach word, $(BLACKWORDS), $(eval BLACKLIST+=$(shell grep -lr $(word) $(LTP_ORIGS_4))))
-$(foreach src, $(BLACKSRCS), $(eval BLACKLIST+=$(filter %$(src),$(LTP_ORIGS_4))))
-
-LTP_ORIGS_1  := $(filter-out $(BLACKLIST), $(LTP_ORIGS_1))
-LTP_ORIGS_2  := $(filter-out $(BLACKLIST), $(LTP_ORIGS_2))
-LTP_ORIGS_3  := $(filter-out $(BLACKLIST), $(LTP_ORIGS_3))
-LTP_ORIGS_4  := $(filter-out $(BLACKLIST), $(LTP_ORIGS_4))
+LTP_ORIGS := $(shell find $(TESTDIR) -name *.c)
+ifneq ($(LTP_ORIGS),)
+$(call SPLITVARIABLE,ORIGS_SPILT,$(LTP_ORIGS),200)
+$(foreach BATCH, $(ORIGS_SPILT_TOTAL), \
+	$(foreach word, $(BLACKWORDS), $(eval BLACKLIST+=$(shell grep -lr $(word) $(ORIGS_SPILT_$(BATCH))))) \
+)
+endif
 
-$(foreach word, $(MAINWORDS), $(eval LTP_MAINCSRCS_1+=$(shell grep -lr $(word) $(LTP_ORIGS_1))))
-$(foreach word, $(MAINWORDS), $(eval LTP_MAINCSRCS_2+=$(shell grep -lr $(word) $(LTP_ORIGS_2))))
-$(foreach word, $(MAINWORDS), $(eval LTP_MAINCSRCS_3+=$(shell grep -lr $(word) $(LTP_ORIGS_3))))
-$(foreach word, $(MAINWORDS), $(eval LTP_MAINCSRCS_4+=$(shell grep -lr $(word) $(LTP_ORIGS_4))))
+$(foreach src, $(BLACKSRCS), $(eval BLACKLIST+=$(filter %$(src),$(LTP_ORIGS))))
 
-LTP_CSRCS_1  := $(filter-out $(LTP_MAINCSRCS_1), $(LTP_ORIGS_1))
-LTP_CSRCS_2  := $(filter-out $(LTP_MAINCSRCS_2), $(LTP_ORIGS_2))
-LTP_CSRCS_3  := $(filter-out $(LTP_MAINCSRCS_3), $(LTP_ORIGS_3))
-LTP_CSRCS_4  := $(filter-out $(LTP_MAINCSRCS_4), $(LTP_ORIGS_4))
+LTP_ORIGS := $(filter-out $(BLACKLIST), $(LTP_ORIGS))
+ifneq ($(LTP_ORIGS),)
+$(call SPLITVARIABLE,ORIGS_SPILT,$(LTP_ORIGS),200)
+$(foreach BATCH, $(ORIGS_SPILT_TOTAL), \
+	$(foreach word, $(MAINWORDS), $(eval LTP_MAINCSRCS+=$(shell grep -lr $(word) $(ORIGS_SPILT_$(BATCH))))) \
+)
+endif
 
+LTP_CSRCS := $(filter-out $(LTP_MAINCSRCS), $(LTP_ORIGS))
+ifneq ($(LTP_MAINCSRCS),)
+$(call SPLITVARIABLE,MAINCSRC_SPILT,$(LTP_MAINCSRCS),50)
+$(foreach BATCH, $(MAINCSRC_SPILT_TOTAL), \
+	$(eval PROGNAME+=$(basename $(shell echo $(MAINCSRC_SPILT_$(BATCH)) | xargs -n 1 | awk -F "[/]" '{print "ltp_"$$(NF-2)"_"$$(NF-1)"_"$$(NF)}' | sed s/-/_/g))) \
+)
+endif
 
-PROGNAME     := $(basename $(shell echo $(LTP_MAINCSRCS_1) | xargs -n 1 | awk -F "[/]" '{print "ltp_"$$(NF-2)"_"$$(NF-1)"_"$$(NF)}' | sed s/-/_/g))
-PROGNAME     += $(basename $(shell echo $(LTP_MAINCSRCS_2) | xargs -n 1 | awk -F "[/]" '{print "ltp_"$$(NF-2)"_"$$(NF-1)"_"$$(NF)}' | sed s/-/_/g))
-PROGNAME     += $(basename $(shell echo $(LTP_MAINCSRCS_3) | xargs -n 1 | awk -F "[/]" '{print "ltp_"$$(NF-2)"_"$$(NF-1)"_"$$(NF)}' | sed s/-/_/g))
-PROGNAME     += $(basename $(shell echo $(LTP_MAINCSRCS_4) | xargs -n 1 | awk -F "[/]" '{print "ltp_"$$(NF-2)"_"$$(NF-1)"_"$$(NF)}' | sed s/-/_/g))
-MAINSRC       = $(LTP_MAINCSRCS_1) $(LTP_MAINCSRCS_2) $(LTP_MAINCSRCS_3) $(LTP_MAINCSRCS_4)
+MAINSRC       = $(LTP_MAINCSRCS)
 PRIORITY      = SCHED_PRIORITY_DEFAULT
 STACKSIZE     = $(CONFIG_DEFAULT_TASK_STACKSIZE)
 MODULE        = $(CONFIG_TESTING_LTP)
 
-CSRCS        := $(LTP_CSRCS_1) $(LTP_CSRCS_2) $(LTP_CSRCS_3) $(LTP_CSRCS_4)
+CSRCS        := $(LTP_CSRCS)
 CFLAGS       += -I$(CURDIR)
 CFLAGS       += -I$(TESTDIR)/include
 endif