[U-Boot] [PATCH] Allow for parallel builds and saved output

[Andy Fleming]

The MAKEALL script cleverly runs make with the appropriate options
to use all of the cores on the system, but your average U-Boot build
can't make much use of more than a few cores.  If you happen to have
a many-core server, your builds will leave most of the system idle.

In order to make full use of such a system, we need to build multiple
targets in parallel, and this requires directing make output into
multiple directories. We add a BUILD_NBUILDS variable, which allows
users to specify how many builds to run in parallel. I've found that
16 is too many on my system (fork starts to fail). When BUILD_NBUILDS
is set greater than 1, we redefine BUILD_DIR for each build to be
${BUILD_DIR}/${target}. Also, we make "./build" the default BUILD_DIR
when BUILD_NBUILDS is greater than 1.

Once each build finishes, we run make clean on its directory, to reduce
the footprint, and also remove all .depend files.

As a result, we are left with a build directory with all of the built
targets still there for use, which means anyone who wanted to use
MAKEALL as part of a test harness can now do so.

Signed-off-by: Andy Fleming <afleming at freescale.com>
---
Inspired by all the MAKEALL improvements, I decided to clean up my old
one for parallel builds. I think this version addresses the concerns
raised last time...

 MAKEALL |   74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--------
 1 files changed, 64 insertions(+), 10 deletions(-)

diff --git a/MAKEALL b/MAKEALL
index 95b7cd3..4583ddb 100755
--- a/MAKEALL
+++ b/MAKEALL
@@ -31,6 +31,7 @@ usage()
 	  CROSS_COMPILE    cross-compiler toolchain prefix (default: "")
 	  MAKEALL_LOGDIR   output all logs to here (default: ./LOG/)
 	  BUILD_DIR        output build directory (default: ./)
+	  BUILD_NBUILDS	   number of parallel targets (default: 1)
 
 	Examples:
 	  - build all Power Architecture boards:
@@ -160,10 +161,23 @@ else
 	LOG_DIR="LOG"
 fi
 
+if [ ! "${BUILD_NBUILDS}" ] ; then
+	BUILD_NBUILDS="1"
+fi
+
+if [ "$BUILD_NBUILDS" -gt 1 ] ; then
+	if [ ! "${BUILD_DIR}" ] ; then
+		BUILD_DIR="./build"
+	fi
+	mkdir -p ${BUILD_DIR}/ERR
+fi
+
 if [ ! "${BUILD_DIR}" ] ; then
 	BUILD_DIR="."
 fi
 
+OUTPUT_PREFIX="${BUILD_DIR}"
+
 [ -d ${LOG_DIR} ] || mkdir ${LOG_DIR} || exit 1
 
 LIST=""
@@ -483,32 +497,52 @@ LIST_sparc="$(boards_by_arch sparc)"
 
 LIST_nds32="$(boards_by_arch nds32)"
 
+CURRENT_COUNT=0
+
 #-----------------------------------------------------------------------
 
 build_target() {
 	target=$1
 
+	if [ "$BUILD_NBUILDS" -gt 1 ] ; then
+		output_dir="${OUTPUT_PREFIX}/${target}"
+		mkdir -p ${output_dir}
+	else
+		output_dir="${OUTPUT_PREFIX}"
+	fi
+
+	export BUILD_DIR="${output_dir}"
+
 	${MAKE} distclean >/dev/null
 	${MAKE} -s ${target}_config
 
-	${MAKE} ${JOBS} all 2>&1 >${LOG_DIR}/$target.MAKELOG \
-				| tee ${LOG_DIR}/$target.ERR
+	${MAKE} ${JOBS} all \
+		2>&1 >${LOG_DIR}/$target.MAKELOG | tee ${LOG_DIR}/$target.ERR
 
 	# Check for 'make' errors
 	if [ ${PIPESTATUS[0]} -ne 0 ] ; then
 		RC=1
 	fi
 
-	if [ -s ${LOG_DIR}/$target.ERR ] ; then
-		ERR_CNT=$((ERR_CNT + 1))
-		ERR_LIST="${ERR_LIST} $target"
+	if [ "$BUILD_NBUILDS" -gt 1 ] ; then
+		${MAKE} clean
+		find "${output_dir}" -type f -name '*.depend' | xargs rm
+
+		if [ -s ${LOG_DIR}/${target}.ERR ] ; then
+			touch ${OUTPUT_PREFIX}/ERR/${target}
+		else
+			rm ${LOG_DIR}/${target}.ERR
+		fi
 	else
-		rm ${LOG_DIR}/$target.ERR
+		if [ -s ${LOG_DIR}/${target}.ERR ] ; then
+			ERR_CNT=$((ERR_CNT + 1))
+			ERR_LIST="${ERR_LIST} $target"
+		else
+			rm ${LOG_DIR}/${target}.ERR
+		fi
 	fi
 
-	TOTAL_CNT=$((TOTAL_CNT + 1))
-
-	${CROSS_COMPILE}size ${BUILD_DIR}/u-boot \
+	${CROSS_COMPILE}size ${output_dir}/u-boot \
 				| tee -a ${LOG_DIR}/$target.MAKELOG
 }
 build_targets() {
@@ -523,7 +557,20 @@ build_targets() {
 		if [ -n "${list}" ] ; then
 			build_targets ${list}
 		else
-			build_target ${t}
+			if [ "$BUILD_NBUILDS" -gt 1 ] ; then
+				build_target ${t} &
+			else
+				build_target ${t}
+			fi
+
+			TOTAL_CNT=$((TOTAL_CNT + 1))
+			CURRENT_COUNT=$((CURRENT_COUNT + 1))
+		fi
+
+		# Limit number of parallel builds
+		if [ ${CURRENT_COUNT} -gt ${BUILD_NBUILDS} ] ; then
+			CURRENT_COUNT=0;
+			wait;
 		fi
 	done
 }
@@ -531,6 +578,12 @@ build_targets() {
 #-----------------------------------------------------------------------
 
 print_stats() {
+	if [ "$BUILD_NBUILDS" -gt 1 ] ; then
+		for file in `ls ${OUTPUT_PREFIX}/ERR/`; do
+			ERR_LIST="${ERR_LIST} $file"
+		done
+		ERR_CNT=`ls -l ${OUTPUT_PREFIX}/ERR/ | wc | awk '{print $1}'`
+	fi
 	echo ""
 	echo "--------------------- SUMMARY ----------------------------"
 	echo "Boards compiled: ${TOTAL_CNT}"
@@ -549,3 +602,4 @@ set -- ${SELECTED} "$@"
 # run PowerPC by default
 [ $# = 0 ] && set -- powerpc
 build_targets "$@"
+wait
-- 
1.7.3.4