perf(ci): bring every PR job under 10 minutes

.github/workflows/build.yml

@@ -148,7 +148,8 @@ jobs:
             sudo apt-get install --fix-missing gcc-multilib
 
       - name: Build
-        run: make build
+        # 只需要验证 386 下能编译: build_only 跳过 depends 里几十个 dapp cmd 产物的编译(约省 70s)
+        run: make build_only
 
       - name: Run unit tests
         run: go test ./... -covermode=atomic

.github/workflows/build_evm.yml

@@ -15,5 +15,6 @@ jobs:
           go-version-file: 'go.mod'
       - name: deploy
         run: |
+          make build_ci_dapp dapp=evm
           make docker-compose dapp=evm
           make docker-compose-down dapp=evm

.github/workflows/build_mix.yml

@@ -28,6 +28,6 @@ jobs:
           cd -
       - name: deploy
         run: |
-          make build_ci
+          make build_ci_dapp dapp=mix
           make docker-compose dapp=mix
           make docker-compose-down dapp=mix

.github/workflows/build_paracross.yml

@@ -20,6 +20,10 @@ jobs:
         run: export PATH=${PATH}:`go env GOPATH`/bin
 
       - name: deploy
+        # 注意: 不能换成 build_ci_dapp。paracross 的 docker-compose.sh 会走 rpc_test ->
+        # dapp_test_rpc, 后者固定执行 ./ticket/test-rpc.sh 以及 build/ci/dapptest 下所有
+        # dapp 的 test-rpc.sh(package 脚本由各 dapp 的 cmd/build.sh 生成), 因此仍然需要
+        # depends 的完整 build/ci 产物; 只编 paracross 会导致 ticket/test-rpc.sh 缺失而失败。
         run: |
           make docker-compose dapp=paracross
           make docker-compose-down dapp=paracross

.github/workflows/build_relay.yml

@@ -34,7 +34,7 @@ jobs:
 
       - name: deploy
         run: |
-          make build_ci
+          make build_ci_dapp dapp=relay
           make docker-compose dapp=relay
 
       - name: cleanup

.github/workflows/build_rgbx.yml

@@ -19,6 +19,7 @@ jobs:
         id: go
       - name: deploy
         run: |
+          make build_ci_dapp dapp=rgbx
           make docker-compose dapp=rgbx
       - name: cleanup
         if: always()

.github/workflows/ci_parachain_rollup.yml

@@ -15,5 +15,6 @@ jobs:
           go-version-file: 'go.mod'
       - name: deploy
         run: |
+          make build_ci_dapp dapp=rollup
           make docker-compose dapp=rollup
           make docker-compose-down dapp=rollup

Makefile

@@ -19,14 +19,20 @@ BUILD_FLAGS := -ldflags '-X "github.com/33cn/plugin/version.GitCommit=$(GitCommi
 MKPATH=$(abspath $(lastword $(MAKEFILE_LIST)))
 MKDIR=$(dir $(MKPATH))
 proj := "build"
-.PHONY: default dep all build release cli linter race test fmt vet bench msan coverage coverhtml docker docker-compose protobuf clean help autotest
+.PHONY: default dep all build build_only release cli linter race test fmt vet bench msan coverage coverhtml docker docker-compose protobuf clean help autotest
 
 default: depends build
 
-build: depends
+build: depends build_only
+
+# 只编 chain33/chain33-cli/fork-config, 不跑 depends(不生成 build/ci 下各 dapp 的集成测试产物)。
+# 用于只需要编译产物、不做 docker 集成测试的场景, 例如 GOARCH 386 的构建检查:
+# depends 会串行编译几十个 dapp 的 cmd 产物, 对这类 job 是纯开销。
+build_only: ## Build chain33 binaries without dapp cmd artifacts (no depends)
 	go build $(BUILD_FLAGS) -v -o $(APP)
 	go build $(BUILD_FLAGS) -v -o $(CLI) $(SRC_CLI)
 	go build $(BUILD_FLAGS) -v -o build/fork-config github.com/33cn/plugin/cli/fork_config/
+	@mkdir -p build/ci/paracross
 	@cp chain33.toml build/
 	@cp chain33.para.toml build/ci/paracross/
 
@@ -236,7 +242,9 @@ protobuf: ## Generate protbuf file of types package
 	@find ./plugin/dapp -maxdepth 2 -type d  -name proto -exec make -C {} \;
 
 depends: ## Generate depends file of types package
-	@find ./plugin/dapp -maxdepth 2 -print -type d  -name cmd -exec make -C {} OUT="$(MKDIR)build/ci" FLAG= \;
+	@# 各 dapp 的 cmd/build.sh 只写自己的 build/ci/<dapp> 和 build/ci/dapptest/<dapp>, 互不重叠,
+	@# 可以并行; 原来 -exec ... \; 是串行, 几十个 dapp 合起来要 50~70s。
+	@find ./plugin/dapp -maxdepth 2 -type d -name cmd | xargs -P "$${DEPENDS_JOBS:-4}" -I{} make -C {} OUT="$(MKDIR)build/ci" FLAG=
 
 
 help: ## Display this help screen

build/tools/coverage.sh

@@ -3,21 +3,17 @@
 # Code coverage generation
 set -e -o pipefail
 
-COVERAGE_DIR="${COVERAGE_DIR:-build/coverage}"
+# go test 从 1.20 起支持一次对多个包生成**合并**的覆盖率文件, 并且会自己按 -p 并行跑包。
+# 原来用 bash for 循环逐包串行执行: 全量 126 个有测试的包, 单个最慢约 2 分钟,
+# 串行合计约 16 分钟(CI 实测 ut coverage 步骤 991s)。改成一次调用后,
+# 墙钟时间降到"最慢的包 + 排队"的量级。
+JOBS="${COVERAGE_JOBS:-$(nproc 2>/dev/null || echo 4)}"
 
 PKG_LIST=$(go list ./... | grep -v "vendor" | grep -v "chain33/test" | grep -v "mock" | grep -v "mocks")
 
-# Create the coverage files directory
-mkdir -p "$COVERAGE_DIR"
-
-# Create a coverage file for each package
-for package in ${PKG_LIST}; do
-    go test -parallel=8 -covermode=count -coverprofile "${COVERAGE_DIR}/${package##*/}.cov" "$package"
-done
-
-# Merge the coverage profile files
-echo 'mode: count' >./coverage.cov
-tail -q -n +2 "${COVERAGE_DIR}"/*.cov >>./coverage.cov
+# PKG_LIST 需要按空白拆分成多个参数, 这里刻意不加引号
+# shellcheck disable=SC2086
+go test -p "${JOBS}" -parallel=8 -covermode=count -coverprofile=./coverage.cov ${PKG_LIST}
 
 # Display the global code coverage
 go tool cover -func=./coverage.cov
@@ -26,6 +22,3 @@ go tool cover -func=./coverage.cov
 if [ "$1" == "html" ]; then
     go tool cover -html=./coverage.cov -o coverage.html
 fi
-
-# Remove the coverage files directory
-rm -rf "$COVERAGE_DIR"

plugin/dapp/evmxgo/cmd/build.sh

@@ -8,4 +8,8 @@ OUT_DIR="${1}/$strapp"
 #FLAG=$2
 
 mkdir -p "${OUT_DIR}"
-cp ./build/* "${OUT_DIR}"
+# 本 dapp 没有 ./build 目录(历史遗留), 原来的 cp 必然失败; depends 以前用
+# `find -exec` 会静默吞掉失败, 现在 depends 并行化并用 xargs 透出失败, 这里显式判断
+if [ -d ./build ]; then
+    cp ./build/* "${OUT_DIR}"
+fi

plugin/dapp/lightclient/cmd/build.sh

@@ -8,4 +8,8 @@ OUT_DIR="${1}/$strapp"
 #FLAG=$2
 
 mkdir -p "${OUT_DIR}"
-cp ./build/* "${OUT_DIR}"
+# 本 dapp 没有 ./build 目录(历史遗留), 原来的 cp 必然失败; depends 以前用
+# `find -exec` 会静默吞掉失败, 现在 depends 并行化并用 xargs 透出失败, 这里显式判断
+if [ -d ./build ]; then
+    cp ./build/* "${OUT_DIR}"
+fi