feat: run firmware builds in parallel

Jenkinsfile

@@ -72,7 +72,8 @@ pipeline {
                     ]) {
                         sh '''
                         make download-emissary-release
-                        make all
+                        MAX_LOAD_AVERAGE=$(( $(nproc --all ) / 2 ))
+                        make -j -l $MAX_LOAD_AVERAGE all
                         '''
                     }
                 }

Makefile

@@ -19,6 +19,7 @@ IMAGEBUILDER_ARCHIVE_PATH ?= tmp/imagebuilder-$(OPENWRT_VERSION)-$(OPENWRT_TARGE
 IMAGEBUILDER_DIR_PATH ?= $(PWD)/imagebuilder/$(OPENWRT_VERSION)/$(OPENWRT_TARGET)
 IMAGEBUILDER_CUSTOM_PACKAGES_DIR_PATH := $(IMAGEBUILDER_DIR_PATH)/packages
 IMAGEBUILDER_CUSTOM_FILES_DIR_PATH := $(IMAGEBUILDER_DIR_PATH)/files
+INSTALL_DIR := files/$(OPENWRT_VERSION)/$(OPENWRT_TARGET)/$(OPENWRT_PROFILE)
 
 BIN_DIR := "$(shell readlink -f bin)/$(OPENWRT_VERSION)/$(OPENWRT_TARGET)/$(OPENWRT_PROFILE)$(BIN_DIR_NAME_SUFFIX)"
 
@@ -81,10 +82,11 @@ $(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH):
 	echo "FIRMWARE_VERSION=$(FULL_VERSION)" >> "$(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH)/etc/emissary_firmware"
 	echo "OPENWRT_PROFILE=$(OPENWRT_PROFILE)" >> "$(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH)/etc/emissary_firmware"
 
-	rm -rf files/*
+	rm -rf $(INSTALL_DIR)/*
+	mkdir -p "$(INSTALL_DIR)"
 	$(MAKE) $(BASE_INSTALL) $(ADDITIONAL_INSTALL)
 
-	cp -rf files/* $(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH)/
+	cp -rf $(INSTALL_DIR)/* $(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH)/
 
 flash:
 	OPENWRT_DEVICE=$(OPENWRT_DEVICE) OPENWRT_PROFILE=$(OPENWRT_PROFILE) OPENWRT_VERSION=$(OPENWRT_VERSION) misc/script/flash.sh

install/bananapi.mk

@@ -1,3 +1,3 @@
 install-bpi-r3-network-config:
-	mkdir -p files/etc/config
-	cp misc/bpi-r3/uci/network files/etc/config/network
+	mkdir -p "$(INSTALL_DIR)/etc/config"
+	cp misc/bpi-r3/uci/network "$(INSTALL_DIR)/etc/config/network"

install/common.mk

@@ -1,6 +1,6 @@
 install-common-uci-defaults:
-	mkdir -p files/etc/uci-defaults
-	cp misc/common/uci-defaults/* files/etc/uci-defaults/
+	mkdir -p "$(INSTALL_DIR)/etc/uci-defaults"
+	cp misc/common/uci-defaults/* "$(INSTALL_DIR)/etc/uci-defaults/"
 
-install-common-additional-agent-collectors: tools/yq/bin/yq
-	tools/yq/bin/yq -i '.agent.collectors += load("misc/common/agent/collectors.yml")' files/etc/emissary/agent.yml
+install-common-additional-agent-collectors: tools/yq/bin/yq install-emissary-files
+	tools/yq/bin/yq -i '.agent.collectors += load("misc/common/agent/collectors.yml")' "$(INSTALL_DIR)/etc/emissary/agent.yml"

install/emissary-files.mk

@@ -1,49 +1,49 @@
 install-emissary-files: tools/yq/bin/yq tools/upx/bin/upx
-	mkdir -p files/etc/config
-	cp -r misc/emissary/config/* files/etc/config/
+	mkdir -p "$(INSTALL_DIR)/etc/config"
+	cp -r misc/emissary/config/* "$(INSTALL_DIR)/etc/config/"
 
-	mkdir -p files/etc/init.d
-	cp -r misc/emissary/init.d/* files/etc/init.d/
+	mkdir -p "$(INSTALL_DIR)/etc/init.d"
+	cp -r misc/emissary/init.d/* "$(INSTALL_DIR)/etc/init.d/"
 
-	mkdir -p files/etc/uci-defaults
-	cp -r misc/emissary/uci-defaults/* files/etc/uci-defaults/
+	mkdir -p "$(INSTALL_DIR)/etc/uci-defaults"
+	cp -r misc/emissary/uci-defaults/* "$(INSTALL_DIR)/etc/uci-defaults/"
 
 	# Copy keep.d files
-	mkdir -p files/lib/upgrade/keep.d
-	cp -r misc/emissary/keep.d/* files/lib/upgrade/keep.d/
+	mkdir -p "$(INSTALL_DIR)/lib/upgrade/keep.d"
+	cp -r misc/emissary/keep.d/* "$(INSTALL_DIR)/lib/upgrade/keep.d/"
 
 	# Copy profile.d files
-	mkdir -p files/etc/profile.d
-	cp -r misc/emissary/profile.d/* files/etc/profile.d/
+	mkdir -p "$(INSTALL_DIR)/etc/profile.d"
+	cp -r misc/emissary/profile.d/* "$(INSTALL_DIR)/etc/profile.d/"
 
 	mkdir -p gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH)
 	cd gitea-dl && tar -xzf emissary-agent_*_linux_$(EMISSARY_ARCH).tar.gz -C emissary-agent_linux_$(EMISSARY_ARCH)
 	
 	# Copy agent config
-	mkdir -p files/etc/emissary
-	cp gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH)/misc/packaging/common/config-agent.yml files/etc/emissary/agent.yml
+	mkdir -p "$(INSTALL_DIR)/etc/emissary"
+	cp gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH)/misc/packaging/common/config-agent.yml "$(INSTALL_DIR)/etc/emissary/agent.yml"
 
 	# Patch agent config
-	tools/yq/bin/yq -i '.agent.serverUrl = "$${EMISSARY_SERVER_URL}"' files/etc/emissary/agent.yml
-	tools/yq/bin/yq -i '.agent.reconciliationInterval = "$${EMISSARY_RECONCILIATION_INTERVAL}"' files/etc/emissary/agent.yml
-	tools/yq/bin/yq -i '.agent.privateKeyPath = "/data/emissary/agent-key.json"' files/etc/emissary/agent.yml
-	tools/yq/bin/yq -i '.agent.controllers.persistence.stateFile = "/data/emissary/agent-state.json"' files/etc/emissary/agent.yml
-	tools/yq/bin/yq -i '.agent.controllers.sysupgrade.firmwareVersionCommand = ["sh", "-c", "source /etc/emissary_firmware && echo \"$$FIRMWARE_VERSION\""]' files/etc/emissary/agent.yml
-	tools/yq/bin/yq -i '.agent.controllers.app.dataDir = "/data/emissary/apps/data"' files/etc/emissary/agent.yml
-	tools/yq/bin/yq -i '.agent.controllers.app.downloadDir = "/data/emissary/apps/bundles"' files/etc/emissary/agent.yml
-	tools/yq/bin/yq -i '.sentry.dsn = "$${EMISSARY_SENTRY_DSN}"' files/etc/emissary/agent.yml
-	tools/yq/bin/yq -i '.sentry.environment = "$${EMISSARY_SENTRY_ENVIRONMENT}"' files/etc/emissary/agent.yml
+	tools/yq/bin/yq -i '.agent.serverUrl = "$${EMISSARY_SERVER_URL}"' "$(INSTALL_DIR)/etc/emissary/agent.yml"
+	tools/yq/bin/yq -i '.agent.reconciliationInterval = "$${EMISSARY_RECONCILIATION_INTERVAL}"' "$(INSTALL_DIR)/etc/emissary/agent.yml"
+	tools/yq/bin/yq -i '.agent.privateKeyPath = "/data/emissary/agent-key.json"' "$(INSTALL_DIR)/etc/emissary/agent.yml"
+	tools/yq/bin/yq -i '.agent.controllers.persistence.stateFile = "/data/emissary/agent-state.json"' "$(INSTALL_DIR)/etc/emissary/agent.yml"
+	tools/yq/bin/yq -i '.agent.controllers.sysupgrade.firmwareVersionCommand = ["sh", "-c", "source /etc/emissary_firmware && echo \"$$FIRMWARE_VERSION\""]' "$(INSTALL_DIR)/etc/emissary/agent.yml"
+	tools/yq/bin/yq -i '.agent.controllers.app.dataDir = "/data/emissary/apps/data"' "$(INSTALL_DIR)/etc/emissary/agent.yml"
+	tools/yq/bin/yq -i '.agent.controllers.app.downloadDir = "/data/emissary/apps/bundles"' "$(INSTALL_DIR)/etc/emissary/agent.yml"
+	tools/yq/bin/yq -i '.sentry.dsn = "$${EMISSARY_SENTRY_DSN}"' "$(INSTALL_DIR)/etc/emissary/agent.yml"
+	tools/yq/bin/yq -i '.sentry.environment = "$${EMISSARY_SENTRY_ENVIRONMENT}"' "$(INSTALL_DIR)/etc/emissary/agent.yml"
 
 	# Copy emissary binary
-	mkdir -p files/usr/local/bin
-	cp gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH)/emissary files/usr/local/bin/emissary
-	chmod +x files/usr/local/bin/emissary
+	mkdir -p "$(INSTALL_DIR)/usr/local/bin"
+	cp gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH)/emissary "$(INSTALL_DIR)/usr/local/bin/emissary"
+	chmod +x "$(INSTALL_DIR)/usr/local/bin/emissary"
 
 	# Set defaults
-	mkdir -p files/data/emissary
-	rm -rf files/data/emissary/default.conf
-	echo "EMISSARY_RECONCILIATION_INTERVAL='$(EMISSARY_RECONCILIATION_INTERVAL)'" > files/data/emissary/default.conf
-	echo "EMISSARY_SERVER_URL='$(EMISSARY_SERVER_URL)'" >> files/data/emissary/default.conf
+	mkdir -p "$(INSTALL_DIR)/data/emissary"
+	rm -rf "$(INSTALL_DIR)/data/emissary/default.conf"
+	echo "EMISSARY_RECONCILIATION_INTERVAL='$(EMISSARY_RECONCILIATION_INTERVAL)'" > "$(INSTALL_DIR)/data/emissary/default.conf"
+	echo "EMISSARY_SERVER_URL='$(EMISSARY_SERVER_URL)'" >> "$(INSTALL_DIR)/data/emissary/default.conf"
 
 	# Compress emissary binary
-	tools/upx/bin/upx -9 files/usr/local/bin/emissary
+	tools/upx/bin/upx -9 "$(INSTALL_DIR)/usr/local/bin/emissary"

install/raspberrypi.mk

@@ -1,3 +1,3 @@
 install-rpi-network-config:
-	mkdir -p files/etc/config
-	cp misc/rpi/uci/network files/etc/config/network
+	mkdir -p "$(INSTALL_DIR)/etc/config"
+	cp misc/rpi/uci/network "$(INSTALL_DIR)/etc/config/network"

install/turris-omnia.mk

@@ -1,3 +1,3 @@
 install-turris-omnia-uci-defaults:
-	mkdir -p files/etc/uci-defaults
-	cp misc/turris/omnia/uci-defaults/* files/etc/uci-defaults/
+	mkdir -p "$(INSTALL_DIR)/etc/uci-defaults"
+	cp misc/turris/omnia/uci-defaults/* "$(INSTALL_DIR)/etc/uci-defaults/"

install/x86.mk

@@ -1,7 +1,7 @@
 install-x86-network-config:
-	mkdir -p files/etc/config
-	cp misc/x86/uci/network files/etc/config/network
+	mkdir -p "$(INSTALL_DIR)/etc/config"
+	cp misc/x86/uci/network "$(INSTALL_DIR)/etc/config/network"
 
 install-x86-uci-defaults:
-	mkdir -p files/etc/uci-defaults
-	cp misc/x86/uci-defaults/* files/etc/uci-defaults/
+	mkdir -p "$(INSTALL_DIR)/etc/uci-defaults"
+	cp misc/x86/uci-defaults/* "$(INSTALL_DIR)/etc/uci-defaults/"

targets/linksys-wrt.mk

@@ -1,6 +1,6 @@
 all: linksys-wrtXXXXac
 
-linksys-wrtXXXXac: linksys-wrt1200ac linksys-wrt1900ac linksys-wrt3200acm
+linksys-wrtXXXXac: linksys-wrt1200ac .WAIT linksys-wrt1900ac .WAIT linksys-wrt3200acm
 
 linksys-wrt1200ac:
 	$(MAKE) \

targets/raspberrypi.mk

@@ -1,4 +1,4 @@
-all: rpi-4 rpi-3
+all: rpi-4 .WAIT rpi-3
 
 rpi-4:
 	$(MAKE) \