From d0dbf6d74246646b255cb1151a480f6c7dbc6a31 Mon Sep 17 00:00:00 2001 From: William Petit Date: Sat, 2 Dec 2023 18:57:57 +0100 Subject: [PATCH] feat: run firmware builds in parallel --- Jenkinsfile | 3 +- Makefile | 6 ++-- install/bananapi.mk | 4 +-- install/common.mk | 8 +++--- install/emissary-files.mk | 58 +++++++++++++++++++-------------------- install/raspberrypi.mk | 4 +-- install/turris-omnia.mk | 4 +-- install/x86.mk | 8 +++--- targets/linksys-wrt.mk | 2 +- targets/raspberrypi.mk | 2 +- 10 files changed, 51 insertions(+), 48 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e307b63..b44fc2b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -72,7 +72,8 @@ pipeline { ]) { sh ''' make download-emissary-release - make all + MAX_LOAD_AVERAGE=$(( $(nproc --all ) / 2 )) + make -j -l $MAX_LOAD_AVERAGE all ''' } } diff --git a/Makefile b/Makefile index aaca836..1085931 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ IMAGEBUILDER_ARCHIVE_PATH ?= tmp/imagebuilder-$(OPENWRT_VERSION)-$(OPENWRT_TARGE IMAGEBUILDER_DIR_PATH ?= $(PWD)/imagebuilder/$(OPENWRT_VERSION)/$(OPENWRT_TARGET) IMAGEBUILDER_CUSTOM_PACKAGES_DIR_PATH := $(IMAGEBUILDER_DIR_PATH)/packages IMAGEBUILDER_CUSTOM_FILES_DIR_PATH := $(IMAGEBUILDER_DIR_PATH)/files +INSTALL_DIR := files/$(OPENWRT_VERSION)/$(OPENWRT_TARGET)/$(OPENWRT_PROFILE) BIN_DIR := "$(shell readlink -f bin)/$(OPENWRT_VERSION)/$(OPENWRT_TARGET)/$(OPENWRT_PROFILE)$(BIN_DIR_NAME_SUFFIX)" @@ -81,10 +82,11 @@ $(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH): echo "FIRMWARE_VERSION=$(FULL_VERSION)" >> "$(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH)/etc/emissary_firmware" echo "OPENWRT_PROFILE=$(OPENWRT_PROFILE)" >> "$(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH)/etc/emissary_firmware" - rm -rf files/* + rm -rf $(INSTALL_DIR)/* + mkdir -p "$(INSTALL_DIR)" $(MAKE) $(BASE_INSTALL) $(ADDITIONAL_INSTALL) - cp -rf files/* $(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH)/ + cp -rf $(INSTALL_DIR)/* $(IMAGEBUILDER_CUSTOM_FILES_DIR_PATH)/ flash: OPENWRT_DEVICE=$(OPENWRT_DEVICE) OPENWRT_PROFILE=$(OPENWRT_PROFILE) OPENWRT_VERSION=$(OPENWRT_VERSION) misc/script/flash.sh diff --git a/install/bananapi.mk b/install/bananapi.mk index 3a976a2..3ed47cc 100644 --- a/install/bananapi.mk +++ b/install/bananapi.mk @@ -1,3 +1,3 @@ install-bpi-r3-network-config: - mkdir -p files/etc/config - cp misc/bpi-r3/uci/network files/etc/config/network \ No newline at end of file + mkdir -p "$(INSTALL_DIR)/etc/config" + cp misc/bpi-r3/uci/network "$(INSTALL_DIR)/etc/config/network" \ No newline at end of file diff --git a/install/common.mk b/install/common.mk index b0e3acd..3c6c1d4 100644 --- a/install/common.mk +++ b/install/common.mk @@ -1,6 +1,6 @@ install-common-uci-defaults: - mkdir -p files/etc/uci-defaults - cp misc/common/uci-defaults/* files/etc/uci-defaults/ + mkdir -p "$(INSTALL_DIR)/etc/uci-defaults" + cp misc/common/uci-defaults/* "$(INSTALL_DIR)/etc/uci-defaults/" -install-common-additional-agent-collectors: tools/yq/bin/yq - tools/yq/bin/yq -i '.agent.collectors += load("misc/common/agent/collectors.yml")' files/etc/emissary/agent.yml \ No newline at end of file +install-common-additional-agent-collectors: tools/yq/bin/yq install-emissary-files + tools/yq/bin/yq -i '.agent.collectors += load("misc/common/agent/collectors.yml")' "$(INSTALL_DIR)/etc/emissary/agent.yml" \ No newline at end of file diff --git a/install/emissary-files.mk b/install/emissary-files.mk index 153ef0a..d552bfd 100644 --- a/install/emissary-files.mk +++ b/install/emissary-files.mk @@ -1,49 +1,49 @@ install-emissary-files: tools/yq/bin/yq tools/upx/bin/upx - mkdir -p files/etc/config - cp -r misc/emissary/config/* files/etc/config/ + mkdir -p "$(INSTALL_DIR)/etc/config" + cp -r misc/emissary/config/* "$(INSTALL_DIR)/etc/config/" - mkdir -p files/etc/init.d - cp -r misc/emissary/init.d/* files/etc/init.d/ + mkdir -p "$(INSTALL_DIR)/etc/init.d" + cp -r misc/emissary/init.d/* "$(INSTALL_DIR)/etc/init.d/" - mkdir -p files/etc/uci-defaults - cp -r misc/emissary/uci-defaults/* files/etc/uci-defaults/ + mkdir -p "$(INSTALL_DIR)/etc/uci-defaults" + cp -r misc/emissary/uci-defaults/* "$(INSTALL_DIR)/etc/uci-defaults/" # Copy keep.d files - mkdir -p files/lib/upgrade/keep.d - cp -r misc/emissary/keep.d/* files/lib/upgrade/keep.d/ + mkdir -p "$(INSTALL_DIR)/lib/upgrade/keep.d" + cp -r misc/emissary/keep.d/* "$(INSTALL_DIR)/lib/upgrade/keep.d/" # Copy profile.d files - mkdir -p files/etc/profile.d - cp -r misc/emissary/profile.d/* files/etc/profile.d/ + mkdir -p "$(INSTALL_DIR)/etc/profile.d" + cp -r misc/emissary/profile.d/* "$(INSTALL_DIR)/etc/profile.d/" mkdir -p gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH) cd gitea-dl && tar -xzf emissary-agent_*_linux_$(EMISSARY_ARCH).tar.gz -C emissary-agent_linux_$(EMISSARY_ARCH) # Copy agent config - mkdir -p files/etc/emissary - cp gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH)/misc/packaging/common/config-agent.yml files/etc/emissary/agent.yml + mkdir -p "$(INSTALL_DIR)/etc/emissary" + cp gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH)/misc/packaging/common/config-agent.yml "$(INSTALL_DIR)/etc/emissary/agent.yml" # Patch agent config - tools/yq/bin/yq -i '.agent.serverUrl = "$${EMISSARY_SERVER_URL}"' files/etc/emissary/agent.yml - tools/yq/bin/yq -i '.agent.reconciliationInterval = "$${EMISSARY_RECONCILIATION_INTERVAL}"' files/etc/emissary/agent.yml - tools/yq/bin/yq -i '.agent.privateKeyPath = "/data/emissary/agent-key.json"' files/etc/emissary/agent.yml - tools/yq/bin/yq -i '.agent.controllers.persistence.stateFile = "/data/emissary/agent-state.json"' files/etc/emissary/agent.yml - tools/yq/bin/yq -i '.agent.controllers.sysupgrade.firmwareVersionCommand = ["sh", "-c", "source /etc/emissary_firmware && echo \"$$FIRMWARE_VERSION\""]' files/etc/emissary/agent.yml - tools/yq/bin/yq -i '.agent.controllers.app.dataDir = "/data/emissary/apps/data"' files/etc/emissary/agent.yml - tools/yq/bin/yq -i '.agent.controllers.app.downloadDir = "/data/emissary/apps/bundles"' files/etc/emissary/agent.yml - tools/yq/bin/yq -i '.sentry.dsn = "$${EMISSARY_SENTRY_DSN}"' files/etc/emissary/agent.yml - tools/yq/bin/yq -i '.sentry.environment = "$${EMISSARY_SENTRY_ENVIRONMENT}"' files/etc/emissary/agent.yml + tools/yq/bin/yq -i '.agent.serverUrl = "$${EMISSARY_SERVER_URL}"' "$(INSTALL_DIR)/etc/emissary/agent.yml" + tools/yq/bin/yq -i '.agent.reconciliationInterval = "$${EMISSARY_RECONCILIATION_INTERVAL}"' "$(INSTALL_DIR)/etc/emissary/agent.yml" + tools/yq/bin/yq -i '.agent.privateKeyPath = "/data/emissary/agent-key.json"' "$(INSTALL_DIR)/etc/emissary/agent.yml" + tools/yq/bin/yq -i '.agent.controllers.persistence.stateFile = "/data/emissary/agent-state.json"' "$(INSTALL_DIR)/etc/emissary/agent.yml" + tools/yq/bin/yq -i '.agent.controllers.sysupgrade.firmwareVersionCommand = ["sh", "-c", "source /etc/emissary_firmware && echo \"$$FIRMWARE_VERSION\""]' "$(INSTALL_DIR)/etc/emissary/agent.yml" + tools/yq/bin/yq -i '.agent.controllers.app.dataDir = "/data/emissary/apps/data"' "$(INSTALL_DIR)/etc/emissary/agent.yml" + tools/yq/bin/yq -i '.agent.controllers.app.downloadDir = "/data/emissary/apps/bundles"' "$(INSTALL_DIR)/etc/emissary/agent.yml" + tools/yq/bin/yq -i '.sentry.dsn = "$${EMISSARY_SENTRY_DSN}"' "$(INSTALL_DIR)/etc/emissary/agent.yml" + tools/yq/bin/yq -i '.sentry.environment = "$${EMISSARY_SENTRY_ENVIRONMENT}"' "$(INSTALL_DIR)/etc/emissary/agent.yml" # Copy emissary binary - mkdir -p files/usr/local/bin - cp gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH)/emissary files/usr/local/bin/emissary - chmod +x files/usr/local/bin/emissary + mkdir -p "$(INSTALL_DIR)/usr/local/bin" + cp gitea-dl/emissary-agent_linux_$(EMISSARY_ARCH)/emissary "$(INSTALL_DIR)/usr/local/bin/emissary" + chmod +x "$(INSTALL_DIR)/usr/local/bin/emissary" # Set defaults - mkdir -p files/data/emissary - rm -rf files/data/emissary/default.conf - echo "EMISSARY_RECONCILIATION_INTERVAL='$(EMISSARY_RECONCILIATION_INTERVAL)'" > files/data/emissary/default.conf - echo "EMISSARY_SERVER_URL='$(EMISSARY_SERVER_URL)'" >> files/data/emissary/default.conf + mkdir -p "$(INSTALL_DIR)/data/emissary" + rm -rf "$(INSTALL_DIR)/data/emissary/default.conf" + echo "EMISSARY_RECONCILIATION_INTERVAL='$(EMISSARY_RECONCILIATION_INTERVAL)'" > "$(INSTALL_DIR)/data/emissary/default.conf" + echo "EMISSARY_SERVER_URL='$(EMISSARY_SERVER_URL)'" >> "$(INSTALL_DIR)/data/emissary/default.conf" # Compress emissary binary - tools/upx/bin/upx -9 files/usr/local/bin/emissary \ No newline at end of file + tools/upx/bin/upx -9 "$(INSTALL_DIR)/usr/local/bin/emissary" \ No newline at end of file diff --git a/install/raspberrypi.mk b/install/raspberrypi.mk index c4d9935..ebaacdb 100644 --- a/install/raspberrypi.mk +++ b/install/raspberrypi.mk @@ -1,3 +1,3 @@ install-rpi-network-config: - mkdir -p files/etc/config - cp misc/rpi/uci/network files/etc/config/network \ No newline at end of file + mkdir -p "$(INSTALL_DIR)/etc/config" + cp misc/rpi/uci/network "$(INSTALL_DIR)/etc/config/network" \ No newline at end of file diff --git a/install/turris-omnia.mk b/install/turris-omnia.mk index d66a025..8e8e7eb 100644 --- a/install/turris-omnia.mk +++ b/install/turris-omnia.mk @@ -1,3 +1,3 @@ install-turris-omnia-uci-defaults: - mkdir -p files/etc/uci-defaults - cp misc/turris/omnia/uci-defaults/* files/etc/uci-defaults/ \ No newline at end of file + mkdir -p "$(INSTALL_DIR)/etc/uci-defaults" + cp misc/turris/omnia/uci-defaults/* "$(INSTALL_DIR)/etc/uci-defaults/" \ No newline at end of file diff --git a/install/x86.mk b/install/x86.mk index a3d0ce8..b878aa0 100644 --- a/install/x86.mk +++ b/install/x86.mk @@ -1,7 +1,7 @@ install-x86-network-config: - mkdir -p files/etc/config - cp misc/x86/uci/network files/etc/config/network + mkdir -p "$(INSTALL_DIR)/etc/config" + cp misc/x86/uci/network "$(INSTALL_DIR)/etc/config/network" install-x86-uci-defaults: - mkdir -p files/etc/uci-defaults - cp misc/x86/uci-defaults/* files/etc/uci-defaults/ \ No newline at end of file + mkdir -p "$(INSTALL_DIR)/etc/uci-defaults" + cp misc/x86/uci-defaults/* "$(INSTALL_DIR)/etc/uci-defaults/" \ No newline at end of file diff --git a/targets/linksys-wrt.mk b/targets/linksys-wrt.mk index 1a91e90..58045c9 100644 --- a/targets/linksys-wrt.mk +++ b/targets/linksys-wrt.mk @@ -1,6 +1,6 @@ all: linksys-wrtXXXXac -linksys-wrtXXXXac: linksys-wrt1200ac linksys-wrt1900ac linksys-wrt3200acm +linksys-wrtXXXXac: linksys-wrt1200ac .WAIT linksys-wrt1900ac .WAIT linksys-wrt3200acm linksys-wrt1200ac: $(MAKE) \ diff --git a/targets/raspberrypi.mk b/targets/raspberrypi.mk index 23da983..cd3b764 100644 --- a/targets/raspberrypi.mk +++ b/targets/raspberrypi.mk @@ -1,4 +1,4 @@ -all: rpi-4 rpi-3 +all: rpi-4 .WAIT rpi-3 rpi-4: $(MAKE) \