Compare commits

..

21 Commits

Author SHA1 Message Date
562d698066 feat(controller, app): add fetch module
Some checks failed
arcad/emissary/pipeline/head There was a failure building this commit
2023-04-02 18:05:53 +02:00
909549f056 feat(agent): do not block execution of controllers on error
Some checks reported errors
arcad/emissary/pipeline/head Something is wrong with the build of this commit
2023-04-01 19:44:00 +02:00
7d551a8312 feat(auth): accept clock skew for token validation
Some checks reported errors
arcad/emissary/pipeline/head Something is wrong with the build of this commit
2023-04-01 19:30:45 +02:00
d02eb91b11 feat(agent): add contactedAt attribute to agent
Some checks reported errors
arcad/emissary/pipeline/head Something is wrong with the build of this commit
2023-04-01 14:33:19 +02:00
d2bcdd2999 feat(storage,agent): add label attribute
Some checks reported errors
arcad/emissary/pipeline/head Something is wrong with the build of this commit
2023-04-01 13:28:18 +02:00
c638fe102b chore(jenkins): use global git config for credentials
Some checks reported errors
arcad/emissary/pipeline/head Something is wrong with the build of this commit
2023-03-31 17:31:44 +02:00
273265c3ef feat(agent,run): start proxy controller after app one
Some checks failed
arcad/emissary/pipeline/head There was a failure building this commit
2023-03-31 17:29:33 +02:00
3e02a9f031 chore(controller,app): refactor app server mutex usage
Some checks failed
arcad/emissary/pipeline/head There was a failure building this commit
2023-03-31 17:28:52 +02:00
b52c687643 feat(sqlite): add default pragmas to dsn 2023-03-31 17:27:54 +02:00
8119a01bf6 chore: add jenkins pipeline
Some checks failed
arcad/emissary/pipeline/head There was a failure building this commit
2023-03-31 17:24:33 +02:00
e5b6c5e949 chore(sqlite): use wal journal mode and enable fk checks by default 2023-03-29 20:58:46 +02:00
9c69dc7ec8 feat(auth): use utc time 2023-03-29 20:49:44 +02:00
4e6b450338 feat(storage,sqlite): log row closing errors 2023-03-29 20:10:06 +02:00
351f22e216 feat(controller,app): automatically redirect requests to cookie domain 2023-03-29 17:29:16 +02:00
854a6ae41b fix(controller,app): include auth configuration in changes detection 2023-03-29 15:32:23 +02:00
a48c2ebe14 fix(controller,app): update app repository on spec changes 2023-03-29 11:27:47 +02:00
cdd78e4031 chore: update go.mod 2023-03-28 20:48:00 +02:00
e832b7dedd feat(controller,app): add app module 2023-03-28 20:43:45 +02:00
3d0b0102c2 chore: only watch json files in internal directory 2023-03-28 11:03:47 +02:00
e60c441d43 feat: move proxy package from arcad/edge 2023-03-28 11:02:53 +02:00
f7d70da174 chore: generate armv7 binaries 2023-03-27 17:17:42 +02:00
53 changed files with 1285 additions and 488 deletions

2
.gitignore vendored
View File

@ -4,7 +4,7 @@ dist/
/tools
/tmp
/state.json
/emissary.sqlite
/emissary.sqlite*
/.gitea-release
/agent-key.json
/apps

View File

@ -46,6 +46,9 @@ builds:
- arm64
- arm
- "386"
goarm:
- "6"
- "7"
main: ./cmd/agent
archives:
- id: server

70
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,70 @@
@Library('cadoles') _
pipeline {
agent {
dockerfile {
label 'docker'
filename 'Dockerfile'
dir 'misc/jenkins'
}
}
stages {
stage('Run unit tests') {
steps {
script {
withCredentials([
usernamePassword([
credentialsId: 'forge-jenkins',
usernameVariable: 'GIT_USERNAME',
passwordVariable: 'GIT_PASSWORD'
])
]) {
sh '''
git config --global credential.https://forge.cadoles.com.username "$GIT_USERNAME"
git config --global credential.https://forge.cadoles.com.helper '!f() { test "$1" = get && echo "password=$GIT_PASSWORD"; }; f'
export GOPRIVATE=forge.cadoles.com/arcad/edge
make test
'''
}
}
}
}
stage('Release') {
when {
anyOf {
branch 'master'
branch 'develop'
}
}
steps {
script {
withCredentials([
usernamePassword([
credentialsId: 'forge-jenkins',
usernameVariable: 'GITEA_RELEASE_USERNAME',
passwordVariable: 'GITEA_RELEASE_PASSWORD'
])
]) {
sh 'make gitea-release'
}
def currentVersion = sh(returnStdout: true, script: 'make full-version').trim()
build(
job: "../emissary-firmware/${env.GIT_BRANCH}",
parameters: [
[$class: 'StringParameterValue', name: 'emissaryRelease', value: currentVersion]
]
)
}
}
}
}
post {
always {
cleanWs()
}
}
}

View File

@ -135,7 +135,7 @@ gitea-release: tools/gitea-release/bin/gitea-release.sh goreleaser
GITEA_RELEASE_COMMITISH_TARGET="$(GIT_VERSION)" \
GITEA_RELEASE_IS_DRAFT="false" \
GITEA_RELEASE_BODY="" \
GITEA_RELEASE_ATTACHMENTS="$(shell find .gitea-release/* -type f)" \
GITEA_RELEASE_ATTACHMENTS="$$(find .gitea-release/* -type f)" \
tools/gitea-release/bin/gitea-release.sh
tools/gitea-release/bin/gitea-release.sh:
@ -144,4 +144,13 @@ tools/gitea-release/bin/gitea-release.sh:
chmod +x tools/gitea-release/bin/gitea-release.sh
.emissary-token:
$(MAKE) run-emissary-server EMISSARY_CMD="--debug --config tmp/server.yml server auth create-token --role writer > .emissary-token"
$(MAKE) run-emissary-server EMISSARY_CMD="--debug --config tmp/server.yml server auth create-token --role writer > .emissary-token"
AGENT_ID ?= 1
load-sample-specs:
cat misc/spec-samples/app.emissary.cadoles.com.json | ./bin/server api agent spec update -a $(AGENT_ID) --no-patch --spec-data - --spec-name app.emissary.cadoles.com
cat misc/spec-samples/proxy.emissary.cadoles.com.json | ./bin/server api agent spec update -a $(AGENT_ID) --no-patch --spec-data - --spec-name proxy.emissary.cadoles.com
full-version:
@echo $(FULL_VERSION)

View File

@ -7,6 +7,7 @@ import (
"forge.cadoles.com/Cadoles/emissary/internal/command/agent"
"forge.cadoles.com/Cadoles/emissary/internal/command/api"
_ "forge.cadoles.com/Cadoles/emissary/internal/imports/format"
_ "forge.cadoles.com/Cadoles/emissary/internal/imports/spec"
)

11
go.mod
View File

@ -3,7 +3,8 @@ module forge.cadoles.com/Cadoles/emissary
go 1.19
require (
forge.cadoles.com/arcad/edge v0.0.0-20230322170544-cf8a3f8ac077
forge.cadoles.com/arcad/edge v0.0.0-20230402160147-f08f645432c6
github.com/Masterminds/sprig/v3 v3.2.3
github.com/alecthomas/participle/v2 v2.0.0-beta.5
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
github.com/btcsuite/btcd/btcutil v1.1.3
@ -29,6 +30,8 @@ require (
)
require (
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
github.com/barnybug/go-cast v0.0.0-20201201064555-a87ccbc26692 // indirect
github.com/dop251/goja_nodejs v0.0.0-20230320130059-dcf93ba651dd // indirect
github.com/gabriel-vasile/mimetype v1.4.1 // indirect
@ -37,10 +40,16 @@ require (
github.com/google/pprof v0.0.0-20230309165930-d61513b1440d // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/hashicorp/mdns v1.0.5 // indirect
github.com/huandu/xstrings v1.3.3 // indirect
github.com/igm/sockjs-go/v3 v3.0.2 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/miekg/dns v1.1.51 // indirect
github.com/mitchellh/copystructure v1.0.0 // indirect
github.com/mitchellh/reflectwalk v1.0.0 // indirect
github.com/oklog/ulid/v2 v2.1.0 // indirect
github.com/orcaman/concurrent-map v1.0.0 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
golang.org/x/net v0.8.0 // indirect
google.golang.org/genproto v0.0.0-20220728213248-dd149ef739b9 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect

23
go.sum
View File

@ -54,8 +54,10 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
forge.cadoles.com/arcad/edge v0.0.0-20230322170544-cf8a3f8ac077 h1:vsYcNHZevZrs0VeOTasvJoqvPynb8OvH+MMpIUvNT6Q=
forge.cadoles.com/arcad/edge v0.0.0-20230322170544-cf8a3f8ac077/go.mod h1:ONd6vyQ0IM0vHi1i+bmZBRc1Fd0BoXMuDdY/+0sZefw=
forge.cadoles.com/arcad/edge v0.0.0-20230328183829-d8ce2901d2ab h1:xOtzLAYOUcKd/VBx/PzL2riC0zNuQ/cxxf5r3AmEvJE=
forge.cadoles.com/arcad/edge v0.0.0-20230328183829-d8ce2901d2ab/go.mod h1:ONd6vyQ0IM0vHi1i+bmZBRc1Fd0BoXMuDdY/+0sZefw=
forge.cadoles.com/arcad/edge v0.0.0-20230402160147-f08f645432c6 h1:MxMEBSEvwagUrFORUJ9snZekFIKkaV3OB0EplXra+LU=
forge.cadoles.com/arcad/edge v0.0.0-20230402160147-f08f645432c6/go.mod h1:ONd6vyQ0IM0vHi1i+bmZBRc1Fd0BoXMuDdY/+0sZefw=
gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg=
github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
@ -84,6 +86,12 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA=
github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM=
github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
@ -761,6 +769,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
@ -771,6 +781,7 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ=
@ -974,6 +985,8 @@ github.com/miekg/dns v1.1.51/go.mod h1:2Z9d3CP1LQWihRZUf29mQ19yDThaI4DAYzte2CaQW
github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@ -987,6 +1000,8 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
@ -1180,6 +1195,7 @@ github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
@ -1204,6 +1220,8 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd
github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4=
github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
@ -1379,6 +1397,7 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

View File

@ -44,8 +44,6 @@ func (a *Agent) Run(ctx context.Context) error {
if err := a.registerAgent(ctx, client, state); err != nil {
logger.Error(ctx, "could not register agent", logger.E(errors.WithStack(err)))
return
}
logger.Debug(ctx, "state before reconciliation", logger.F("state", state))
@ -81,7 +79,7 @@ func (a *Agent) Reconcile(ctx context.Context, state *State) error {
)
if err := ctrl.Reconcile(ctrlCtx, state); err != nil {
return errors.WithStack(err)
logger.Error(ctx, "could not reconcile", logger.E(errors.WithStack(err)))
}
}

View File

@ -0,0 +1,192 @@
package app
import (
"bytes"
"context"
"database/sql"
"path/filepath"
"sync"
"text/template"
"forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
appSpec "forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
"forge.cadoles.com/Cadoles/emissary/internal/jwk"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus"
"forge.cadoles.com/arcad/edge/pkg/bus/memory"
edgeHTTP "forge.cadoles.com/arcad/edge/pkg/http"
"forge.cadoles.com/arcad/edge/pkg/module"
appModule "forge.cadoles.com/arcad/edge/pkg/module/app"
"forge.cadoles.com/arcad/edge/pkg/module/auth"
"forge.cadoles.com/arcad/edge/pkg/module/blob"
"forge.cadoles.com/arcad/edge/pkg/module/cast"
fetchModule "forge.cadoles.com/arcad/edge/pkg/module/fetch"
"forge.cadoles.com/arcad/edge/pkg/module/net"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"github.com/Masterminds/sprig/v3"
"github.com/dop251/goja"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/pkg/errors"
)
const defaultSQLiteParams = "?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_txlock=immediate"
func (c *Controller) getHandlerOptions(ctx context.Context, appKey string, specs *spec.Spec) ([]edgeHTTP.HandlerOptionFunc, error) {
dataDir, err := c.ensureAppDataDir(ctx, appKey)
if err != nil {
return nil, errors.Wrap(err, "could not retrieve app data dir")
}
dbFile := filepath.Join(dataDir, appKey+".sqlite")
db, err := sqlite.Open(dbFile + defaultSQLiteParams)
if err != nil {
return nil, errors.Wrapf(err, "could not open database file '%s'", dbFile)
}
keySet, err := getAuthKeySet(specs.Config)
if err != nil {
return nil, errors.Wrap(err, "could not retrieve auth key set")
}
bundles := make([]string, 0, len(specs.Apps))
for appKey, app := range specs.Apps {
path := c.getAppBundlePath(appKey, app.Format)
bundles = append(bundles, path)
}
bus := memory.NewBus()
modules := c.getAppModules(bus, db, specs, keySet)
options := []edgeHTTP.HandlerOptionFunc{
edgeHTTP.WithBus(bus),
edgeHTTP.WithServerModules(modules...),
}
return options, nil
}
func getAuthKeySet(config *spec.Config) (jwk.Set, error) {
keySet := jwk.NewSet()
if config == nil {
return nil, nil
}
auth := config.Auth
if auth == nil {
return nil, nil
}
switch {
case auth.Local != nil:
var (
key jwk.Key
err error
)
switch typedKey := auth.Local.Key.(type) {
case string:
key, err = jwk.FromRaw([]byte(typedKey))
if err != nil {
return nil, errors.Wrap(err, "could not parse local auth key")
}
if err := key.Set(jwk.AlgorithmKey, jwa.HS256); err != nil {
return nil, errors.WithStack(err)
}
default:
return nil, errors.Errorf("unexpected key type '%T'", auth.Local.Key)
}
if err := keySet.AddKey(key); err != nil {
return nil, errors.WithStack(err)
}
}
return keySet, nil
}
func createGetAppURL(specs *spec.Spec) GetURLFunc {
var (
compileOnce sync.Once
urlTemplate *template.Template
err error
)
return func(ctx context.Context, manifest *app.Manifest) (string, error) {
if err != nil {
return "", errors.WithStack(err)
}
var appURLTemplate string
if specs.Config == nil || specs.Config.AppURLTemplate == "" {
appURLTemplate = `http://{{ last ( splitList "." ( toString .Manifest.ID ) ) }}.local`
} else {
appURLTemplate = specs.Config.AppURLTemplate
}
compileOnce.Do(func() {
urlTemplate, err = template.New("").Funcs(sprig.TxtFuncMap()).Parse(appURLTemplate)
})
var buf bytes.Buffer
data := struct {
Manifest *app.Manifest
Specs *spec.Spec
}{
Manifest: manifest,
Specs: specs,
}
if err := urlTemplate.Execute(&buf, data); err != nil {
return "", errors.WithStack(err)
}
return buf.String(), nil
}
}
func (c *Controller) getAppModules(bus bus.Bus, db *sql.DB, spec *appSpec.Spec, keySet jwk.Set) []app.ServerModuleFactory {
ds := sqlite.NewDocumentStoreWithDB(db)
bs := sqlite.NewBlobStoreWithDB(db)
return []app.ServerModuleFactory{
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),
cast.CastModuleFactory(),
module.LifecycleModuleFactory(),
net.ModuleFactory(bus),
module.RPCModuleFactory(bus),
module.StoreModuleFactory(ds),
blob.ModuleFactory(bus, bs),
module.Extends(
auth.ModuleFactory(
auth.WithJWT(func() (jwk.Set, error) {
return keySet, nil
}),
),
func(o *goja.Object) {
if err := o.Set("CLAIM_TENANT", "arcad_tenant"); err != nil {
panic(errors.New("could not set 'CLAIM_TENANT' property"))
}
if err := o.Set("CLAIM_ENTRYPOINT", "arcad_entrypoint"); err != nil {
panic(errors.New("could not set 'CLAIM_ENTRYPOINT' property"))
}
if err := o.Set("CLAIM_ROLE", "arcad_role"); err != nil {
panic(errors.New("could not set 'CLAIM_ROLE' property"))
}
if err := o.Set("CLAIM_PREFERRED_USERNAME", "preferred_username"); err != nil {
panic(errors.New("could not set 'CLAIM_PREFERRED_USERNAME' property"))
}
},
),
appModule.ModuleFactory(c.appRepository),
fetchModule.ModuleFactory(bus),
}
}

View File

@ -0,0 +1,128 @@
package app
import (
"context"
"sync"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bundle"
appModule "forge.cadoles.com/arcad/edge/pkg/module/app"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type GetURLFunc func(context.Context, *app.Manifest) (string, error)
type AppRepository struct {
getURL GetURLFunc
bundles []string
mutex sync.RWMutex
}
// Get implements app.Repository
func (r *AppRepository) Get(ctx context.Context, id app.ID) (*app.Manifest, error) {
r.mutex.RLock()
defer r.mutex.RUnlock()
manifest, err := r.findManifest(ctx, id)
if err != nil {
return nil, errors.WithStack(err)
}
return manifest, nil
}
// GetURL implements app.Repository
func (r *AppRepository) GetURL(ctx context.Context, id app.ID) (string, error) {
r.mutex.RLock()
defer r.mutex.RUnlock()
manifest, err := r.findManifest(ctx, id)
if err != nil {
return "", errors.WithStack(err)
}
url, err := r.getURL(ctx, manifest)
if err != nil {
return "", errors.WithStack(err)
}
return url, nil
}
// List implements app.Repository
func (r *AppRepository) List(ctx context.Context) ([]*app.Manifest, error) {
r.mutex.RLock()
defer r.mutex.RUnlock()
manifests := make([]*app.Manifest, 0)
for _, path := range r.bundles {
bundleCtx := logger.With(ctx, logger.F("path", path))
bundle, err := bundle.FromPath(path)
if err != nil {
logger.Error(bundleCtx, "could not load bundle", logger.E(errors.WithStack(err)))
continue
}
manifest, err := app.LoadManifest(bundle)
if err != nil {
logger.Error(bundleCtx, "could not load manifest", logger.E(errors.WithStack(err)))
continue
}
manifests = append(manifests, manifest)
}
return manifests, nil
}
func (r *AppRepository) Update(getURL GetURLFunc, bundles []string) {
r.mutex.Lock()
defer r.mutex.Unlock()
r.getURL = getURL
r.bundles = bundles
}
func (r *AppRepository) findManifest(ctx context.Context, id app.ID) (*app.Manifest, error) {
for _, path := range r.bundles {
bundleCtx := logger.With(ctx, logger.F("path", path))
bundle, err := bundle.FromPath(path)
if err != nil {
logger.Error(bundleCtx, "could not load bundle", logger.E(errors.WithStack(err)))
continue
}
manifest, err := app.LoadManifest(bundle)
if err != nil {
logger.Error(bundleCtx, "could not load manifest", logger.E(errors.WithStack(err)))
continue
}
if manifest.ID != id {
continue
}
return manifest, nil
}
return nil, errors.WithStack(appModule.ErrNotFound)
}
func NewAppRepository() *AppRepository {
return &AppRepository{
getURL: func(ctx context.Context, m *app.Manifest) (string, error) {
return "", errors.New("unavailable")
},
bundles: []string{},
}
}
var _ appModule.Repository = &AppRepository{}

View File

@ -8,24 +8,24 @@ import (
"path/filepath"
"forge.cadoles.com/Cadoles/emissary/internal/agent"
"forge.cadoles.com/Cadoles/emissary/internal/spec/app"
"forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
"forge.cadoles.com/arcad/edge/pkg/bundle"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"github.com/mitchellh/hashstructure/v2"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type serverEntry struct {
SpecHash uint64
Server *Server
AppDefHash uint64
Server *Server
}
type Controller struct {
client *http.Client
downloadDir string
dataDir string
servers map[string]*serverEntry
client *http.Client
downloadDir string
dataDir string
servers map[string]*serverEntry
appRepository *AppRepository
}
// Name implements node.Controller.
@ -35,9 +35,9 @@ func (c *Controller) Name() string {
// Reconcile implements node.Controller.
func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error {
appSpec := app.NewSpec()
appSpec := spec.NewSpec()
if err := state.GetSpec(app.NameApp, appSpec); err != nil {
if err := state.GetSpec(spec.Name, appSpec); err != nil {
if errors.Is(err, agent.ErrSpecNotFound) {
logger.Info(ctx, "could not find app spec")
@ -56,7 +56,7 @@ func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error {
return nil
}
func (c *Controller) stopAllApps(ctx context.Context, spec *app.Spec) {
func (c *Controller) stopAllApps(ctx context.Context, spec *spec.Spec) {
if len(c.servers) > 0 {
logger.Info(ctx, "stopping all apps")
}
@ -76,122 +76,148 @@ func (c *Controller) stopAllApps(ctx context.Context, spec *app.Spec) {
}
}
func (c *Controller) updateApps(ctx context.Context, spec *app.Spec) {
func (c *Controller) updateApps(ctx context.Context, specs *spec.Spec) {
// Stop and remove obsolete apps
for appID, entry := range c.servers {
if _, exists := spec.Apps[appID]; exists {
for appKey, server := range c.servers {
if _, exists := specs.Apps[appKey]; exists {
continue
}
logger.Info(ctx, "stopping app", logger.F("appID", appID))
logger.Info(ctx, "stopping app", logger.F("appKey", appKey))
if err := entry.Server.Stop(); err != nil {
if err := server.Server.Stop(); err != nil {
logger.Error(
ctx, "error while stopping app",
logger.F("gatewayID", appID),
logger.F("appKey", appKey),
logger.E(errors.WithStack(err)),
)
delete(c.servers, appID)
delete(c.servers, appKey)
}
}
// (Re)start apps
for appID, appSpec := range spec.Apps {
appCtx := logger.With(ctx, logger.F("appID", appID))
c.updateAppRepository(ctx, specs)
if err := c.updateApp(ctx, appID, appSpec, spec.Auth); err != nil {
// (Re)start apps if necessary
for appKey := range specs.Apps {
appCtx := logger.With(ctx, logger.F("appKey", appKey))
if err := c.updateApp(ctx, specs, appKey); err != nil {
logger.Error(appCtx, "could not update app", logger.E(errors.WithStack(err)))
continue
}
}
}
func (c *Controller) updateApp(ctx context.Context, appID string, appSpec app.AppEntry, auth *app.Auth) (err error) {
newAppSpecHash, err := hashstructure.Hash(appSpec, hashstructure.FormatV2, nil)
func (c *Controller) updateAppRepository(ctx context.Context, specs *spec.Spec) {
bundles := make([]string, 0, len(specs.Apps))
for appKey, app := range specs.Apps {
path := c.getAppBundlePath(appKey, app.Format)
bundles = append(bundles, path)
}
getURL := createGetAppURL(specs)
c.appRepository.Update(getURL, bundles)
}
func (c *Controller) updateApp(ctx context.Context, specs *spec.Spec, appKey string) (err error) {
appEntry := specs.Apps[appKey]
var auth *spec.Auth
if specs.Config != nil {
auth = specs.Config.Auth
}
appDef := struct {
App spec.AppEntry
Auth *spec.Auth
}{
App: appEntry,
Auth: auth,
}
newAppDefHash, err := hashstructure.Hash(appDef, hashstructure.FormatV2, nil)
if err != nil {
return errors.WithStack(err)
}
bundle, sha256sum, err := c.ensureAppBundle(ctx, appID, appSpec)
bundle, sha256sum, err := c.ensureAppBundle(ctx, appKey, appEntry)
if err != nil {
return errors.Wrap(err, "could not download app bundle")
}
dataDir, err := c.ensureAppDataDir(ctx, appID)
if err != nil {
return errors.Wrap(err, "could not retrieve app data dir")
}
var entry *serverEntry
entry, exists := c.servers[appID]
server, exists := c.servers[appKey]
if !exists {
logger.Info(ctx, "app currently not running")
} else if sha256sum != appSpec.SHA256Sum {
} else if sha256sum != appEntry.SHA256Sum {
logger.Info(
ctx, "bundle hash mismatch, stopping app",
logger.F("currentHash", sha256sum),
logger.F("specHash", appSpec.SHA256Sum),
logger.F("specHash", appEntry.SHA256Sum),
)
if err := entry.Server.Stop(); err != nil {
if err := server.Server.Stop(); err != nil {
return errors.Wrap(err, "could not stop app")
}
entry = nil
server = nil
}
if entry == nil {
dbFile := filepath.Join(dataDir, appID+".sqlite")
db, err := sqlite.Open(dbFile)
if server == nil {
options, err := c.getHandlerOptions(ctx, appKey, specs)
if err != nil {
return errors.Wrapf(err, "could not opend database file '%s'", dbFile)
return errors.Wrap(err, "could not create handler options")
}
entry = &serverEntry{
Server: NewServer(bundle, db, auth),
SpecHash: 0,
var auth *spec.Auth
if specs.Config != nil {
auth = specs.Config.Auth
}
c.servers[appID] = entry
server = &serverEntry{
Server: NewServer(bundle, auth, options...),
AppDefHash: 0,
}
c.servers[appKey] = server
}
specChanged := newAppSpecHash != entry.SpecHash
defChanged := newAppDefHash != server.AppDefHash
if entry.Server.Running() && !specChanged {
if server.Server.Running() && !defChanged {
return nil
}
if specChanged && entry.SpecHash != 0 {
if defChanged && server.AppDefHash != 0 {
logger.Info(
ctx, "restarting app",
logger.F("address", appSpec.Address),
logger.F("address", appEntry.Address),
)
} else {
logger.Info(
ctx, "starting app",
logger.F("address", appSpec.Address),
logger.F("address", appEntry.Address),
)
}
if err := entry.Server.Start(ctx, appSpec.Address); err != nil {
delete(c.servers, appID)
if err := server.Server.Start(ctx, appEntry.Address); err != nil {
delete(c.servers, appKey)
return errors.Wrap(err, "could not start app")
}
entry.SpecHash = newAppSpecHash
server.AppDefHash = newAppDefHash
return nil
}
func (c *Controller) ensureAppBundle(ctx context.Context, appID string, spec app.AppEntry) (bundle.Bundle, string, error) {
func (c *Controller) ensureAppBundle(ctx context.Context, appID string, spec spec.AppEntry) (bundle.Bundle, string, error) {
if err := os.MkdirAll(c.downloadDir, os.ModePerm); err != nil {
return nil, "", errors.WithStack(err)
}
bundlePath := filepath.Join(c.downloadDir, appID+"."+spec.Format)
bundlePath := c.getAppBundlePath(appID, spec.Format)
_, err := os.Stat(bundlePath)
if err != nil && !errors.Is(err, os.ErrNotExist) {
@ -285,6 +311,10 @@ func (c *Controller) ensureAppDataDir(ctx context.Context, appID string) (string
return dataDir, nil
}
func (c *Controller) getAppBundlePath(appKey string, format string) string {
return filepath.Join(c.downloadDir, appKey+"."+format)
}
func NewController(funcs ...OptionFunc) *Controller {
opts := defaultOptions()
for _, fn := range funcs {
@ -292,10 +322,11 @@ func NewController(funcs ...OptionFunc) *Controller {
}
return &Controller{
client: opts.Client,
downloadDir: opts.DownloadDir,
dataDir: opts.DataDir,
servers: make(map[string]*serverEntry),
client: opts.Client,
downloadDir: opts.DownloadDir,
dataDir: opts.DataDir,
servers: make(map[string]*serverEntry),
appRepository: NewAppRepository(),
}
}

View File

@ -2,26 +2,19 @@ package app
import (
"context"
"database/sql"
"net/http"
"strings"
"sync"
"time"
appSpec "forge.cadoles.com/Cadoles/emissary/internal/spec/app"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus"
"forge.cadoles.com/arcad/edge/pkg/bus/memory"
"forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
appSpec "forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
"forge.cadoles.com/Cadoles/emissary/internal/proxy/wildcard"
edgeHTTP "forge.cadoles.com/arcad/edge/pkg/http"
"forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/module/auth"
authHTTP "forge.cadoles.com/arcad/edge/pkg/module/auth/http"
"forge.cadoles.com/arcad/edge/pkg/module/cast"
"forge.cadoles.com/arcad/edge/pkg/module/net"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"gitlab.com/wpetit/goweb/logger"
"forge.cadoles.com/arcad/edge/pkg/bundle"
"github.com/dop251/goja"
"github.com/go-chi/chi/middleware"
"github.com/go-chi/chi/v5"
"github.com/lestrrat-go/jwx/v2/jwa"
@ -31,67 +24,37 @@ import (
_ "forge.cadoles.com/Cadoles/emissary/internal/imports/passwd"
)
const defaultCookieDuration time.Duration = 24 * time.Hour
type Server struct {
bundle bundle.Bundle
db *sql.DB
server *http.Server
serverMutex sync.RWMutex
auth *appSpec.Auth
keySet jwk.Set
bundle bundle.Bundle
handlerOptions []edgeHTTP.HandlerOptionFunc
server *http.Server
serverMutex sync.RWMutex
auth *appSpec.Auth
}
func (s *Server) Start(ctx context.Context, addr string) (err error) {
if s.server != nil {
if s.Running() {
if err := s.Stop(); err != nil {
return errors.WithStack(err)
}
}
s.serverMutex.Lock()
defer s.serverMutex.Unlock()
router := chi.NewRouter()
router.Use(middleware.Logger)
bus := memory.NewBus()
ds := sqlite.NewDocumentStoreWithDB(s.db)
bs := sqlite.NewBlobStoreWithDB(s.db)
handler := edgeHTTP.NewHandler(
edgeHTTP.WithBus(bus),
edgeHTTP.WithServerModules(s.getAppModules(bus, ds, bs)...),
)
handler := edgeHTTP.NewHandler(s.handlerOptions...)
if err := handler.Load(s.bundle); err != nil {
return errors.Wrap(err, "could not load app bundle")
}
if s.auth != nil {
if s.auth.Local != nil {
var rawKey any = s.auth.Local.Key
if strKey, ok := rawKey.(string); ok {
rawKey = []byte(strKey)
}
key, err := jwk.FromRaw(rawKey)
if err != nil {
return errors.WithStack(err)
}
if err := key.Set(jwk.AlgorithmKey, jwa.HS256); err != nil {
return errors.WithStack(err)
}
keySet := jwk.NewSet()
if err := keySet.AddKey(key); err != nil {
return errors.WithStack(err)
}
s.keySet = keySet
router.Handle("/auth/*", authHTTP.NewLocalHandler(
jwa.HS256, key,
authHTTP.WithRoutePrefix("/auth"),
authHTTP.WithAccounts(s.auth.Local.Accounts...),
))
}
if err := s.configureAuth(router, s.auth); err != nil {
return errors.WithStack(err)
}
router.Handle("/*", handler)
@ -125,9 +88,7 @@ func (s *Server) Start(ctx context.Context, addr string) (err error) {
}
}()
s.serverMutex.Lock()
s.server = server
s.serverMutex.Unlock()
return nil
}
@ -140,66 +101,102 @@ func (s *Server) Running() bool {
}
func (s *Server) Stop() error {
if !s.Running() {
return nil
}
s.serverMutex.Lock()
defer s.serverMutex.Unlock()
if s.server == nil {
return nil
}
defer func() {
s.serverMutex.Lock()
s.server = nil
s.serverMutex.Unlock()
}()
if err := s.server.Close(); err != nil {
panic(errors.WithStack(err))
s.server = nil
return errors.WithStack(err)
}
s.server = nil
return nil
}
func (s *Server) configureAuth(router chi.Router, auth *spec.Auth) error {
if auth == nil {
return nil
}
switch {
case auth.Local != nil:
var rawKey any = s.auth.Local.Key
if strKey, ok := rawKey.(string); ok {
rawKey = []byte(strKey)
}
key, err := jwk.FromRaw(rawKey)
if err != nil {
return errors.WithStack(err)
}
cookieDuration := defaultCookieDuration
if s.auth.Local.CookieDuration != "" {
cookieDuration, err = time.ParseDuration(s.auth.Local.CookieDuration)
if err != nil {
return errors.WithStack(err)
}
}
if s.auth.Local.CookieDomain != "" {
router.Use(invalidCookieDomainRedirect(s.auth.Local.CookieDomain))
}
router.Handle("/auth/*", authHTTP.NewLocalHandler(
jwa.HS256, key,
authHTTP.WithRoutePrefix("/auth"),
authHTTP.WithAccounts(s.auth.Local.Accounts...),
authHTTP.WithCookieOptions(s.auth.Local.CookieDomain, cookieDuration),
))
}
return nil
}
func (s *Server) getAppModules(bus bus.Bus, ds storage.DocumentStore, bs storage.BlobStore) []app.ServerModuleFactory {
return []app.ServerModuleFactory{
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),
cast.CastModuleFactory(),
module.LifecycleModuleFactory(),
net.ModuleFactory(bus),
module.RPCModuleFactory(bus),
module.StoreModuleFactory(ds),
module.BlobModuleFactory(bus, bs),
module.Extends(
auth.ModuleFactory(
auth.WithJWT(s.getJWTKeySet),
),
func(o *goja.Object) {
if err := o.Set("CLAIM_TENANT", "arcad_tenant"); err != nil {
panic(errors.New("could not set 'CLAIM_TENANT' property"))
}
if err := o.Set("CLAIM_ENTRYPOINT", "arcad_entrypoint"); err != nil {
panic(errors.New("could not set 'CLAIM_ENTRYPOINT' property"))
}
if err := o.Set("CLAIM_ROLE", "arcad_role"); err != nil {
panic(errors.New("could not set 'CLAIM_ROLE' property"))
}
if err := o.Set("CLAIM_PREFERRED_USERNAME", "preferred_username"); err != nil {
panic(errors.New("could not set 'CLAIM_PREFERRED_USERNAME' property"))
}
},
),
}
}
func (s *Server) getJWTKeySet() (jwk.Set, error) {
return s.keySet, nil
}
func NewServer(bundle bundle.Bundle, db *sql.DB, auth *appSpec.Auth) *Server {
func NewServer(bundle bundle.Bundle, auth *appSpec.Auth, handlerOptions ...edgeHTTP.HandlerOptionFunc) *Server {
return &Server{
bundle: bundle,
db: db,
auth: auth,
bundle: bundle,
auth: auth,
handlerOptions: handlerOptions,
}
}
func invalidCookieDomainRedirect(cookieDomain string) func(http.Handler) http.Handler {
domain := strings.TrimPrefix(cookieDomain, ".")
hostPattern := "*" + domain
return func(h http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
hostParts := strings.SplitN(r.Host, ":", 2)
if !wildcard.Match(hostParts[0], hostPattern) {
url := r.URL
newHost := domain
if len(hostParts) > 1 {
newHost += ":" + hostParts[1]
}
url.Host = newHost
http.Redirect(w, r, url.String(), http.StatusTemporaryRedirect)
return
}
h.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}

View File

@ -1,4 +1,4 @@
package app
package spec
import (
_ "embed"
@ -11,7 +11,7 @@ import (
var schema []byte
func init() {
if err := spec.Register(NameApp, schema); err != nil {
if err := spec.Register(Name, schema); err != nil {
panic(errors.WithStack(err))
}
}

View File

@ -71,6 +71,12 @@
"algo"
]
}
},
"cookieDomain": {
"type": "string"
},
"cookieDuration": {
"type": "string"
}
},
"required": [
@ -78,6 +84,11 @@
]
}
}
},
"config": {
"appUrlTemplate": {
"type": "string"
}
}
},
"required": [

View File

@ -1,16 +1,16 @@
package app
package spec
import (
"forge.cadoles.com/Cadoles/emissary/internal/spec"
edgeAuth "forge.cadoles.com/arcad/edge/pkg/module/auth/http"
)
const NameApp spec.Name = "app.emissary.cadoles.com"
const Name spec.Name = "app.emissary.cadoles.com"
type Spec struct {
Revision int `json:"revision"`
Apps map[string]AppEntry `json:"apps"`
Auth *Auth `json:"auth"`
Config *Config `json:"config"`
}
type AppEntry struct {
@ -25,12 +25,19 @@ type Auth struct {
}
type LocalAuth struct {
Key any `json:"key"`
Accounts []edgeAuth.LocalAccount `json:"accounts"`
Key any `json:"key"`
Accounts []edgeAuth.LocalAccount `json:"accounts"`
CookieDomain string `json:"cookieDomain"`
CookieDuration string `json:"cookieDuration"`
}
type Config struct {
Auth *Auth `json:"auth"`
AppURLTemplate string `json:"appUrlTemplate"`
}
func (s *Spec) SpecName() spec.Name {
return NameApp
return Name
}
func (s *Spec) SpecRevision() int {
@ -39,8 +46,8 @@ func (s *Spec) SpecRevision() int {
func (s *Spec) SpecData() map[string]any {
return map[string]any{
"apps": s.Apps,
"auth": s.Auth,
"apps": s.Apps,
"config": s.Config,
}
}

View File

@ -1,4 +1,4 @@
package app
package spec
import (
"context"
@ -28,7 +28,7 @@ func TestValidator(t *testing.T) {
t.Parallel()
validator := spec.NewValidator()
if err := validator.Register(NameApp, schema); err != nil {
if err := validator.Register(Name, schema); err != nil {
t.Fatalf("+%v", errors.WithStack(err))
}

View File

@ -5,8 +5,8 @@ import (
"net/url"
"forge.cadoles.com/Cadoles/emissary/internal/agent"
"forge.cadoles.com/Cadoles/emissary/internal/spec/proxy"
edgeProxy "forge.cadoles.com/arcad/edge/pkg/proxy"
"forge.cadoles.com/Cadoles/emissary/internal/proxy"
spec "forge.cadoles.com/Cadoles/emissary/internal/spec/proxy"
"github.com/mitchellh/hashstructure/v2"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
@ -18,7 +18,7 @@ type proxyEntry struct {
}
type Controller struct {
proxies map[proxy.ID]*proxyEntry
proxies map[spec.ID]*proxyEntry
}
// Name implements node.Controller.
@ -28,9 +28,9 @@ func (c *Controller) Name() string {
// Reconcile implements node.Controller.
func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error {
proxySpec := proxy.NewSpec()
proxySpec := spec.NewSpec()
if err := state.GetSpec(proxy.NameProxy, proxySpec); err != nil {
if err := state.GetSpec(spec.NameProxy, proxySpec); err != nil {
if errors.Is(err, agent.ErrSpecNotFound) {
logger.Info(ctx, "could not find proxy spec")
@ -69,7 +69,7 @@ func (c *Controller) stopAllProxies(ctx context.Context) {
}
}
func (c *Controller) updateProxies(ctx context.Context, spec *proxy.Spec) {
func (c *Controller) updateProxies(ctx context.Context, spec *spec.Spec) {
// Stop and remove obsolete proxys
for proxyID, entry := range c.proxies {
if _, exists := spec.Proxies[proxyID]; exists {
@ -100,7 +100,7 @@ func (c *Controller) updateProxies(ctx context.Context, spec *proxy.Spec) {
}
}
func (c *Controller) updateProxy(ctx context.Context, proxyID proxy.ID, proxySpec proxy.ProxyEntry) (err error) {
func (c *Controller) updateProxy(ctx context.Context, proxyID spec.ID, proxySpec spec.ProxyEntry) (err error) {
newProxySpecHash, err := hashstructure.Hash(proxySpec, hashstructure.FormatV2, nil)
if err != nil {
return errors.WithStack(err)
@ -140,7 +140,7 @@ func (c *Controller) updateProxy(ctx context.Context, proxyID proxy.ID, proxySpe
)
}
options := make([]edgeProxy.OptionFunc, 0)
options := make([]proxy.OptionFunc, 0)
allowedHosts := make([]string, len(proxySpec.Mappings))
mappings := make(map[string]*url.URL, len(proxySpec.Mappings))
@ -156,8 +156,8 @@ func (c *Controller) updateProxy(ctx context.Context, proxyID proxy.ID, proxySpe
options = append(
options,
edgeProxy.WithAllowedHosts(allowedHosts...),
edgeProxy.WithRewriteHosts(mappings),
proxy.WithAllowedHosts(allowedHosts...),
proxy.WithRewriteHosts(mappings),
)
if err := entry.Proxy.Start(ctx, proxySpec.Address, options...); err != nil {
@ -173,7 +173,7 @@ func (c *Controller) updateProxy(ctx context.Context, proxyID proxy.ID, proxySpe
func NewController() *Controller {
return &Controller{
proxies: make(map[proxy.ID]*proxyEntry),
proxies: make(map[spec.ID]*proxyEntry),
}
}

View File

@ -5,10 +5,9 @@ import (
"net/http"
"sync"
"forge.cadoles.com/Cadoles/emissary/internal/proxy"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
"forge.cadoles.com/arcad/edge/pkg/proxy"
)
type ReverseProxy struct {

View File

@ -4,6 +4,7 @@ import (
"context"
"net/http"
"strings"
"time"
"forge.cadoles.com/Cadoles/emissary/internal/auth"
"forge.cadoles.com/Cadoles/emissary/internal/datastore"
@ -13,8 +14,11 @@ import (
"gitlab.com/wpetit/goweb/logger"
)
const DefaultAcceptableSkew = 5 * time.Minute
type Authenticator struct {
repo datastore.AgentRepository
repo datastore.AgentRepository
acceptableSkew time.Duration
}
// Authenticate implements auth.Authenticator.
@ -71,11 +75,19 @@ func (a *Authenticator) Authenticate(ctx context.Context, r *http.Request) (auth
[]byte(rawToken),
jwt.WithKeySet(agent.KeySet.Set, jws.WithRequireKid(false)),
jwt.WithValidate(true),
jwt.WithAcceptableSkew(a.acceptableSkew),
)
if err != nil {
return nil, errors.WithStack(err)
}
contactedAt := time.Now()
agent, err = a.repo.Update(ctx, agent.ID, datastore.WithAgentUpdateContactedAt(contactedAt))
if err != nil {
return nil, errors.WithStack(err)
}
user := &User{
agent: agent,
}
@ -83,9 +95,10 @@ func (a *Authenticator) Authenticate(ctx context.Context, r *http.Request) (auth
return user, nil
}
func NewAuthenticator(repo datastore.AgentRepository) *Authenticator {
func NewAuthenticator(repo datastore.AgentRepository, acceptableSkew time.Duration) *Authenticator {
return &Authenticator{
repo: repo,
repo: repo,
acceptableSkew: acceptableSkew,
}
}

View File

@ -18,7 +18,7 @@ func GenerateToken(key jwk.Key, thumbprint string) (string, error) {
return "", errors.WithStack(err)
}
now := time.Now()
now := time.Now().UTC()
if err := token.Set(jwt.NotBeforeKey, now); err != nil {
return "", errors.WithStack(err)

View File

@ -4,6 +4,7 @@ import (
"context"
"net/http"
"strings"
"time"
"forge.cadoles.com/Cadoles/emissary/internal/auth"
"forge.cadoles.com/Cadoles/emissary/internal/jwk"
@ -11,9 +12,12 @@ import (
"gitlab.com/wpetit/goweb/logger"
)
const DefaultAcceptableSkew = 5 * time.Minute
type Authenticator struct {
keys jwk.Set
issuer string
keys jwk.Set
issuer string
acceptableSkew time.Duration
}
// Authenticate implements auth.Authenticator.
@ -30,7 +34,7 @@ func (a *Authenticator) Authenticate(ctx context.Context, r *http.Request) (auth
return nil, errors.WithStack(auth.ErrUnauthenticated)
}
token, err := parseToken(ctx, a.keys, a.issuer, rawToken)
token, err := parseToken(ctx, a.keys, a.issuer, rawToken, a.acceptableSkew)
if err != nil {
return nil, errors.WithStack(err)
}
@ -57,10 +61,11 @@ func (a *Authenticator) Authenticate(ctx context.Context, r *http.Request) (auth
return user, nil
}
func NewAuthenticator(keys jwk.Set, issuer string) *Authenticator {
func NewAuthenticator(keys jwk.Set, issuer string, acceptableSkew time.Duration) *Authenticator {
return &Authenticator{
keys: keys,
issuer: issuer,
keys: keys,
issuer: issuer,
acceptableSkew: acceptableSkew,
}
}

View File

@ -13,12 +13,13 @@ import (
const keyRole = "role"
func parseToken(ctx context.Context, keys jwk.Set, issuer string, rawToken string) (jwt.Token, error) {
func parseToken(ctx context.Context, keys jwk.Set, issuer string, rawToken string, acceptableSkew time.Duration) (jwt.Token, error) {
token, err := jwt.Parse(
[]byte(rawToken),
jwt.WithKeySet(keys, jws.WithRequireKid(false)),
jwt.WithIssuer(issuer),
jwt.WithValidate(true),
jwt.WithAcceptableSkew(acceptableSkew),
)
if err != nil {
return nil, errors.WithStack(err)
@ -42,7 +43,7 @@ func GenerateToken(ctx context.Context, key jwk.Key, issuer, subject string, rol
return "", errors.WithStack(err)
}
now := time.Now()
now := time.Now().UTC()
if err := token.Set(jwt.NotBeforeKey, now); err != nil {
return "", errors.WithStack(err)

View File

@ -10,6 +10,7 @@ import (
type UpdateAgentOptions struct {
Status *int
Label *string
Options []OptionFunc
}
@ -21,6 +22,12 @@ func WithAgentStatus(status int) UpdateAgentOptionFunc {
}
}
func WithAgentLabel(label string) UpdateAgentOptionFunc {
return func(opts *UpdateAgentOptions) {
opts.Label = &label
}
}
func WithUpdateAgentsOptions(funcs ...OptionFunc) UpdateAgentOptionFunc {
return func(opts *UpdateAgentOptions) {
opts.Options = funcs
@ -39,6 +46,10 @@ func (c *Client) UpdateAgent(ctx context.Context, agentID datastore.AgentID, fun
payload["status"] = *opts.Status
}
if opts.Label != nil {
payload["label"] = *opts.Label
}
response := withResponse[struct {
Agent *datastore.Agent `json:"agent"`
}]()

View File

@ -49,10 +49,6 @@ func RunCommand() *cli.Command {
controllers = append(controllers, spec.NewController())
}
if ctrlConf.Proxy.Enabled {
controllers = append(controllers, proxy.NewController())
}
if ctrlConf.UCI.Enabled {
controllers = append(controllers, openwrt.NewUCIController(
string(ctrlConf.UCI.BinPath),
@ -66,6 +62,10 @@ func RunCommand() *cli.Command {
))
}
if ctrlConf.Proxy.Enabled {
controllers = append(controllers, proxy.NewController())
}
if ctrlConf.SysUpgrade.Enabled {
sysUpgradeArgs := make([]string, 0)
if len(ctrlConf.SysUpgrade.SysUpgradeCommand) > 1 {

View File

@ -22,6 +22,11 @@ func UpdateCommand() *cli.Command {
Usage: "Set `STATUS` to selected agent",
Value: -1,
},
&cli.StringFlag{
Name: "label",
Usage: "Set `LABEL` to selected agent",
Value: "",
},
),
Action: func(ctx *cli.Context) error {
baseFlags := clientFlag.GetBaseFlags(ctx)
@ -43,6 +48,11 @@ func UpdateCommand() *cli.Command {
options = append(options, client.WithAgentStatus(status))
}
label := ctx.String("label")
if label != "" {
options = append(options, client.WithAgentLabel(label))
}
client := client.New(baseFlags.ServerURL, client.WithToken(token))
agent, err := client.UpdateAgent(ctx.Context, agentID, options...)

View File

@ -7,9 +7,10 @@ func agentHints(outputMode format.OutputMode) format.Hints {
OutputMode: outputMode,
Props: []format.Prop{
format.NewProp("ID", "ID"),
format.NewProp("Label", "Label"),
format.NewProp("Thumbprint", "Thumbprint"),
format.NewProp("Status", "Status"),
format.NewProp("CreatedAt", "CreatedAt"),
format.NewProp("ContactedAt", "ContactedAt"),
format.NewProp("UpdatedAt", "UpdatedAt"),
},
}

View File

@ -15,6 +15,6 @@ type DatabaseConfig struct {
func NewDefaultDatabaseConfig() DatabaseConfig {
return DatabaseConfig{
Driver: "sqlite",
DSN: "sqlite://emissary.sqlite",
DSN: "sqlite://emissary.sqlite?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_txlock=immediate",
}
}

View File

@ -20,13 +20,15 @@ const (
)
type Agent struct {
ID AgentID `json:"id"`
Thumbprint string `json:"thumbprint"`
KeySet *SerializableKeySet `json:"keyset,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Status AgentStatus `json:"status"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID AgentID `json:"id"`
Label string `json:"label"`
Thumbprint string `json:"thumbprint"`
KeySet *SerializableKeySet `json:"keyset,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Status AgentStatus `json:"status"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ContactedAt *time.Time `json:"contactedAt,omitempty"`
}
type SerializableKeySet struct {

View File

@ -2,6 +2,7 @@ package datastore
import (
"context"
"time"
"github.com/lestrrat-go/jwx/v2/jwk"
)
@ -68,10 +69,12 @@ func WithAgentQueryThumbprints(thumbprints ...string) AgentQueryOptionFunc {
type AgentUpdateOptionFunc func(*AgentUpdateOptions)
type AgentUpdateOptions struct {
Status *AgentStatus
Metadata *map[string]any
KeySet *jwk.Set
Thumbprint *string
Label *string
Status *AgentStatus
ContactedAt *time.Time
Metadata *map[string]any
KeySet *jwk.Set
Thumbprint *string
}
func WithAgentUpdateStatus(status AgentStatus) AgentUpdateOptionFunc {
@ -97,3 +100,15 @@ func WithAgentUpdateThumbprint(thumbprint string) AgentUpdateOptionFunc {
opts.Thumbprint = &thumbprint
}
}
func WithAgentUpdateLabel(label string) AgentUpdateOptionFunc {
return func(opts *AgentUpdateOptions) {
opts.Label = &label
}
}
func WithAgentUpdateContactedAt(contactedAt time.Time) AgentUpdateOptionFunc {
return func(opts *AgentUpdateOptions) {
opts.ContactedAt = &contactedAt
}
}

View File

@ -45,7 +45,11 @@ func (r *AgentRepository) GetSpecs(ctx context.Context, agentID datastore.AgentI
return nil, errors.WithStack(err)
}
defer rows.Close()
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
for rows.Next() {
spec := &datastore.Spec{}
@ -61,6 +65,10 @@ func (r *AgentRepository) GetSpecs(ctx context.Context, agentID datastore.AgentI
specs = append(specs, spec)
}
if err := rows.Err(); err != nil {
return nil, errors.WithStack(err)
}
return specs, nil
}
@ -119,7 +127,7 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
count := 0
err := r.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT id, thumbprint, status, created_at, updated_at FROM agents`
query := `SELECT id, label, thumbprint, status, contacted_at, created_at, updated_at FROM agents`
limit := 10
if options.Limit != nil {
@ -176,22 +184,34 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
return errors.WithStack(err)
}
defer rows.Close()
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
for rows.Next() {
agent := &datastore.Agent{}
metadata := JSONMap{}
contactedAt := sql.NullTime{}
if err := rows.Scan(&agent.ID, &agent.Thumbprint, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
if err := rows.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &agent.Status, &contactedAt, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
return errors.WithStack(err)
}
agent.Metadata = metadata
if contactedAt.Valid {
agent.ContactedAt = &contactedAt.Time
}
agents = append(agents, agent)
}
if err := rows.Err(); err != nil {
return errors.WithStack(err)
}
row := tx.QueryRowContext(ctx, `SELECT count(id) FROM agents `+filters, args...)
if err := row.Scan(&count); err != nil {
return errors.WithStack(err)
@ -299,7 +319,7 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
err := r.withTx(ctx, func(tx *sql.Tx) error {
query := `
SELECT "id", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
SELECT "id", "label", "thumbprint", "keyset", "metadata", "status", "contacted_at", "created_at", "updated_at"
FROM agents
WHERE id = $1
`
@ -307,9 +327,10 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
row := r.db.QueryRowContext(ctx, query, id)
metadata := JSONMap{}
contactedAt := sql.NullTime{}
var rawKeySet []byte
if err := row.Scan(&agent.ID, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
if err := row.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &contactedAt, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return datastore.ErrNotFound
}
@ -318,6 +339,9 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
}
agent.Metadata = metadata
if contactedAt.Valid {
agent.ContactedAt = &contactedAt.Time
}
keySet := jwk.NewSet()
if err := json.Unmarshal(rawKeySet, &keySet); err != nil {
@ -346,15 +370,11 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
err := r.withTx(ctx, func(tx *sql.Tx) error {
query := `
UPDATE agents SET updated_at = $2
UPDATE agents SET id = $1
`
now := time.Now().UTC()
args := []any{
id, now,
}
index := 3
args := []any{id}
index := 2
if options.Status != nil {
query += fmt.Sprintf(`, status = $%d`, index)
@ -379,23 +399,51 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
index++
}
if options.Label != nil {
query += fmt.Sprintf(`, label = $%d`, index)
args = append(args, *options.Label)
index++
}
if options.ContactedAt != nil {
query += fmt.Sprintf(`, contacted_at = $%d`, index)
utc := options.ContactedAt.UTC()
args = append(args, utc)
index++
}
if options.Metadata != nil {
query += fmt.Sprintf(`, metadata = $%d`, index)
args = append(args, JSONMap(*options.Metadata))
index++
}
updated := options.Metadata != nil ||
options.Status != nil ||
options.Label != nil ||
options.KeySet != nil ||
options.Thumbprint != nil
if updated {
now := time.Now().UTC()
query += fmt.Sprintf(`, updated_at = $%d`, index)
args = append(args, now)
index++
}
query += `
WHERE id = $1
RETURNING "id", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
RETURNING "id", "label", "thumbprint", "keyset", "metadata", "status", "contacted_at", "created_at", "updated_at"
`
logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args))
row := tx.QueryRowContext(ctx, query, args...)
metadata := JSONMap{}
contactedAt := sql.NullTime{}
var rawKeySet []byte
if err := row.Scan(&agent.ID, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
if err := row.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &contactedAt, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return datastore.ErrNotFound
}
@ -404,6 +452,9 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
}
agent.Metadata = metadata
if contactedAt.Valid {
agent.ContactedAt = &contactedAt.Time
}
keySet := jwk.NewSet()
if err := json.Unmarshal(rawKeySet, &keySet); err != nil {

View File

@ -1,8 +1,8 @@
package spec
import (
_ "forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
_ "forge.cadoles.com/Cadoles/emissary/internal/agent/controller/openwrt/spec/sysupgrade"
_ "forge.cadoles.com/Cadoles/emissary/internal/spec/app"
_ "forge.cadoles.com/Cadoles/emissary/internal/spec/proxy"
_ "forge.cadoles.com/Cadoles/emissary/internal/spec/uci"
)

View File

@ -23,6 +23,13 @@ type (
ParseOption = jwk.ParseOption
)
var (
FromRaw = jwk.FromRaw
NewSet = jwk.NewSet
)
const AlgorithmKey = jwk.AlgorithmKey
func Parse(src []byte, options ...jwk.ParseOption) (Set, error) {
return jwk.Parse(src, options...)
}

View File

@ -0,0 +1,29 @@
package proxy
import (
"net/http"
"forge.cadoles.com/Cadoles/emissary/internal/proxy/wildcard"
)
func FilterHosts(allowedHostPatterns ...string) Middleware {
return func(h http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
if matches := wildcard.MatchAny(r.Host, allowedHostPatterns...); !matches {
http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden)
return
}
h.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}
func WithAllowedHosts(allowedHostPatterns ...string) OptionFunc {
return func(o *Options) {
o.Middlewares = append(o.Middlewares, FilterHosts(allowedHostPatterns...))
}
}

View File

@ -0,0 +1,66 @@
package proxy
import (
"net/http"
"net/url"
"sort"
"forge.cadoles.com/Cadoles/emissary/internal/proxy/wildcard"
"gitlab.com/wpetit/goweb/logger"
)
func RewriteHosts(mappings map[string]*url.URL) Middleware {
patterns := make([]string, len(mappings))
for p := range mappings {
patterns = append(patterns, p)
}
sort.Strings(patterns)
reverse(patterns)
return func(h http.Handler) http.Handler {
fn := func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var match *url.URL
for _, p := range patterns {
logger.Debug(ctx, "matching host to pattern", logger.F("host", r.Host), logger.F("pattern", p))
if matches := wildcard.Match(r.Host, p); !matches {
continue
}
match = mappings[p]
break
}
if match == nil {
h.ServeHTTP(w, r)
return
}
ctx = logger.With(ctx, logger.F("originalHost", r.Host))
r = r.WithContext(ctx)
originalURL := r.URL.String()
r.URL.Host = match.Host
r.URL.Scheme = match.Scheme
logger.Debug(ctx, "rewriting url", logger.F("from", originalURL), logger.F("to", r.URL.String()))
h.ServeHTTP(w, r)
}
return http.HandlerFunc(fn)
}
}
func WithRewriteHosts(mappings map[string]*url.URL) OptionFunc {
return func(o *Options) {
o.Middlewares = append(o.Middlewares, RewriteHosts(mappings))
}
}

View File

@ -0,0 +1,33 @@
package proxy
import "net/http"
type Middleware func(h http.Handler) http.Handler
type ProxyResponseTransformer interface {
TransformResponse(*http.Response) error
}
type defaultProxyResponseTransformer struct{}
// TransformResponse implements ProxyResponseTransformer
func (*defaultProxyResponseTransformer) TransformResponse(*http.Response) error {
return nil
}
var _ ProxyResponseTransformer = &defaultProxyResponseTransformer{}
type ProxyResponseMiddleware func(ProxyResponseTransformer) ProxyResponseTransformer
type ProxyRequestTransformer interface {
TransformRequest(*http.Request)
}
type ProxyRequestMiddleware func(ProxyRequestTransformer) ProxyRequestTransformer
type defaultProxyRequestTransformer struct{}
// TransformRequest implements ProxyRequestTransformer
func (*defaultProxyRequestTransformer) TransformRequest(*http.Request) {}
var _ ProxyRequestTransformer = &defaultProxyRequestTransformer{}

29
internal/proxy/options.go Normal file
View File

@ -0,0 +1,29 @@
package proxy
type Options struct {
Middlewares []Middleware
ProxyRequestMiddlewares []ProxyRequestMiddleware
ProxyResponseMiddlewares []ProxyResponseMiddleware
}
func defaultOptions() *Options {
return &Options{
Middlewares: make([]Middleware, 0),
ProxyRequestMiddlewares: make([]ProxyRequestMiddleware, 0),
ProxyResponseMiddlewares: make([]ProxyResponseMiddleware, 0),
}
}
type OptionFunc func(*Options)
func WithProxyRequestMiddlewares(middlewares ...ProxyRequestMiddleware) OptionFunc {
return func(o *Options) {
o.ProxyRequestMiddlewares = middlewares
}
}
func WithproxyResponseMiddlewares(middlewares ...ProxyResponseMiddleware) OptionFunc {
return func(o *Options) {
o.ProxyResponseMiddlewares = middlewares
}
}

131
internal/proxy/proxy.go Normal file
View File

@ -0,0 +1,131 @@
package proxy
import (
"fmt"
"net/http"
"net/http/httputil"
"net/url"
"sync"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type Proxy struct {
reversers sync.Map
handler http.Handler
proxyResponseTransformer ProxyResponseTransformer
proxyRequestTransformer ProxyRequestTransformer
}
// ServeHTTP implements http.Handler
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
p.handler.ServeHTTP(w, r)
}
func (p *Proxy) proxyRequest(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var reverser *httputil.ReverseProxy
key := fmt.Sprintf("%s://%s", r.URL.Scheme, r.URL.Host)
createAndStore := func() {
target := &url.URL{
Scheme: r.URL.Scheme,
Host: r.URL.Host,
}
reverser = httputil.NewSingleHostReverseProxy(target)
originalDirector := reverser.Director
if p.proxyRequestTransformer != nil {
reverser.Director = func(r *http.Request) {
originalURL := r.URL.String()
originalDirector(r)
p.proxyRequestTransformer.TransformRequest(r)
logger.Debug(ctx, "proxying request", logger.F("targetURL", r.URL.String()), logger.F("originalURL", originalURL))
}
}
if p.proxyResponseTransformer != nil {
reverser.ModifyResponse = func(r *http.Response) error {
if err := p.proxyResponseTransformer.TransformResponse(r); err != nil {
return errors.WithStack(err)
}
return nil
}
}
p.reversers.Store(key, reverser)
}
raw, exists := p.reversers.Load(key)
if !exists {
createAndStore()
}
reverser, ok := raw.(*httputil.ReverseProxy)
if !ok {
createAndStore()
}
reverser.ServeHTTP(w, r)
}
func New(funcs ...OptionFunc) *Proxy {
opts := defaultOptions()
for _, fn := range funcs {
fn(opts)
}
proxy := &Proxy{}
handler := http.HandlerFunc(proxy.proxyRequest)
proxy.handler = createMiddlewareChain(handler, opts.Middlewares)
proxy.proxyRequestTransformer = createProxyRequestChain(&defaultProxyRequestTransformer{}, opts.ProxyRequestMiddlewares)
proxy.proxyResponseTransformer = createProxyResponseChain(&defaultProxyResponseTransformer{}, opts.ProxyResponseMiddlewares)
return proxy
}
var _ http.Handler = &Proxy{}
func createMiddlewareChain(handler http.Handler, middlewares []Middleware) http.Handler {
reverse(middlewares)
for _, m := range middlewares {
handler = m(handler)
}
return handler
}
func createProxyResponseChain(transformer ProxyResponseTransformer, middlewares []ProxyResponseMiddleware) ProxyResponseTransformer {
reverse(middlewares)
for _, m := range middlewares {
transformer = m(transformer)
}
return transformer
}
func createProxyRequestChain(transformer ProxyRequestTransformer, middlewares []ProxyRequestMiddleware) ProxyRequestTransformer {
reverse(middlewares)
for _, m := range middlewares {
transformer = m(transformer)
}
return transformer
}
func reverse[S ~[]E, E any](s S) {
for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
s[i], s[j] = s[j], s[i]
}
}

View File

@ -0,0 +1,44 @@
package wildcard
const wildcard = '*'
func Match(str, pattern string) bool {
if pattern == "" {
return str == pattern
}
if pattern == string(wildcard) {
return true
}
return deepMatchRune([]rune(str), []rune(pattern))
}
func MatchAny(str string, patterns ...string) bool {
for _, p := range patterns {
if matches := Match(str, p); matches {
return matches
}
}
return false
}
func deepMatchRune(str, pattern []rune) bool {
for len(pattern) > 0 {
switch pattern[0] {
default:
if len(str) == 0 || str[0] != pattern[0] {
return false
}
case wildcard:
return deepMatchRune(str, pattern[1:]) ||
(len(str) > 0 && deepMatchRune(str[1:], pattern))
}
str = str[1:]
pattern = pattern[1:]
}
return len(str) == 0 && len(pattern) == 0
}

View File

@ -145,6 +145,7 @@ func (s *Server) registerAgent(w http.ResponseWriter, r *http.Request) {
type updateAgentRequest struct {
Status *datastore.AgentStatus `json:"status" validate:"omitempty,oneof=0 1 2 3"`
Label *string `json:"label" validate:"omitempty"`
}
func (s *Server) updateAgent(w http.ResponseWriter, r *http.Request) {
@ -166,6 +167,10 @@ func (s *Server) updateAgent(w http.ResponseWriter, r *http.Request) {
options = append(options, datastore.WithAgentUpdateStatus(*updateAgentReq.Status))
}
if updateAgentReq.Label != nil {
options = append(options, datastore.WithAgentUpdateLabel(*updateAgentReq.Label))
}
agent, err := s.agentRepo.Update(
ctx,
datastore.AgentID(agentID),

View File

@ -105,8 +105,8 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e
r.Group(func(r chi.Router) {
r.Use(auth.Middleware(
thirdparty.NewAuthenticator(keys, string(s.conf.Issuer)),
agent.NewAuthenticator(s.agentRepo),
thirdparty.NewAuthenticator(keys, string(s.conf.Issuer), thirdparty.DefaultAcceptableSkew),
agent.NewAuthenticator(s.agentRepo, agent.DefaultAcceptableSkew),
))
r.Route("/agents", func(r chi.Router) {

View File

@ -0,0 +1 @@
ALTER TABLE agents DROP COLUMN label;

View File

@ -0,0 +1 @@
ALTER TABLE agents ADD COLUMN label TEXT DEFAULT "";

View File

@ -0,0 +1 @@
ALTER TABLE agents DROP COLUMN contacted_at;

View File

@ -0,0 +1 @@
ALTER TABLE agents ADD COLUMN contacted_at datetime;

24
misc/jenkins/Dockerfile Normal file
View File

@ -0,0 +1,24 @@
FROM reg.cadoles.com/proxy_cache/library/ubuntu:22.04
ARG HTTP_PROXY=
ARG HTTPS_PROXY=
ARG http_proxy=
ARG https_proxy=
ARG GO_VERSION=1.19.2
# Install dev environment dependencies
RUN export DEBIAN_FRONTEND=noninteractive &&\
apt-get update -y &&\
apt-get install -y --no-install-recommends curl ca-certificates build-essential wget unzip tar git jq
# Install Go
RUN mkdir -p /tmp \
&& wget -O /tmp/go${GO_VERSION}.linux-amd64.tar.gz https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \
&& rm -rf /usr/local/go \
&& mkdir -p /usr/local \
&& tar -C /usr/local -xzf /tmp/go${GO_VERSION}.linux-amd64.tar.gz
ENV PATH="${PATH}:/usr/local/go/bin"
# Add LetsEncrypt certificates
RUN curl -k https://forge.cadoles.com/Cadoles/Jenkins/raw/branch/master/resources/com/cadoles/common/add-letsencrypt-ca.sh | bash

View File

@ -9,7 +9,7 @@ server:
port: 3000
database:
driver: sqlite
dsn: sqlite:///var/lib/emissary/data.sqlite
dsn: sqlite:///var/lib/emissary/data.sqlite?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_txlock=immediate
cors:
allowedOrigins: []
allowCredentials: true

View File

@ -1,10 +0,0 @@
{
"apps": {
"edge.sdk.client.test": {
"url": "http://localhost:3001/edge.sdk.client.test_0.0.0.zip",
"sha256sum": "58019192dacdae17755707719707db007e26dac856102280583fbd18427dd352",
"format": "zip",
"address": "127.0.0.1:8081"
}
}
}

View File

@ -1,65 +0,0 @@
@baseUrl = http://localhost:3000
### Get agents
# @name getAgents
GET {{ baseUrl }}/api/v1/agents
Content-Type: application/json
@agentId = {{ getAgents.response.body.Data.Agents.0.ID }}
### Update an agent (accept it)
PUT {{ baseUrl }}/api/v1/agents/{{ agentId }}
Content-Type: application/json
{
"Status": 1
}
### Get an agent
GET {{ baseUrl }}/api/v1/agents/{{ agentId }}
Content-Type: application/json
### Get an agent specs
# @name getSpecs
GET {{ baseUrl }}/api/v1/agents/{{ agentId }}/specs
Content-Type: application/json
@specName = {{ getSpecs.response.body.Data.Specs.0.Name }}
### Update an agent specs
POST {{ baseUrl }}/api/v1/agents/{{ agentId }}/specs
Content-Type: application/json
{
"Name": "gateway.emissary.cadoles.com",
"Revision": 2,
"Data": {
"gateways": {
"cadoles.com":{
"address":":3003",
"target":"https://www.cadoles.com"
}
}
}
}
### Delete an agent spec
DELETE {{ baseUrl }}/api/v1/agents/{{ agentId }}/specs
Content-Type: application/json
{
"Name": "gateway.emissary.cadoles.com"
}
### Update UCI spec with uhttpd config
POST {{ baseUrl }}/api/v1/agents/2/specs
Content-Type: application/json
< ./uci-spec.payload.json

View File

@ -1,163 +0,0 @@
{
"Name": "uci.emissary.cadoles.com",
"Revision": 6,
"Data": {
"config": {
"packages": [
{
"name": "uhttpd",
"configs": [
{
"name": "uhttpd",
"section": "main",
"options": [
{
"type": "list",
"name": "listen_http",
"value": "0.0.0.0:8080"
},
{
"type": "list",
"name": "listen_http",
"value": "[::]:8080"
},
{
"type": "list",
"name": "listen_https",
"value": "0.0.0.0:8443"
},
{
"type": "list",
"name": "listen_https",
"value": "[::]:8443"
},
{
"type": "option",
"name": "redirect_https",
"value": "0"
},
{
"type": "option",
"name": "home",
"value": "/www"
},
{
"type": "option",
"name": "rfc1918_filter",
"value": "1"
},
{
"type": "option",
"name": "max_requests",
"value": "3"
},
{
"type": "option",
"name": "max_connections",
"value": "100"
},
{
"type": "option",
"name": "cert",
"value": "/etc/uhttpd.crt"
},
{
"type": "option",
"name": "key",
"value": "/etc/uhttpd.key"
},
{
"type": "option",
"name": "cgi_prefix",
"value": "/cgi-bin"
},
{
"type": "list",
"name": "lua_prefix",
"value": "/cgi-bin/luci=/usr/lib/lua/luci/sgi/uhttpd.lua"
},
{
"type": "option",
"name": "script_timeout",
"value": "60"
},
{
"type": "option",
"name": "network_timeout",
"value": "30"
},
{
"type": "option",
"name": "http_keepalive",
"value": "20"
},
{
"type": "option",
"name": "tcp_keepalive",
"value": "1"
},
{
"type": "option",
"name": "ubus_prefix",
"value": "/ubus"
}
]
},
{
"name": "cert",
"section": "defaults",
"options": [
{
"type": "option",
"name": "days",
"value": "730"
},
{
"type": "option",
"name": "key_type",
"value": "ec"
},
{
"type": "option",
"name": "bits",
"value": "2048"
},
{
"type": "option",
"name": "ec_curve",
"value": "P-256"
},
{
"type": "option",
"name": "country",
"value": "ZZ"
},
{
"type": "option",
"name": "state",
"value": "Somewhere"
},
{
"type": "option",
"name": "location",
"value": "Unknown"
},
{
"type": "option",
"name": "commonname",
"value": "OpenWrt"
}
]
}
]
}
]
},
"postImportCommands": [
{
"command": "reload_config",
"args": []
}
]
}
}

View File

@ -0,0 +1,51 @@
{
"apps": {
"portal": {
"url": "https://emissary.cadol.es/files/apps/arcad.portal_v2023.3.28-3feda80.zip",
"sha256sum": "921402c44a5fa554d5b630d1284957b05416aa6872b402314cf52e964e06fac5",
"address": "127.0.0.1:8082",
"format": "zip"
},
"hextris": {
"url": "https://emissary.cadol.es/files/apps/app.arcad.edge.hextris_v2023.3.22-33ece28.zip",
"sha256sum": "5f9f3c8d6f22796beb051d747d7ff12efa17af9d1552c0ab08baef13703a2aba",
"address": "127.0.0.1:8083",
"format": "zip"
},
"test": {
"url": "https://emissary.cadol.es/files/apps/edge.sdk.client.test_v2023.3.24-ed535b6.zip",
"sha256sum": "e97b7b79159bb5d6a13b05644c091272b02a1a3cbb1b613dd5eda37e1eb84623",
"address": "127.0.0.1:8084",
"format": "zip"
},
"diffusion": {
"url": "https://emissary.cadol.es/files/apps/arcad.diffusion_v2023.3.29-5b3fab4.zip",
"sha256sum": "1282e75719beedbc7c7e67879389d0f3e11c86d3d2c37cf13da624a66faaeb58",
"address": "127.0.0.1:8085",
"format": "zip"
}
},
"config": {
"appUrlTemplate": "http://{{ last ( splitList \".\" ( toString .Manifest.ID ) ) }}.arcad.local:8080",
"auth": {
"local": {
"key": "absolutlynotsecret",
"cookieDomain": ".arcad.local",
"cookieDuration": "1h",
"accounts": [
{
"username": "admin",
"algo": "plain",
"password": "admin",
"claims": {
"arcad_role": "admin",
"arcad_tenant": "x86",
"preferred_username": "Admin",
"sub": "admin"
}
}
]
}
}
}
}

View File

@ -0,0 +1,29 @@
{
"proxies": {
"main": {
"address": ":8080",
"mappings": [
{
"hostPattern": "portal.arcad.local:*",
"target": "http://localhost:8082"
},
{
"hostPattern": "hextris.arcad.local:*",
"target": "http://localhost:8083"
},
{
"hostPattern": "test.arcad.local:*",
"target": "http://localhost:8084"
},
{
"hostPattern": "diffusion.arcad.local:*",
"target": "http://localhost:8085"
},
{
"hostPattern": "*",
"target": "http://localhost:8082"
}
]
}
}
}

View File

@ -1,5 +1,5 @@
**/*.go
**/*.json
internal/**/*.json
modd.conf
tmp/config.yml
.env {