From 6cddfb9a1f6c588205ccc46071435fc0ed8bd577 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Wed, 15 Mar 2023 15:34:52 +0100 Subject: [PATCH] implement consul config --- README.md | 32 ++++++-- example/albatros.json | 4 + go.mod | 11 +++ go.sum | 93 ++++++++++++++++++++- main.go | 187 ++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 309 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 89db61f..21cd375 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,7 @@ we might do it in Albatros... ## Deploy -Requirements: Nomad, Consul -Optional: Gitea +Requirements: Nomad, Consul, Gitea ``` nomad namespace apply -description "Continuous Integration" ci @@ -22,6 +21,8 @@ nomad run hcl/builder.hcl go run main.go ``` +*Note: in the future, Gitea might become optional*. + ## Register a build Add to Consul a key in albatros hierarchy @@ -35,11 +36,17 @@ The key must contain a JSON file with your desired token, trust conditions, and ```json { - "token": "s3cr3t", - "trusted_if": { - "sender": [ "quentin", "lx" ] - } - "secret": "SECRET1=xx\nSECRET2=yy", + "hook": { + "token": "s3cr3t" + }, + "gitea": { + "url": "https://git.deuxfleurs.fr", + "token": "c0ffee..." + }, + "trusted": { + "senders": [ "quentin", "lx" ] + }, + "inject": "SECRET1=xx\nSECRET2=yy" } ``` @@ -63,7 +70,8 @@ can use it to follow your logs (don't forget to urlencode it): ``` $ curl http://localhost:8080/build?job=builder%2Fdispatch-1678866433-15aad86a&log=stderr - ++ go build +... ``` Of course, most of that will be handled by Gitea. @@ -93,7 +101,11 @@ SECRET_PATH=/var/run/secrets/albatros/secret.txt ## Gitea integration -*Todo* +Register an access token for your Albatros application in Gitea. +The URL is `/user/settings/applications`. + +Name your app `albatros` for example, the token will be displayed once +in light blue. Store it in your vault. ## Security model @@ -122,6 +134,8 @@ will be executed. ## Ideas +- [ ] Validate the gitea payload with hmac - [ ] Register the builder programatically - [ ] Allow users to define their own set of builders (ones with more CPU+RAM, etc.) - [ ] Allow users to choose their image/rootfs +- [ ] Make Gitea optional diff --git a/example/albatros.json b/example/albatros.json index b0b93fe..c8614d5 100644 --- a/example/albatros.json +++ b/example/albatros.json @@ -2,6 +2,10 @@ "ref": "refs/heads/main", "after": "101f621d5b87147b6c2d8fca0615d3004cff4d7b", "repository": { + "owner": { + "username": "quentin" + }, + "name": "albatros", "clone_url": "https://git.deuxfleurs.fr/quentin/albatros.git" } } diff --git a/go.mod b/go.mod index 289ff1e..9d9ffdd 100644 --- a/go.mod +++ b/go.mod @@ -4,19 +4,30 @@ go 1.18 require ( code.gitea.io/sdk/gitea v0.15.1 + github.com/caarlos0/env/v7 v7.1.0 + github.com/hashicorp/consul/api v1.20.0 github.com/hashicorp/nomad/api v0.0.0-20230314144600-1a01e8719272 ) require ( + github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect + github.com/fatih/color v1.9.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/cronexpr v1.1.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v0.12.0 // indirect + github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-version v1.2.1 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/mattn/go-colorable v0.1.6 // indirect + github.com/mattn/go-isatty v0.0.12 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a // indirect + golang.org/x/sys v0.1.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 2dbf3a8..fa9c8e4 100644 --- a/go.sum +++ b/go.sum @@ -1,51 +1,142 @@ code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= code.gitea.io/sdk/gitea v0.15.1 h1:WJreC7YYuxbn0UDaPuWIe/mtiNKTvLN8MLkaw71yx/M= code.gitea.io/sdk/gitea v0.15.1/go.mod h1:klY2LVI3s3NChzIk/MzMn7G1FHrfU7qd63iSMVoHRBA= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/caarlos0/env/v7 v7.1.0 h1:9lzTF5amyQeWHZzuZeKlCb5FWSUxpG1js43mhbY8ozg= +github.com/caarlos0/env/v7 v7.1.0/go.mod h1:LPPWniDUq4JaO6Q41vtlyikhMknqymCLBw0eX4dcH1E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/consul/api v1.20.0 h1:9IHTjNVSZ7MIwjlW3N3a7iGiykCMDpxZu8jsxFJh0yc= +github.com/hashicorp/consul/api v1.20.0/go.mod h1:nR64eD44KQ59Of/ECwt2vUmIK2DKsDzAwTmwmLl8Wpo= +github.com/hashicorp/consul/sdk v0.13.1 h1:EygWVWWMczTzXGpO93awkHFzfUka6hLYJ0qhETd+6lY= github.com/hashicorp/cronexpr v1.1.1 h1:NJZDd87hGXjoZBdvyCF9mX4DCq5Wy7+A/w+A7q0wn6c= github.com/hashicorp/cronexpr v1.1.1/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0 h1:d4QkX8FRTYaKaCZBoXYY8zJX2BXjWxurN/GA2tkrmZM= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= +github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= github.com/hashicorp/nomad/api v0.0.0-20230314144600-1a01e8719272 h1:e8BbZuEbMCuuxZnY77a0dCJQ1TAQ+PNmXGYC4B7AXJk= github.com/hashicorp/nomad/api v0.0.0-20230314144600-1a01e8719272/go.mod h1:bKUb1ytds5KwUioHdvdq9jmrDqCThv95si0Ub7iNeBg= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/shoenig/test v0.6.2 h1:tdq+WGnznwE5xcOMXkqqXuudK75RkSGBazBGcP1lX6w= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a h1:tlXy25amD5A7gOfbXdqCGN5k8ESEed/Ee1E5RcrYnqU= golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200325010219-a49f79bcc224/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index 2800f0f..4eac0b6 100644 --- a/main.go +++ b/main.go @@ -1,14 +1,16 @@ package main - import ( + b64 "encoding/base64" "encoding/json" "fmt" nomad "github.com/hashicorp/nomad/api" + consul "github.com/hashicorp/consul/api" "io" "log" "net/http" "strings" - //"code.gitea.io/sdk/gitea" + "github.com/caarlos0/env/v7" + "code.gitea.io/sdk/gitea" ) type GitUser struct { @@ -38,6 +40,8 @@ type GiteaAccount struct { type GiteaRepository struct { Id int64 `json:"id"` Owner GiteaAccount `json:"owner"` + Name string `json:"name"` + FullName string `json:"full_name"` Description string `json:"description"` Private bool `json:"private"` Fork bool `json:"private"` @@ -66,21 +70,128 @@ type GiteaNotification struct { Sender GiteaAccount `json:"sender"` } +type SecretHook struct { + Token string `json:"token"` +} + +type SecretGitea struct { + Url string `json:"url"` + Token string `json:"token"` +} + +type SecretTrusted struct { + Senders []string `json:"senders"` +} + +type ConsulSecret struct { + Hook SecretHook `json:"hook"` + Gitea SecretGitea `json:"gitea"` + Trusted SecretTrusted `json:"trusted"` + Inject string `json:"inject"` +} + +func nomadToGiteaStatus(summary *nomad.TaskGroupSummary) gitea.StatusState { + if summary.Failed > 0 { + return gitea.StatusError + } + if summary.Lost > 0 || summary.Unknown > 0 { + return gitea.StatusFailure + } + if summary.Queued > 0 || summary.Starting > 0 || summary.Running > 0 { + return gitea.StatusPending + } + if summary.Complete > 0 { + return gitea.StatusSuccess + } + // When the job is just started, all the counters are = 0. + return gitea.StatusPending +} + +func lifecycle(notification *GiteaNotification, dispatch *nomad.JobDispatchResponse, giteaCreds *SecretGitea) { + log.Printf("[lifecyle] Gitea URL: %s\n", giteaCreds.Url) + // init Gitea + forge, err := gitea.NewClient(giteaCreds.Url, gitea.SetToken(giteaCreds.Token)) + if err != nil { + log.Printf("Unable to create gitea client: %+v\n", err) + return + } + + // get job's deployment + jobs := NomadClient.Jobs() + queryOpt := nomad.QueryOptions{ + AllowStale: true, + } + + safeguard := 1000 + for ; safeguard > 0; safeguard-- { + // Blocking fetch on deployment info + job, meta, err := jobs.Summary(dispatch.DispatchedJobID, &queryOpt) + if err != nil { + log.Printf("[lifecycle] can't fetch job: %+v\n", err) + break + } + queryOpt.WaitIndex = meta.LastIndex + + summary, ok := job.Summary["runner"]; + if !ok { + log.Printf("[lifecycle] your job %s must contain a 'runner' task\n", job.JobID) + break + } + log.Printf("[lifecycle] Summary for job %s: %+v\n", job.JobID, summary) + + // Compute new job state + state := nomadToGiteaStatus(&summary) + + // Try updating Gitea commit status + _, _, err = forge.CreateStatus( + notification.Repository.Owner.Username, + notification.Repository.Name, + notification.After, + gitea.CreateStatusOption { + State: state, + TargetURL: GlobalConfig.AlbatrosURL + "/build?log=stderr&job=" + dispatch.DispatchedJobID, + Description: "build", + Context: "Albatros", + }) + + if err != nil { + log.Printf( + "[lifecycle] can't update gitea repo %s/%s:%s: %+v\n", + notification.Repository.Owner.Username, + notification.Repository.Name, + notification.After, + err) + } + + // Continue the loop only if the job is pending + if state != gitea.StatusPending { + log.Printf("Job %s teminated with status %s\n", job.JobID, state) + break + } + } + + if safeguard == 0 { + // To avoid dangerous infinite loops, we put an upperbound here + // of 1k refresh here. Reaching this limit will allow us to know + // that something did not work as expected... + log.Println("!!! [lifecycle] we refreshed 1k times this deployment and it's still running, giving up...") + } +} + func hook(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "Hook only support POST requests", http.StatusBadRequest) } q := r.URL.Query() - token, ok := q["token"] - if !ok || len(token) < 1 { + maybeToken, ok := q["token"] + if !ok || len(maybeToken) < 1 { http.Error(w, "Missing query parameter 'token'. Try adding '?token=xxx'", http.StatusBadRequest) return } + token := maybeToken[0] flavor := "default" - //@FIXME check for token in consul - var notification GiteaNotification dec := json.NewDecoder(r.Body) if err := dec.Decode(¬ification); err != nil { @@ -89,6 +200,34 @@ func hook(w http.ResponseWriter, r *http.Request) { } log.Printf("Gitea notification: %+v\n", notification) + + // Fetch our repo descriptor + kv := ConsulClient.KV() + encodedRepoUrl := b64.StdEncoding.EncodeToString([]byte(notification.Repository.CloneUrl)) + key := "albatros/"+encodedRepoUrl + log.Printf("Fetching key %s\n", key) + pair, _, err := kv.Get(key, nil) + if err != nil { + http.Error(w, "Can't fetch the repo descriptor in Consul", http.StatusInternalServerError) + return + } + if pair == nil || pair.Value == nil { + http.Error(w, "You must declare your repo in Consul in order to build it", http.StatusForbidden) + return + } + // Parse our repo descriptor + var repoDesc ConsulSecret + if err = json.Unmarshal(pair.Value, &repoDesc); err != nil { + http.Error(w, "Can't decode your Consul configuration for this repo", http.StatusInternalServerError) + return + } + // Check token + if repoDesc.Hook.Token != token { + http.Error(w, "Your albatros token does not match the one declared in Consul", http.StatusForbidden) + return + } + + // Build job parameters for Nomad meta := map[string]string{ "REPO_URL": notification.Repository.CloneUrl, "COMMIT": notification.After, @@ -109,6 +248,10 @@ func hook(w http.ResponseWriter, r *http.Request) { log.Printf("Query info: %+v\n", dmeta) log.Printf("Job info: %+v\n", dres) + // Start a lifecycle observer to update gitea status + // @FIXME: need to inject gitea descriptor + go lifecycle(¬ification, dres, &repoDesc.Gitea) + io.WriteString(w, dres.DispatchedJobID) } @@ -153,6 +296,7 @@ func build(w http.ResponseWriter, r *http.Request) { scancel := make(chan struct{}) sframe, serr := allocFS.Logs(myAlloc, true, "runner", logFilter, "start", 0, scancel, &nomad.QueryOptions{}) + // stream logs to client's browser build_loop: for { select { @@ -185,21 +329,48 @@ func build(w http.ResponseWriter, r *http.Request) { } var NomadClient *nomad.Client +var ConsulClient *consul.Client + +type config struct { + AlbatrosURL string `env:"ALBATROS_URL,required"` + // @TODO get nomad config from env + // @TODO get consul config from env +} +var GlobalConfig config func main() { var err error + + // init config + if err = env.Parse(&GlobalConfig); err != nil { + log.Fatal(fmt.Sprintf("unable to parse config, error: %+v\n", err)) + return + } + log.Printf("Albatros public URL: %s\n", GlobalConfig.AlbatrosURL) + + // init nomad nomadConfig := nomad.DefaultConfig() nomadConfig.Namespace = "ci" - // @TODO read env for encrypted parameters NomadClient, err = nomad.NewClient(nomadConfig) if err != nil { log.Fatal("Unable to connect to Nomad, check your config and setup") + return } + // init consul + consulConfig := consul.DefaultConfig() + ConsulClient, err = consul.NewClient(consulConfig) + if err != nil { + log.Fatal("Unable to connect to Consul, check your config and setup") + return + } + + // init webserver http.HandleFunc("/hook", hook) http.HandleFunc("/build", build) - fmt.Println("albatros (:8080)") + // launch + log.Println("Listen on :8080") if err = http.ListenAndServe(":8080", nil); err != nil { log.Fatal("Can't start HTTP server") }