diff --git a/bin/alba.go b/bin/alba.go index 59cf5aa..b19b49d 100644 --- a/bin/alba.go +++ b/bin/alba.go @@ -2,13 +2,13 @@ package main import ( "fmt" - "os" "git.deuxfleurs.fr/deuxfleurs/albatros/cmd" + "os" ) func main() { if err := cmd.RootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) + fmt.Println(err) + os.Exit(1) } } diff --git a/bin/albatros.go b/bin/albatros.go index 8e0c82c..c8f6503 100644 --- a/bin/albatros.go +++ b/bin/albatros.go @@ -1,10 +1,10 @@ package main import ( - "log" - "net/http" + "log" + "net/http" - "git.deuxfleurs.fr/deuxfleurs/albatros/pkg" + "git.deuxfleurs.fr/deuxfleurs/albatros/pkg" ) func main() { diff --git a/cmd/container.go b/cmd/container.go index 4580b1a..f52ba5e 100644 --- a/cmd/container.go +++ b/cmd/container.go @@ -1,10 +1,10 @@ package cmd import ( - "crypto/sha256" - "errors" "context" + "crypto/sha256" "encoding/json" + "errors" "fmt" "io" "os" @@ -13,26 +13,27 @@ import ( "strconv" "strings" - "github.com/containers/image/v5/transports/alltransports" - "github.com/containers/image/v5/signature" "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/signature" + "github.com/containers/image/v5/transports/alltransports" "github.com/spf13/cobra" "gocloud.dev/blob" _ "gocloud.dev/blob/s3blob" ) var pctx *signature.PolicyContext -const distributionPrefix = "v2" +const distributionPrefix = "v2" //--- //--- Image converter type OCIImageManifest struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Config OCIRef `json:"config"` - Layers []OCIRef `json:layers"` + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config OCIRef `json:"config"` + Layers []OCIRef `json:layers"` } + func LoadOCIImageManifest(path string) (*OCIImageManifest, error) { fd, err := os.Open(path) if err != nil { @@ -56,26 +57,27 @@ func LoadOCIImageManifest(path string) (*OCIImageManifest, error) { type OCIPlatform struct { Architecture string `json:"architecture"` - OS string `json:"os"` + OS string `json:"os"` } type OCIRef struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int `json:"size"` - Platform *OCIPlatform `json:"platform,omitempty"` + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int `json:"size"` + Platform *OCIPlatform `json:"platform,omitempty"` } type OCIImageIndex struct { - SchemaVersion int `json:"schemaVersion"` - MediaType string `json:"mediaType"` - Manifests []OCIRef `json:"manifests"` + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Manifests []OCIRef `json:"manifests"` } + func NewOCIImageIndex(o *OCIMultiArch) (OCIImageIndex, error) { - idx := OCIImageIndex { + idx := OCIImageIndex{ SchemaVersion: 2, - MediaType: "application/vnd.oci.image.index.v1+json", - Manifests: []OCIRef{}, + MediaType: "application/vnd.oci.image.index.v1+json", + Manifests: []OCIRef{}, } for _, syst := range o.images { @@ -102,7 +104,7 @@ func LoadOCIImageIndex(path string) (*OCIImageIndex, error) { } defer fd.Close() - b, err := io.ReadAll(fd) + b, err := io.ReadAll(fd) if err != nil { return nil, err } @@ -130,6 +132,7 @@ func (i OCIImageIndex) WriteTo(root string) error { type OCILayout struct { ImageLayoutVersion string `json:"imageLayoutVersion"` } + func NewOCILayout() OCILayout { return OCILayout{"1.0.0"} } @@ -138,7 +141,7 @@ func (o OCILayout) WriteTo(root string) error { if err != nil { return err } - + txtPath := filepath.Join(root, "oci-layout") err = os.WriteFile(txtPath, txt, 0644) return err @@ -146,16 +149,17 @@ func (o OCILayout) WriteTo(root string) error { //---- Alba logic type OCISystemImage struct { - path string - os string - arch string + path string + os string + arch string index *OCIImageIndex } + func NewOCISystemImage(path, os, arch string) (OCISystemImage, error) { - si := OCISystemImage { - path: path, - os: os, - arch: arch, + si := OCISystemImage{ + path: path, + os: os, + arch: arch, index: nil, } @@ -171,9 +175,9 @@ func NewOCISystemImage(path, os, arch string) (OCISystemImage, error) { } // Enrich manifest - si.index.Manifests[0].Platform = &OCIPlatform { + si.index.Manifests[0].Platform = &OCIPlatform{ Architecture: arch, - OS: os, + OS: os, } return si, nil } @@ -183,15 +187,15 @@ func (o OCISystemImage) String() string { } type OCIMultiArch struct { - Name string - Tag string - path string - multi *OCIImageIndex + Name string + Tag string + path string + multi *OCIImageIndex images []OCISystemImage } func NewOCIMultiArch(nametag string) (*OCIMultiArch, error) { - ntspl := strings.Split(nametag, ":") + ntspl := strings.Split(nametag, ":") if len(ntspl) != 2 { return nil, errors.New("nametag must be of the form 'name:tag'") } @@ -201,10 +205,10 @@ func NewOCIMultiArch(nametag string) (*OCIMultiArch, error) { return nil, err } - return &OCIMultiArch { - Name: ntspl[0], - Tag: ntspl[1], - path: tmp, + return &OCIMultiArch{ + Name: ntspl[0], + Tag: ntspl[1], + path: tmp, images: []OCISystemImage{}, }, nil } @@ -252,7 +256,6 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error { return err } - // Convert the docker archive to an oci image _, err = copy.Image(context.Background(), pctx, dstRef, srcRef, ©.Options{}) if err != nil { @@ -288,7 +291,6 @@ func (o *OCIMultiArch) MergeSystemImages() error { return err } - // Create the oci-layout file err = NewOCILayout().WriteTo(multiArchRoot) if err != nil { @@ -308,7 +310,6 @@ func (o *OCIMultiArch) MergeSystemImages() error { o.multi = &idx fmt.Printf("-> index.json\n") - // Copy blobs for _, img := range o.images { blobCounter := 0 @@ -454,15 +455,14 @@ func (o *OCIMultiArch) UploadImageS3(buck *blob.Bucket) error { return err } counter += 1 - + } fmt.Printf("[blob %s %s] %d items sent\n", ref.Platform.OS, ref.Platform.Architecture, counter) } - + return nil } - type StaticRegistryManager struct { name string buck *blob.Bucket @@ -476,7 +476,7 @@ func NewStaticRegistryManager(buck *blob.Bucket, name string) *StaticRegistryMan } type TagList struct { - Name string `json:"name"` + Name string `json:"name"` Tags []string `json:"tags"` } @@ -489,7 +489,7 @@ func (l *StaticRegistryManager) ComputeTagList() (TagList, error) { } iter := l.buck.List(&blob.ListOptions{ - Prefix: fmt.Sprintf("v2/%s/manifests/", l.name), + Prefix: fmt.Sprintf("v2/%s/manifests/", l.name), Delimiter: "/", }) for { @@ -500,7 +500,7 @@ func (l *StaticRegistryManager) ComputeTagList() (TagList, error) { if err != nil { return tagList, err } - + ksplit := strings.Split(obj.Key, "/") if len(ksplit) < 1 { return tagList, errors.New(fmt.Sprintf("Invalid key name %s", obj.Key)) @@ -509,7 +509,7 @@ func (l *StaticRegistryManager) ComputeTagList() (TagList, error) { if len(fname) >= cutDigestPrefix && fname[:cutDigestPrefix] == digestPrefix { // we ignore sha256 addressed manifests - continue + continue } tagList.Tags = append(tagList.Tags, fname) } @@ -543,66 +543,66 @@ func (l *StaticRegistryManager) UpdateTagList() error { //--- Command logic var containerCmd = &cobra.Command{ - Use: "container", - Short: "Manage container images", - Long: "Publish software on an S3 target following the OCI specification", + Use: "container", + Short: "Manage container images", + Long: "Publish software on an S3 target following the OCI specification", } var containerTag string var containerPublishCmd = &cobra.Command{ - Use: "push [folder] [remote]", // https://gocloud.dev/howto/blob/#s3-compatible - Short: "Publish a container image", - Long: "Copy .tar.gz files in the specified folder on the S3 target so that they match the OCI distribution specification", - Args: cobra.ExactArgs(2), - Run: func(cmd *cobra.Command, args []string) { - localPath := args[0] - remotePath := args[1] - - oi, err := NewOCIMultiArch(containerTag) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - defer oi.Close() + Use: "push [folder] [remote]", // https://gocloud.dev/howto/blob/#s3-compatible + Short: "Publish a container image", + Long: "Copy .tar.gz files in the specified folder on the S3 target so that they match the OCI distribution specification", + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + localPath := args[0] + remotePath := args[1] - if err = oi.LoadFromDockerArchives(localPath); err != nil { - fmt.Println(err) - os.Exit(1) - } - - if err = oi.MergeSystemImages(); err != nil { - fmt.Println(err) - os.Exit(1) - } - - if strings.HasPrefix(remotePath, "s3:") { - // open bucket - bucket, err := blob.OpenBucket(context.Background(), remotePath) + oi, err := NewOCIMultiArch(containerTag) if err != nil { fmt.Println(err) os.Exit(1) } - defer bucket.Close() + defer oi.Close() - // upload image - if err = oi.UploadImageS3(bucket); err != nil { - fmt.Println(err) - os.Exit(1) - } - - // update tag - if err = NewStaticRegistryManager(bucket, oi.Name).UpdateTagList(); err != nil { + if err = oi.LoadFromDockerArchives(localPath); err != nil { fmt.Println(err) os.Exit(1) } - } else { - fmt.Printf("Protocol not supported for remote path %s. Supported transports are s3:// and docker://\n", remotePath) - os.Exit(1) - } + if err = oi.MergeSystemImages(); err != nil { + fmt.Println(err) + os.Exit(1) + } - fmt.Printf("✅ push succeeded\n") - }, + if strings.HasPrefix(remotePath, "s3:") { + // open bucket + bucket, err := blob.OpenBucket(context.Background(), remotePath) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + defer bucket.Close() + + // upload image + if err = oi.UploadImageS3(bucket); err != nil { + fmt.Println(err) + os.Exit(1) + } + + // update tag + if err = NewStaticRegistryManager(bucket, oi.Name).UpdateTagList(); err != nil { + fmt.Println(err) + os.Exit(1) + } + } else { + fmt.Printf("Protocol not supported for remote path %s. Supported transports are s3:// and docker://\n", remotePath) + os.Exit(1) + + } + + fmt.Printf("✅ push succeeded\n") + }, } func init() { @@ -614,8 +614,8 @@ func init() { policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} pctx, err = signature.NewPolicyContext(policy) if err != nil { - fmt.Println(err) - os.Exit(1) + fmt.Println(err) + os.Exit(1) } containerPublishCmd.Flags().StringVarP(&containerTag, "tag", "t", "", "Tag of the project, eg. albatros:0.9") diff --git a/cmd/root.go b/cmd/root.go index 719e8af..9615a7b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,7 +5,7 @@ import ( ) var RootCmd = &cobra.Command{ - Use: "alba", - Short: "alba is a companion cli for albatros", - Long: `A companion CLI for your CI that handle its configuration and artifacts publishing`, + Use: "alba", + Short: "alba is a companion cli for albatros", + Long: `A companion CLI for your CI that handle its configuration and artifacts publishing`, } diff --git a/cmd/static.go b/cmd/static.go index 252ffa9..41bf58f 100644 --- a/cmd/static.go +++ b/cmd/static.go @@ -26,7 +26,7 @@ type Tag struct { } type Flavor struct { - Resources []Resource `json:"resources"` + Resources []Resource `json:"resources"` Platform RegistryPlatform `json:"platform"` } @@ -36,27 +36,26 @@ type Resource struct { type RegistryPlatform struct { Architecture string `json:"architecture"` - OS string `json:"os"` + OS string `json:"os"` } type Manifest struct { - Name string `json:"name"` + Name string `json:"name"` Tags []string `json:"tags"` } - //--- //--- Collect data on the filesystem type Platform struct { - OS string - Arch string - Root string + OS string + Arch string + Root string Files []string } func CollectPlatforms(path string) ([]Platform, error) { var p []Platform - + osDirList, err := os.ReadDir(path) if err != nil { return p, err @@ -83,11 +82,10 @@ func CollectPlatforms(path string) ([]Platform, error) { filenames = append(filenames, f.Name()) } - - plat := Platform { - OS: osDir.Name(), - Arch: archDir.Name(), - Root: root, + plat := Platform{ + OS: osDir.Name(), + Arch: archDir.Name(), + Root: root, Files: filenames, } p = append(p, plat) @@ -98,9 +96,9 @@ func CollectPlatforms(path string) ([]Platform, error) { } func (p *Platform) BuildRegistryPlatform() RegistryPlatform { - return RegistryPlatform { + return RegistryPlatform{ Architecture: p.Arch, - OS: p.OS, + OS: p.OS, } } @@ -114,8 +112,8 @@ func (p *Platform) BuildResources() []Resource { //--- type Artifact struct { - Name string - Tag string + Name string + Tag string Platforms []Platform } @@ -127,7 +125,7 @@ func NewArtifact(nametag, path string) (Artifact, error) { ar := Artifact{ Name: ntspl[0], - Tag: ntspl[1], + Tag: ntspl[1], } plat, err := CollectPlatforms(path) @@ -140,7 +138,7 @@ func NewArtifact(nametag, path string) (Artifact, error) { } func (a *Artifact) UpdateManifest() Manifest { - return Manifest { + return Manifest{ Name: a.Name, Tags: []string{a.Tag}, //@FIXME we must fetch the other tags of the repo } @@ -149,9 +147,9 @@ func (a *Artifact) UpdateManifest() Manifest { func (a *Artifact) BuildTag() Tag { t := Tag{Flavors: []Flavor{}} for _, p := range a.Platforms { - f := Flavor { + f := Flavor{ Resources: p.BuildResources(), - Platform: p.BuildRegistryPlatform(), + Platform: p.BuildRegistryPlatform(), } t.Flavors = append(t.Flavors, f) } @@ -182,7 +180,7 @@ func (a *Artifact) Upload(buck *blob.Bucket) error { if err != nil { return err } - + remoteTagPath := path.Join(prefix, a.Name, a.Tag) if err := NewUploadFromByte(buck, tjson).UploadTo(remoteTagPath); err != nil { return err @@ -195,7 +193,7 @@ func (a *Artifact) Upload(buck *blob.Bucket) error { if err != nil { return err } - + remoteManifestPath := path.Join(prefix, a.Name) if err := NewUploadFromByte(buck, mjson).UploadTo(remoteManifestPath); err != nil { return err @@ -208,10 +206,11 @@ func (a *Artifact) Upload(buck *blob.Bucket) error { //--- //--- Bucket Wrapper type BucketUploader struct { - bucket *blob.Bucket - reader io.ReadCloser + bucket *blob.Bucket + reader io.ReadCloser options *blob.WriterOptions } + func NewUploadFromFS(buck *blob.Bucket, path string) (*BucketUploader, error) { fd, err := os.Open(path) if err != nil { @@ -233,7 +232,7 @@ func NewUploadFromByte(buck *blob.Bucket, content []byte) *BucketUploader { func (bu *BucketUploader) ContentType(ct string) *BucketUploader { if bu.options == nil { - bu.options = &blob.WriterOptions{} + bu.options = &blob.WriterOptions{} } bu.options.ContentType = ct @@ -269,7 +268,7 @@ func (bu *BucketUploader) UploadTo(key string) error { if closeLocalErr != nil { return closeLocalErr } - + return nil } @@ -277,45 +276,45 @@ func (bu *BucketUploader) UploadTo(key string) error { //--- Command logic var staticCmd = &cobra.Command{ - Use: "static", - Short: "Manage static artifacts", - Long: "There are many ways to ship software, one is simply to publish a bunch of files on a mirror.", + Use: "static", + Short: "Manage static artifacts", + Long: "There are many ways to ship software, one is simply to publish a bunch of files on a mirror.", } var tag string var publishCmd = &cobra.Command{ - Use: "push [folder] [remote]", // https://gocloud.dev/howto/blob/#s3-compatible - Short: "Publish a static artifact", - Long: "Sending logic for a static artifact", - Args: cobra.ExactArgs(2), - Run: func(cmd *cobra.Command, args []string) { - localFolder := args[0] - remoteUrl := args[1] + Use: "push [folder] [remote]", // https://gocloud.dev/howto/blob/#s3-compatible + Short: "Publish a static artifact", + Long: "Sending logic for a static artifact", + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + localFolder := args[0] + remoteUrl := args[1] - // build artifact in memory - art, err := NewArtifact(tag, localFolder) - if err != nil { - fmt.Println(err) - os.Exit(1) - } + // build artifact in memory + art, err := NewArtifact(tag, localFolder) + if err != nil { + fmt.Println(err) + os.Exit(1) + } - // open bucket - bucket, err := blob.OpenBucket(context.Background(), remoteUrl) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - defer bucket.Close() + // open bucket + bucket, err := blob.OpenBucket(context.Background(), remoteUrl) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + defer bucket.Close() - // send artifacts - err = art.Upload(bucket) - if err != nil { - fmt.Println(err) - os.Exit(1) - } + // send artifacts + err = art.Upload(bucket) + if err != nil { + fmt.Println(err) + os.Exit(1) + } - fmt.Printf("✅ push succeeded\n") - }, + fmt.Printf("✅ push succeeded\n") + }, } func init() { diff --git a/pkg/cluster.go b/pkg/cluster.go index aef1d8a..7d1804d 100644 --- a/pkg/cluster.go +++ b/pkg/cluster.go @@ -1,18 +1,17 @@ package pkg import ( - "log" consul "github.com/hashicorp/consul/api" nomad "github.com/hashicorp/nomad/api" + "log" ) - type Cluster struct { - Nomad *nomad.Client - Consul *consul.Client + Nomad *nomad.Client + Consul *consul.Client } -func NewCluster(conf* Config) Cluster { +func NewCluster(conf *Config) Cluster { cluster := Cluster{} // Init Nomad diff --git a/pkg/config.go b/pkg/config.go index ec7775f..dc819e6 100644 --- a/pkg/config.go +++ b/pkg/config.go @@ -1,9 +1,9 @@ package pkg import ( - "log" - "fmt" - "github.com/caarlos0/env/v7" + "fmt" + "github.com/caarlos0/env/v7" + "log" ) type Config struct { diff --git a/pkg/handler.go b/pkg/handler.go index 833fe5c..4795ba9 100644 --- a/pkg/handler.go +++ b/pkg/handler.go @@ -4,23 +4,23 @@ import ( b64 "encoding/base64" "encoding/json" "fmt" + "golang.org/x/exp/slices" "io" "log" "net/http" - "golang.org/x/exp/slices" "strings" nomad "github.com/hashicorp/nomad/api" ) type HookHandler struct { - Config *Config - Cluster *Cluster + Config *Config + Cluster *Cluster } type BuildHandler struct { - Config *Config - Cluster *Cluster + Config *Config + Cluster *Cluster } func (h HookHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -101,13 +101,13 @@ func (h HookHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { log.Printf("Created job %s for %s\n", dres.DispatchedJobID, notifInfo) // Start a lifecycle observer to update gitea status - j := Job { - Cluster: h.Cluster, - Config: h.Config, + j := Job{ + Cluster: h.Cluster, + Config: h.Config, Notification: ¬ification, - Dispatch: dres, - Creds: &repoDesc.Gitea, - Flavor: flavor, + Dispatch: dres, + Creds: &repoDesc.Gitea, + Flavor: flavor, } go j.follow() diff --git a/pkg/job.go b/pkg/job.go index af43e96..ce43ef8 100644 --- a/pkg/job.go +++ b/pkg/job.go @@ -1,18 +1,18 @@ package pkg import ( - "log" "code.gitea.io/sdk/gitea" nomad "github.com/hashicorp/nomad/api" + "log" ) type Job struct { - Cluster *Cluster - Config *Config + Cluster *Cluster + Config *Config Notification *GiteaNotification - Dispatch *nomad.JobDispatchResponse - Creds *SecretGitea - Flavor string + Dispatch *nomad.JobDispatchResponse + Creds *SecretGitea + Flavor string } func (j *Job) follow() {