From b40d49609687f2da73a0bdb3b08fda70abaecd09 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Fri, 28 Apr 2023 11:41:19 +0200 Subject: [PATCH] static push logic --- .albatros | 2 + cmd/static.go | 196 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 193 insertions(+), 5 deletions(-) diff --git a/.albatros b/.albatros index b5fc9b2..682f13a 100755 --- a/.albatros +++ b/.albatros @@ -8,3 +8,5 @@ nix build --print-build-logs # export HOME=/kaniko # nix develop --command sh -c "executor --force --destination dxflrs/albatros:${COMMIT} --context dir://`pwd` --verbosity=debug" 1>&2 #fi +# +# ./alba static push -t albatros:0.9 df/ 's3://download.deuxfleurs.org?endpoint=garage.deuxfleurs.fr&s3ForcePathStyle=true®ion=garage' diff --git a/cmd/static.go b/cmd/static.go index a602df9..0d07f90 100644 --- a/cmd/static.go +++ b/cmd/static.go @@ -1,21 +1,56 @@ package cmd import ( + "bytes" "context" + "encoding/json" "errors" "fmt" + "io" "os" + "path" "path/filepath" "strings" + "github.com/spf13/cobra" "gocloud.dev/blob" _ "gocloud.dev/blob/s3blob" ) +const prefix = "df-dist-v1" + +//--- +//--- Registry flavors and manifest +type Tag struct { + Flavors []Flavor `json:"flavors"` +} + +type Flavor struct { + Resources []Resource `json:"resources"` + Platform RegistryPlatform `json:"platform"` +} + +type Resource struct { + Path string `json:"path"` +} + +type RegistryPlatform struct { + Architecture string `json:"architecture"` + OS string `json:"os"` +} + +type Manifest struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + + +//--- +//--- Collect data on the filesystem type Platform struct { - OS string - Arch string - Root string + OS string + Arch string + Root string Files []string } @@ -58,6 +93,22 @@ func CollectPlatforms(path string) ([]Platform, error) { return p, nil } +func (p *Platform) BuildRegistryPlatform() RegistryPlatform { + return RegistryPlatform { + Architecture: p.Arch, + OS: p.OS, + } +} + +func (p *Platform) BuildResources() []Resource { + r := []Resource{} + for _, f := range p.Files { + r = append(r, Resource{Path: f}) + } + return r +} + +//--- type Artifact struct { Name string Tag string @@ -84,6 +135,133 @@ func NewArtifact(nametag, path string) (Artifact, error) { return ar, nil } +func (a *Artifact) UpdateManifest() Manifest { + return Manifest { + Name: a.Name, + Tags: []string{a.Tag}, //@FIXME we must fetch the other tags of the repo + } +} + +func (a *Artifact) BuildTag() Tag { + t := Tag{Flavors: []Flavor{}} + for _, p := range a.Platforms { + f := Flavor { + Resources: p.BuildResources(), + Platform: p.BuildRegistryPlatform(), + } + t.Flavors = append(t.Flavors, f) + } + return t +} + +func (a *Artifact) Upload(buck *blob.Bucket) error { + // Upload blobs + for _, plat := range a.Platforms { + for _, file := range plat.Files { + localPath := filepath.Join(plat.Root, file) + bu, err := NewUploadFromFS(buck, localPath) + if err != nil { + return err + } + + remotePath := path.Join(prefix, a.Name, a.Tag, plat.OS, plat.Arch, file) + if err := bu.UploadTo(remotePath); err != nil { + return err + } + fmt.Printf("%s -> %s\n", localPath, remotePath) + } + } + + // Upload tag + tag := a.BuildTag() + tjson, err := json.Marshal(tag) + if err != nil { + return err + } + + remoteTagPath := path.Join(prefix, a.Name, a.Tag) + if err := NewUploadFromByte(buck, tjson).UploadTo(remoteTagPath); err != nil { + return err + } + fmt.Printf("tag -> %s\n", remoteTagPath) + + // Update manifest + manifest := a.UpdateManifest() + mjson, err := json.Marshal(manifest) + if err != nil { + return err + } + + remoteManifestPath := path.Join(prefix, a.Name) + if err := NewUploadFromByte(buck, mjson).UploadTo(remoteManifestPath); err != nil { + return err + } + fmt.Printf("manifest -> %s\n", remoteManifestPath) + + return nil +} + +//--- +//--- Bucket Wrapper +type BucketUploader struct { + bucket *blob.Bucket + reader io.ReadCloser +} +func NewUploadFromFS(buck *blob.Bucket, path string) (*BucketUploader, error) { + fd, err := os.Open(path) + if err != nil { + return nil, err + } + + return &BucketUploader{ + bucket: buck, + reader: fd, + }, nil +} + +func NewUploadFromByte(buck *blob.Bucket, content []byte) *BucketUploader { + return &BucketUploader{ + bucket: buck, + reader: io.NopCloser(bytes.NewReader(content)), + } +} + +func (bu *BucketUploader) UploadTo(key string) error { + // Checks + if bu.bucket == nil || bu.reader == nil { + return errors.New("bucket and reader can't be nil when calling UploadTo") + } + + // Open remote object + w, err := bu.bucket.NewWriter(context.Background(), key, nil) + if err != nil { + return err + } + + // Copy local file bytes to the remote object + _, err = io.Copy(w, bu.reader) + + // Close descriptors + closeRemoteErr := w.Close() + closeLocalErr := bu.reader.Close() + + // Check errors + if err != nil { + return err + } + if closeRemoteErr != nil { + return closeRemoteErr + } + if closeLocalErr != nil { + return closeLocalErr + } + + return nil +} + +//--- +//--- Command logic + var staticCmd = &cobra.Command{ Use: "static", Short: "Manage static artifacts", @@ -92,7 +270,7 @@ var staticCmd = &cobra.Command{ var tag string var publishCmd = &cobra.Command{ - Use: "publish [folder] [remote]", // https://gocloud.dev/howto/blob/#s3-compatible + Use: "push [folder] [remote]", // https://gocloud.dev/howto/blob/#s3-compatible Short: "Publish a static artifact", Long: "Sending logic for a static artifact", Args: cobra.ExactArgs(2), @@ -100,13 +278,14 @@ var publishCmd = &cobra.Command{ localFolder := args[0] remoteUrl := args[1] + // build artifact in memory art, err := NewArtifact(tag, localFolder) if err != nil { fmt.Println(err) os.Exit(1) } - fmt.Printf("%#v\n", art) + // open bucket bucket, err := blob.OpenBucket(context.Background(), remoteUrl) if err != nil { fmt.Println(err) @@ -114,7 +293,14 @@ var publishCmd = &cobra.Command{ } defer bucket.Close() + // send artifacts + err = art.Upload(bucket) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + fmt.Printf("✅ push succeeded\n") }, }