package cmd import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "os" "path" "path/filepath" "sort" "strings" "time" "github.com/spf13/cobra" "gocloud.dev/blob" _ "gocloud.dev/blob/s3blob" ) const prefix = "df-dist-v1" //--- //--- Registry flavors and manifest type Tag struct { Flavors []Flavor `json:"flavors"` } type Flavor struct { Resources []Resource `json:"resources"` Platform RegistryPlatform `json:"platform"` } type Resource struct { Path string `json:"path"` } type RegistryPlatform struct { Architecture string `json:"architecture"` OS string `json:"os"` } //--- //--- Collect data on the filesystem type Platform struct { OS string Arch string Root string Files []string } func CollectPlatforms(path string) ([]Platform, error) { var p []Platform osDirList, err := os.ReadDir(path) if err != nil { return p, err } // Collect platforms for _, osDir := range osDirList { intPath := filepath.Join(path, osDir.Name()) archDirList, err := os.ReadDir(intPath) if err != nil { fmt.Printf("skipping %s: %+v\n", intPath, err) continue } for _, archDir := range archDirList { root := filepath.Join(path, osDir.Name(), archDir.Name()) files, err := os.ReadDir(root) if err != nil { fmt.Printf("skipping %s: %+v\n", root, err) continue } var filenames []string for _, f := range files { filenames = append(filenames, f.Name()) } plat := Platform{ OS: osDir.Name(), Arch: archDir.Name(), Root: root, Files: filenames, } p = append(p, plat) } } return p, nil } func (p *Platform) BuildRegistryPlatform() RegistryPlatform { return RegistryPlatform{ Architecture: p.Arch, OS: p.OS, } } func (p *Platform) BuildResources() []Resource { r := []Resource{} for _, f := range p.Files { r = append(r, Resource{Path: f}) } return r } //--- type Artifact struct { Name string Tag string Platforms []Platform } func NewArtifact(nametag, path string) (Artifact, error) { ntspl := strings.Split(nametag, ":") if len(ntspl) != 2 { return Artifact{}, errors.New("nametag must be of the form 'name:tag'") } ar := Artifact{ Name: ntspl[0], Tag: ntspl[1], } plat, err := CollectPlatforms(path) if err != nil { return ar, err } ar.Platforms = plat return ar, nil } func (a *Artifact) BuildTag() Tag { t := Tag{Flavors: []Flavor{}} for _, p := range a.Platforms { f := Flavor{ Resources: p.BuildResources(), Platform: p.BuildRegistryPlatform(), } t.Flavors = append(t.Flavors, f) } return t } func (a *Artifact) Upload(buck *blob.Bucket) error { // Upload blobs for _, plat := range a.Platforms { for _, file := range plat.Files { localPath := filepath.Join(plat.Root, file) bu, err := NewUploadFromFS(buck, localPath) if err != nil { return err } remotePath := path.Join(prefix, a.Name, a.Tag, plat.OS, plat.Arch, file) if err := bu.UploadTo(remotePath); err != nil { return err } fmt.Printf("%s -> %s\n", localPath, remotePath) } } // Upload tag tag := a.BuildTag() tjson, err := json.Marshal(tag) if err != nil { return err } remoteTagPath := path.Join(prefix, a.Name, a.Tag) if err := NewUploadFromByte(buck, tjson).UploadTo(remoteTagPath); err != nil { return err } fmt.Printf("tag -> %s\n", remoteTagPath) return nil } //--- Manage uploaded artifacts, list them, etc. type ArtifactDescriptor struct { Tag string Date time.Time } type ArtifactManager struct { name string buck *blob.Bucket artifacts []ArtifactDescriptor } func NewArtifactManager(buck *blob.Bucket, name string) *ArtifactManager { return &ArtifactManager{ buck: buck, name: name, } } func (am *ArtifactManager) Scan() error { iter := am.buck.List(&blob.ListOptions{ Prefix: fmt.Sprintf("df-dist-v1/%s/", am.name), Delimiter: "/", }) for { obj, err := iter.Next(context.Background()) if err == io.EOF { break } if err != nil { return err } if obj.IsDir { continue } ksplit := strings.Split(obj.Key, "/") if len(ksplit) < 1 { return errors.New(fmt.Sprintf("Invalid key name %s", obj.Key)) } fname := ksplit[len(ksplit)-1] ad := ArtifactDescriptor{ Tag: fname, Date: obj.ModTime, } am.artifacts = append(am.artifacts, ad) } return nil } func (am *ArtifactManager) TagList() TagList { // Sort by date desc sort.Slice(am.artifacts, func(i, j int) bool { return am.artifacts[i].Date.After(am.artifacts[j].Date) }) // Build tagList tagList := TagList{ Name: am.name, } for _, art := range am.artifacts { tagList.Tags = append(tagList.Tags, art.Tag) } return tagList } func (am *ArtifactManager) UpdateTagList() error { fmt.Printf("--- update taglist ---\n") err := am.Scan() if err != nil { return err } tagList := am.TagList() fmt.Printf("computed tag list: %v\n", tagList) txt, err := json.Marshal(tagList) if err != nil { return err } dst := path.Join("df-dist-v1", am.name) err = NewUploadFromByte(am.buck, txt).ContentType("application/json").UploadTo(dst) if err != nil { return err } fmt.Printf("taglist -> %s\n", dst) return nil } //--- //--- Bucket Wrapper type BucketUploader struct { bucket *blob.Bucket reader io.ReadCloser options *blob.WriterOptions } func NewUploadFromFS(buck *blob.Bucket, path string) (*BucketUploader, error) { fd, err := os.Open(path) if err != nil { return nil, err } return &BucketUploader{ bucket: buck, reader: fd, }, nil } func NewUploadFromByte(buck *blob.Bucket, content []byte) *BucketUploader { return &BucketUploader{ bucket: buck, reader: io.NopCloser(bytes.NewReader(content)), } } func (bu *BucketUploader) ContentType(ct string) *BucketUploader { if bu.options == nil { bu.options = &blob.WriterOptions{} } bu.options.ContentType = ct return bu } func (bu *BucketUploader) UploadTo(key string) error { // Checks if bu.bucket == nil || bu.reader == nil { return errors.New("bucket and reader can't be nil when calling UploadTo") } // Open remote object w, err := bu.bucket.NewWriter(context.Background(), key, bu.options) if err != nil { return err } // Copy local file bytes to the remote object _, err = io.Copy(w, bu.reader) // Close descriptors closeRemoteErr := w.Close() closeLocalErr := bu.reader.Close() // Check errors if err != nil { return err } if closeRemoteErr != nil { return closeRemoteErr } if closeLocalErr != nil { return closeLocalErr } return nil } //--- //--- Command logic var staticCmd = &cobra.Command{ Use: "static", Short: "Manage static artifacts", Long: "There are many ways to ship software, one is simply to publish a bunch of files on a mirror.", } var tag string var publishCmd = &cobra.Command{ Use: "push [folder] [remote]", // https://gocloud.dev/howto/blob/#s3-compatible Short: "Publish a static artifact", Long: "Sending logic for a static artifact", Args: cobra.ExactArgs(2), Run: func(cmd *cobra.Command, args []string) { localFolder := args[0] remoteUrl := args[1] // build artifact in memory art, err := NewArtifact(tag, localFolder) if err != nil { fmt.Println(err) os.Exit(1) } // open bucket bucket, err := blob.OpenBucket(context.Background(), remoteUrl) if err != nil { fmt.Println(err) os.Exit(1) } defer bucket.Close() // send artifacts err = art.Upload(bucket) if err != nil { fmt.Println(err) os.Exit(1) } // update tag list am := NewArtifactManager(bucket, art.Name) err = am.UpdateTagList() if err != nil { fmt.Println(err) os.Exit(1) } fmt.Printf("✅ push succeeded\n") }, } func init() { publishCmd.Flags().StringVarP(&tag, "tag", "t", "", "Tag of the project, eg. albatros:0.9") publishCmd.MarkFlagRequired("tag") staticCmd.AddCommand(publishCmd) RootCmd.AddCommand(staticCmd) }