albatros/cmd/static.go

package cmd

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/spf13/cobra"
	"gocloud.dev/blob"
	_ "gocloud.dev/blob/s3blob"
)

const prefix = "df-dist-v1"

//---
//--- Registry flavors and manifest
type Tag struct {
	Flavors []Flavor `json:"flavors"`
}

type Flavor struct {
	Resources []Resource       `json:"resources"`
	Platform  RegistryPlatform `json:"platform"`
}

type Resource struct {
	Path string `json:"path"`
}

type RegistryPlatform struct {
	Architecture string `json:"architecture"`
	OS           string `json:"os"`
}

//---
//--- Collect data on the filesystem
type Platform struct {
	OS    string
	Arch  string
	Root  string
	Files []string
}

func CollectPlatforms(path string) ([]Platform, error) {
	var p []Platform

	osDirList, err := os.ReadDir(path)
	if err != nil {
		return p, err
	}

	// Collect platforms
	for _, osDir := range osDirList {
		intPath := filepath.Join(path, osDir.Name())
		archDirList, err := os.ReadDir(intPath)
		if err != nil {
			fmt.Printf("skipping %s: %+v\n", intPath, err)
			continue
		}
		for _, archDir := range archDirList {
			root := filepath.Join(path, osDir.Name(), archDir.Name())
			files, err := os.ReadDir(root)
			if err != nil {
				fmt.Printf("skipping %s: %+v\n", root, err)
				continue
			}

			var filenames []string
			for _, f := range files {
				filenames = append(filenames, f.Name())
			}

			plat := Platform{
				OS:    osDir.Name(),
				Arch:  archDir.Name(),
				Root:  root,
				Files: filenames,
			}
			p = append(p, plat)
		}
	}

	return p, nil
}

func (p *Platform) BuildRegistryPlatform() RegistryPlatform {
	return RegistryPlatform{
		Architecture: p.Arch,
		OS:           p.OS,
	}
}

func (p *Platform) BuildResources() []Resource {
	r := []Resource{}
	for _, f := range p.Files {
		r = append(r, Resource{Path: f})
	}
	return r
}

//---
type Artifact struct {
	Name      string
	Tag       string
	Platforms []Platform
}

func NewArtifact(nametag, path string) (Artifact, error) {
	ntspl := strings.Split(nametag, ":")
	if len(ntspl) != 2 {
		return Artifact{}, errors.New("nametag must be of the form 'name:tag'")
	}

	ar := Artifact{
		Name: ntspl[0],
		Tag:  ntspl[1],
	}

	plat, err := CollectPlatforms(path)
	if err != nil {
		return ar, err
	}
	ar.Platforms = plat

	return ar, nil
}

func (a *Artifact) BuildTag() Tag {
	t := Tag{Flavors: []Flavor{}}
	for _, p := range a.Platforms {
		f := Flavor{
			Resources: p.BuildResources(),
			Platform:  p.BuildRegistryPlatform(),
		}
		t.Flavors = append(t.Flavors, f)
	}
	return t
}

func (a *Artifact) Upload(buck *blob.Bucket) error {
	// Upload blobs
	for _, plat := range a.Platforms {
		for _, file := range plat.Files {
			localPath := filepath.Join(plat.Root, file)
			bu, err := NewUploadFromFS(buck, localPath)
			if err != nil {
				return err
			}

			remotePath := path.Join(prefix, a.Name, a.Tag, plat.OS, plat.Arch, file)
			if err := bu.UploadTo(remotePath); err != nil {
				return err
			}
			fmt.Printf("%s -> %s\n", localPath, remotePath)
		}
	}

	// Upload tag
	tag := a.BuildTag()
	tjson, err := json.Marshal(tag)
	if err != nil {
		return err
	}

	remoteTagPath := path.Join(prefix, a.Name, a.Tag)
	if err := NewUploadFromByte(buck, tjson).UploadTo(remoteTagPath); err != nil {
		return err
	}
	fmt.Printf("tag -> %s\n", remoteTagPath)
	return nil
}

//--- Manage uploaded artifacts, list them, etc.
type ArtifactDescriptor struct {
	Tag string
	Date time.Time
}
type ArtifactManager struct {
	name string
	buck *blob.Bucket
	artifacts []ArtifactDescriptor
}

func NewArtifactManager(buck *blob.Bucket, name string) *ArtifactManager {
	return &ArtifactManager{
		buck: buck,
		name: name,
	}
}


func (am *ArtifactManager) Scan() error {
	iter := am.buck.List(&blob.ListOptions{
		Prefix:    fmt.Sprintf("df-dist-v1/%s/", am.name),
		Delimiter: "/",
	})

	for {
		obj, err := iter.Next(context.Background())
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
		if obj.IsDir {
			continue
		}
		ksplit := strings.Split(obj.Key, "/")
		if len(ksplit) < 1 {
			return errors.New(fmt.Sprintf("Invalid key name %s", obj.Key))
		}
		fname := ksplit[len(ksplit)-1]

		ad := ArtifactDescriptor {
			Tag: fname,
			Date: obj.ModTime,
		}

		am.artifacts = append(am.artifacts, ad)
	}
	return nil
}

func (am *ArtifactManager) TagList() TagList {
	// Sort by date desc
	sort.Slice(am.artifacts, func(i, j int) bool {
		return am.artifacts[i].Date.After(am.artifacts[j].Date)
	})

	// Build tagList
	tagList := TagList {
		Name: am.name,
	}
	for _, art := range am.artifacts {
		tagList.Tags = append(tagList.Tags, art.Tag)
	}
	return tagList
}


func (am *ArtifactManager) UpdateTagList() error {
	fmt.Printf("--- update taglist ---\n")

	err := am.Scan()
	if err != nil {
		return err
	}

	tagList := am.TagList()
	fmt.Printf("computed tag list: %v\n", tagList)

	txt, err := json.Marshal(tagList)
	if err != nil {
		return err
	}

	dst := path.Join("df-dist-v1", am.name)
	err = NewUploadFromByte(am.buck, txt).ContentType("application/json").UploadTo(dst)
	if err != nil {
		return err
	}
	fmt.Printf("taglist -> %s\n", dst)
	return nil
}

//---
//--- Bucket Wrapper
type BucketUploader struct {
	bucket  *blob.Bucket
	reader  io.ReadCloser
	options *blob.WriterOptions
}

func NewUploadFromFS(buck *blob.Bucket, path string) (*BucketUploader, error) {
	fd, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	return &BucketUploader{
		bucket: buck,
		reader: fd,
	}, nil
}

func NewUploadFromByte(buck *blob.Bucket, content []byte) *BucketUploader {
	return &BucketUploader{
		bucket: buck,
		reader: io.NopCloser(bytes.NewReader(content)),
	}
}

func (bu *BucketUploader) ContentType(ct string) *BucketUploader {
	if bu.options == nil {
		bu.options = &blob.WriterOptions{}
	}
	bu.options.ContentType = ct

	return bu
}

func (bu *BucketUploader) UploadTo(key string) error {
	// Checks
	if bu.bucket == nil || bu.reader == nil {
		return errors.New("bucket and reader can't be nil when calling UploadTo")
	}

	// Open remote object
	w, err := bu.bucket.NewWriter(context.Background(), key, bu.options)
	if err != nil {
		return err
	}

	// Copy local file bytes to the remote object
	_, err = io.Copy(w, bu.reader)

	// Close descriptors
	closeRemoteErr := w.Close()
	closeLocalErr := bu.reader.Close()

	// Check errors
	if err != nil {
		return err
	}
	if closeRemoteErr != nil {
		return closeRemoteErr
	}
	if closeLocalErr != nil {
		return closeLocalErr
	}

	return nil
}

//---
//--- Command logic

var staticCmd = &cobra.Command{
	Use:   "static",
	Short: "Manage static artifacts",
	Long:  "There are many ways to ship software, one is simply to publish a bunch of files on a mirror.",
}

var tag string
var publishCmd = &cobra.Command{
	Use:   "push [folder] [remote]", // https://gocloud.dev/howto/blob/#s3-compatible
	Short: "Publish a static artifact",
	Long:  "Sending logic for a static artifact",
	Args:  cobra.ExactArgs(2),
	Run: func(cmd *cobra.Command, args []string) {
		localFolder := args[0]
		remoteUrl := args[1]

		// build artifact in memory
		art, err := NewArtifact(tag, localFolder)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		// open bucket
		bucket, err := blob.OpenBucket(context.Background(), remoteUrl)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		defer bucket.Close()

		// send artifacts
		err = art.Upload(bucket)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		// update tag list
		am := NewArtifactManager(bucket, art.Name)
		err = am.UpdateTagList()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		fmt.Printf("✅ push succeeded\n")
	},
}

func init() {
	publishCmd.Flags().StringVarP(&tag, "tag", "t", "", "Tag of the project, eg. albatros:0.9")
	publishCmd.MarkFlagRequired("tag")

	staticCmd.AddCommand(publishCmd)
	RootCmd.AddCommand(staticCmd)
}