albatros/cmd/static.go

402 lines
7.7 KiB
Go

package cmd
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path"
"path/filepath"
"sort"
"strings"
"time"
"github.com/spf13/cobra"
"gocloud.dev/blob"
_ "gocloud.dev/blob/s3blob"
)
const prefix = "df-dist-v1"
//---
//--- Registry flavors and manifest
type Tag struct {
Flavors []Flavor `json:"flavors"`
}
type Flavor struct {
Resources []Resource `json:"resources"`
Platform RegistryPlatform `json:"platform"`
}
type Resource struct {
Path string `json:"path"`
}
type RegistryPlatform struct {
Architecture string `json:"architecture"`
OS string `json:"os"`
}
//---
//--- Collect data on the filesystem
type Platform struct {
OS string
Arch string
Root string
Files []string
}
func CollectPlatforms(path string) ([]Platform, error) {
var p []Platform
osDirList, err := os.ReadDir(path)
if err != nil {
return p, err
}
// Collect platforms
for _, osDir := range osDirList {
intPath := filepath.Join(path, osDir.Name())
archDirList, err := os.ReadDir(intPath)
if err != nil {
fmt.Printf("skipping %s: %+v\n", intPath, err)
continue
}
for _, archDir := range archDirList {
root := filepath.Join(path, osDir.Name(), archDir.Name())
files, err := os.ReadDir(root)
if err != nil {
fmt.Printf("skipping %s: %+v\n", root, err)
continue
}
var filenames []string
for _, f := range files {
filenames = append(filenames, f.Name())
}
plat := Platform{
OS: osDir.Name(),
Arch: archDir.Name(),
Root: root,
Files: filenames,
}
p = append(p, plat)
}
}
return p, nil
}
func (p *Platform) BuildRegistryPlatform() RegistryPlatform {
return RegistryPlatform{
Architecture: p.Arch,
OS: p.OS,
}
}
func (p *Platform) BuildResources() []Resource {
r := []Resource{}
for _, f := range p.Files {
r = append(r, Resource{Path: f})
}
return r
}
//---
type Artifact struct {
Name string
Tag string
Platforms []Platform
}
func NewArtifact(nametag, path string) (Artifact, error) {
ntspl := strings.Split(nametag, ":")
if len(ntspl) != 2 {
return Artifact{}, errors.New("nametag must be of the form 'name:tag'")
}
ar := Artifact{
Name: ntspl[0],
Tag: ntspl[1],
}
plat, err := CollectPlatforms(path)
if err != nil {
return ar, err
}
ar.Platforms = plat
return ar, nil
}
func (a *Artifact) BuildTag() Tag {
t := Tag{Flavors: []Flavor{}}
for _, p := range a.Platforms {
f := Flavor{
Resources: p.BuildResources(),
Platform: p.BuildRegistryPlatform(),
}
t.Flavors = append(t.Flavors, f)
}
return t
}
func (a *Artifact) Upload(buck *blob.Bucket) error {
// Upload blobs
for _, plat := range a.Platforms {
for _, file := range plat.Files {
localPath := filepath.Join(plat.Root, file)
bu, err := NewUploadFromFS(buck, localPath)
if err != nil {
return err
}
remotePath := path.Join(prefix, a.Name, a.Tag, plat.OS, plat.Arch, file)
if err := bu.UploadTo(remotePath); err != nil {
return err
}
fmt.Printf("%s -> %s\n", localPath, remotePath)
}
}
// Upload tag
tag := a.BuildTag()
tjson, err := json.Marshal(tag)
if err != nil {
return err
}
remoteTagPath := path.Join(prefix, a.Name, a.Tag)
if err := NewUploadFromByte(buck, tjson).UploadTo(remoteTagPath); err != nil {
return err
}
fmt.Printf("tag -> %s\n", remoteTagPath)
return nil
}
//--- Manage uploaded artifacts, list them, etc.
type ArtifactDescriptor struct {
Tag string
Date time.Time
}
type ArtifactManager struct {
name string
buck *blob.Bucket
artifacts []ArtifactDescriptor
}
func NewArtifactManager(buck *blob.Bucket, name string) *ArtifactManager {
return &ArtifactManager{
buck: buck,
name: name,
}
}
func (am *ArtifactManager) Scan() error {
iter := am.buck.List(&blob.ListOptions{
Prefix: fmt.Sprintf("df-dist-v1/%s/", am.name),
Delimiter: "/",
})
for {
obj, err := iter.Next(context.Background())
if err == io.EOF {
break
}
if err != nil {
return err
}
if obj.IsDir {
continue
}
ksplit := strings.Split(obj.Key, "/")
if len(ksplit) < 1 {
return errors.New(fmt.Sprintf("Invalid key name %s", obj.Key))
}
fname := ksplit[len(ksplit)-1]
ad := ArtifactDescriptor{
Tag: fname,
Date: obj.ModTime,
}
am.artifacts = append(am.artifacts, ad)
}
return nil
}
func (am *ArtifactManager) TagList() TagList {
// Sort by date desc
sort.Slice(am.artifacts, func(i, j int) bool {
return am.artifacts[i].Date.After(am.artifacts[j].Date)
})
// Build tagList
tagList := TagList{
Name: am.name,
}
for _, art := range am.artifacts {
tagList.Tags = append(tagList.Tags, art.Tag)
}
return tagList
}
func (am *ArtifactManager) UpdateTagList() error {
fmt.Printf("--- update taglist ---\n")
err := am.Scan()
if err != nil {
return err
}
tagList := am.TagList()
fmt.Printf("computed tag list: %v\n", tagList)
txt, err := json.Marshal(tagList)
if err != nil {
return err
}
dst := path.Join("df-dist-v1", am.name)
err = NewUploadFromByte(am.buck, txt).ContentType("application/json").UploadTo(dst)
if err != nil {
return err
}
fmt.Printf("taglist -> %s\n", dst)
return nil
}
//---
//--- Bucket Wrapper
type BucketUploader struct {
bucket *blob.Bucket
reader io.ReadCloser
options *blob.WriterOptions
}
func NewUploadFromFS(buck *blob.Bucket, path string) (*BucketUploader, error) {
fd, err := os.Open(path)
if err != nil {
return nil, err
}
return &BucketUploader{
bucket: buck,
reader: fd,
}, nil
}
func NewUploadFromByte(buck *blob.Bucket, content []byte) *BucketUploader {
return &BucketUploader{
bucket: buck,
reader: io.NopCloser(bytes.NewReader(content)),
}
}
func (bu *BucketUploader) ContentType(ct string) *BucketUploader {
if bu.options == nil {
bu.options = &blob.WriterOptions{}
}
bu.options.ContentType = ct
return bu
}
func (bu *BucketUploader) UploadTo(key string) error {
// Checks
if bu.bucket == nil || bu.reader == nil {
return errors.New("bucket and reader can't be nil when calling UploadTo")
}
// Open remote object
w, err := bu.bucket.NewWriter(context.Background(), key, bu.options)
if err != nil {
return err
}
// Copy local file bytes to the remote object
_, err = io.Copy(w, bu.reader)
// Close descriptors
closeRemoteErr := w.Close()
closeLocalErr := bu.reader.Close()
// Check errors
if err != nil {
return err
}
if closeRemoteErr != nil {
return closeRemoteErr
}
if closeLocalErr != nil {
return closeLocalErr
}
return nil
}
//---
//--- Command logic
var staticCmd = &cobra.Command{
Use: "static",
Short: "Manage static artifacts",
Long: "There are many ways to ship software, one is simply to publish a bunch of files on a mirror.",
}
var tag string
var publishCmd = &cobra.Command{
Use: "push [folder] [remote]", // https://gocloud.dev/howto/blob/#s3-compatible
Short: "Publish a static artifact",
Long: "Sending logic for a static artifact",
Args: cobra.ExactArgs(2),
Run: func(cmd *cobra.Command, args []string) {
localFolder := args[0]
remoteUrl := args[1]
// build artifact in memory
art, err := NewArtifact(tag, localFolder)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// open bucket
bucket, err := blob.OpenBucket(context.Background(), remoteUrl)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer bucket.Close()
// send artifacts
err = art.Upload(bucket)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// update tag list
am := NewArtifactManager(bucket, art.Name)
err = am.UpdateTagList()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
fmt.Printf("✅ push succeeded\n")
},
}
func init() {
publishCmd.Flags().StringVarP(&tag, "tag", "t", "", "Tag of the project, eg. albatros:0.9")
publishCmd.MarkFlagRequired("tag")
staticCmd.AddCommand(publishCmd)
RootCmd.AddCommand(staticCmd)
}