working image push/pull

cmd/container.go

@@ -1,18 +1,24 @@
 package cmd
 
 import (
+	"crypto/sha256"
 	"errors"
 	"context"
+	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
+
 	"github.com/containers/image/v5/transports/alltransports"
 	"github.com/containers/image/v5/signature"
 	"github.com/containers/image/v5/copy"
 	"github.com/spf13/cobra"
+	"gocloud.dev/blob"
+	_ "gocloud.dev/blob/s3blob"
 )
 
 var pctx *signature.PolicyContext
@@ -21,10 +27,39 @@ const distributionPrefix = "v2"
 
 //---
 //--- Image converter
-type OCIPlatform struct {
+type OCIImageManifest struct {
+	SchemaVersion int `json:"schemaVersion"`
+	MediaType string `json:"mediaType"`
+	Config OCIRef `json:"config"`
+	Layers []OCIRef `json:layers"`
+}
+func LoadOCIImageManifest(path string) (*OCIImageManifest, error) {
+	fd, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	b, err := io.ReadAll(fd)
+	if err != nil {
+		return nil, err
 	}
 
-type OCIImageManifest struct {
+	oim := &OCIImageManifest{}
+	err = json.Unmarshal(b, oim)
+	if err != nil {
+		return nil, err
+	}
+
+	return oim, nil
+}
+
+type OCIPlatform struct {
+	Architecture string `json:"architecture"`
+	OS string `json:"os"`
+}
+
+type OCIRef struct {
 	MediaType string `json:"mediaType"`
 	Digest string `json:"digest"`
 	Size int `json:"size"`
@@ -34,10 +69,65 @@ type OCIImageManifest struct {
 type OCIImageIndex struct {
 	SchemaVersion int `json:"schemaVersion"`
 	MediaType string `json:"mediaType"`
-	Manifests []OCIImageManifest `json:"manifests"`
+	Manifests []OCIRef `json:"manifests"`
+}
+func NewOCIImageIndex(o *OCIMultiArch) (OCIImageIndex, error) {
+	idx := OCIImageIndex {
+		SchemaVersion: 2,
+		MediaType: "application/vnd.oci.image.index.v1+json",
+		Manifests: []OCIRef{},
 	}
 
-type OCILayout {
+	for _, syst := range o.images {
+		if syst.index == nil {
+			return idx, errors.New(fmt.Sprintf("Missing index content for %s. Check that the file exists and that it can be parsed", syst.path))
+		}
+		if len(syst.index.Manifests) != 1 {
+			return idx, errors.New(fmt.Sprintf("%s is not a system image as it does not contain exactly one manifest in its index", syst.path))
+		}
+		if syst.index.Manifests[0].Platform == nil {
+			return idx, errors.New(fmt.Sprintf("Manifest for %s has not been enriched with its platform (os+arch). This is a logic bug.", syst.path))
+
+		}
+		idx.Manifests = append(idx.Manifests, syst.index.Manifests[0])
+	}
+
+	return idx, nil
+}
+
+func LoadOCIImageIndex(path string) (*OCIImageIndex, error) {
+	fd, err := os.Open(filepath.Join(path, "index.json"))
+	if err != nil {
+		return nil, err
+	}
+	defer fd.Close()
+
+	b, err := io.ReadAll(fd) 
+	if err != nil {
+		return nil, err
+	}
+
+	var idx OCIImageIndex
+	err = json.Unmarshal(b, &idx)
+	if err != nil {
+		return nil, err
+	}
+
+	return &idx, nil
+}
+
+func (i OCIImageIndex) WriteTo(root string) error {
+	txt, err := json.Marshal(i)
+	if err != nil {
+		return err
+	}
+
+	txtPath := filepath.Join(root, "index.json")
+	err = os.WriteFile(txtPath, txt, 0644)
+	return err
+}
+
+type OCILayout struct {
 	ImageLayoutVersion string `json:"imageLayoutVersion"`
 }
 func NewOCILayout() OCILayout {
@@ -50,23 +140,53 @@ func (o OCILayout) WriteTo(root string) error {
 	}
 	
 	txtPath := filepath.Join(root, "oci-layout")
-	err = ioutil.WriteFile(txtPath, txt, 0644)
+	err = os.WriteFile(txtPath, txt, 0644)
 	return err
 }
 
+//---- Alba logic
 type OCISystemImage struct {
 	path string
 	os string
 	arch string
+	index *OCIImageIndex
 }
+func NewOCISystemImage(path, os, arch string) (OCISystemImage, error) {
+	si := OCISystemImage {
+		path: path,
+		os: os,
+		arch: arch,
+		index: nil,
+	}
+
+	man, err := LoadOCIImageIndex(path)
+	if err != nil {
+		return si, err
+	}
+
+	si.index = man
+
+	if len(si.index.Manifests) != 1 {
+		return si, errors.New(fmt.Sprintf("%s index has not exactly one manifest, this can't be a system image\n", path))
+	}
+
+	// Enrich manifest
+	si.index.Manifests[0].Platform = &OCIPlatform {
+		Architecture: arch,
+		OS: os,
+	}
+	return si, nil
+}
+
 func (o OCISystemImage) String() string {
 	return fmt.Sprintf("[oci system image; os:%s, arch:%s, path:%s]", o.os, o.arch, o.path)
 }
 
 type OCIMultiArch struct {
-	name string
+	Name string
-	tag string
+	Tag string
 	path string
+	multi *OCIImageIndex
 	images []OCISystemImage
 }
 
@@ -82,8 +202,8 @@ func NewOCIMultiArch(nametag string) (*OCIMultiArch, error) {
 	}
 
 	return &OCIMultiArch {
-		name: ntspl[0],
+		Name: ntspl[0],
-		tag: ntspl[1],
+		Tag: ntspl[1],
 		path: tmp,
 		images: []OCISystemImage{},
 	}, nil
@@ -100,6 +220,7 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error {
 		return err
 	}
 
+	// Convert all docker archives to oci images
 	for idx, imgFile := range imgDirList {
 		imgFilename := imgFile.Name()
 
@@ -119,6 +240,7 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error {
 		goos := infos[0]
 		goarch := infos[1]
 
+		// Prepare the image conversion
 		srcRef, err := alltransports.ParseImageName(fmt.Sprintf("docker-archive:%s", archivePath))
 		if err != nil {
 			return err
@@ -131,15 +253,17 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error {
 		}
 
 
+		// Convert the docker archive to an oci image
 		_, err = copy.Image(context.Background(), pctx, dstRef, srcRef, &copy.Options{})
 		if err != nil {
 			return err
 		}
-		img := OCISystemImage {
+
-			path: ociPath,
+		img, err := NewOCISystemImage(ociPath, goos, goarch)
-			os: goos,
+		if err != nil {
-			arch: goarch,
+			return err
 		}
+
 		fmt.Printf("%s -> %s\n", archivePath, img)
 		o.images = append(o.images, img)
 	}
@@ -151,54 +275,62 @@ func (o *OCIMultiArch) MergeSystemImages() error {
 	fmt.Printf("-- merge system images --\n")
 	multiArchRoot := filepath.Join(o.path, "multi")
 
-	// Create root
+	// Create the root folder for the OCI multi arch image
 	err := os.Mkdir(multiArchRoot, 0750)
 	if err != nil {
 		return err
 	}
 
-	// Create blob path
+	// Create the blob folder for the multi arch image
-	multiBlobPath := filepath.Join(multiArchRoot, 'blobs', 'sha256')
+	multiBlobPath := filepath.Join(multiArchRoot, "blobs", "sha256")
-	err := os.MkdirAll(multiBlobPath)
+	err = os.MkdirAll(multiBlobPath, 0750)
 	if err != nil {
 		return err
 	}
 
 
-	// Create oci-layout file
+	// Create the oci-layout file
-	err := NewOCILayout().WriteTo(multiArchRoot)
+	err = NewOCILayout().WriteTo(multiArchRoot)
 	if err != nil {
 		return err
 	}
 	fmt.Printf("-> oci-layout\n")
 
-	// Create index.json
+	// Create the index.json
-	err, index := NewOCIIndex()
+	idx, err := NewOCIImageIndex(o)
+	if err != nil {
+		return err
+	}
+	err = idx.WriteTo(multiArchRoot)
+	if err != nil {
+		return err
+	}
+	o.multi = &idx
 	fmt.Printf("-> index.json\n")
 
 
 	// Copy blobs
 	for _, img := range o.images {
 		blobCounter := 0
-		blobPath := filepath.Join(img.path, 'blobs', 'sha256')
+		blobPath := filepath.Join(img.path, "blobs", "sha256")
 		blobList, err := os.ReadDir(blobPath)
 		if err != nil {
 			return err
 		}
 		for _, blobFile := range blobList {
-			src, err := io.Open(filepath.Join(blobPath, blobFile.Name()))
+			src, err := os.Open(filepath.Join(blobPath, blobFile.Name()))
 			if err != nil {
 				return err
 			}
 			defer src.Close()
 
-			dst, err := io.Create(filepath.Join(multiBlobPath, blobFile.Name()))
+			dst, err := os.Create(filepath.Join(multiBlobPath, blobFile.Name()))
 			if err != nil {
 				return err
 			}
 			defer dst.Close()
 
-			_, err := io.Copy(dst, src)
+			_, err = io.Copy(dst, src)
 			if err != nil {
 				return err
 			}
@@ -210,11 +342,144 @@ func (o *OCIMultiArch) MergeSystemImages() error {
 	return nil
 }
 
-func (o *OCIMultiArch) UploadImage(url string) error {
+func (o *OCIMultiArch) UploadImageS3(buck *blob.Bucket) error {
+	fmt.Printf("-- push to the s3 target --\n")
+
+	// FS paths
+	multiArchRoot := filepath.Join(o.path, "multi")
+	multiBlobPath := filepath.Join(multiArchRoot, "blobs", "sha256")
+
+	// Registry URLs
+	urlPrefix := filepath.Join("v2", o.Name)
+	manifestPrefix := filepath.Join(urlPrefix, "manifests")
+	manifestUrl := filepath.Join(manifestPrefix, o.Tag)
+	blobPrefix := filepath.Join(urlPrefix, "blobs")
+
+	// Utils
+	cutDigestPrefix := len("sha256:")
+
+	// Checks
+	if o.multi == nil {
+		return errors.New("You try to upload a multiarch image that has not been built yet...")
+	}
+
+	// Upload index
+	src := filepath.Join(multiArchRoot, "index.json")
+	up, err := NewUploadFromFS(buck, src)
+	if err != nil {
+		return err
+	}
+
+	err = up.ContentType("application/vnd.oci.image.index.v1+json").UploadTo(manifestUrl)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("[index] index.json -> %s\n", manifestUrl)
+
+	// Upload the same index but with its sha256
+	fd, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer fd.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, fd); err != nil {
+		return err
+	}
+	digest := fmt.Sprintf("%x", h.Sum(nil))
+
+	src = filepath.Join(multiArchRoot, "index.json")
+	upSha, err := NewUploadFromFS(buck, src)
+	if err != nil {
+		return err
+	}
+	dst := path.Join(manifestPrefix, "sha256:"+digest)
+	err = upSha.ContentType("application/vnd.oci.image.index.v1+json").UploadTo(dst)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("[index] index.json -> %s\n", dst)
+
+	// Upload manifest of each system image
+	for _, m := range o.multi.Manifests {
+		src := filepath.Join(multiBlobPath, m.Digest[cutDigestPrefix:])
+		upDigest, err := NewUploadFromFS(buck, src)
+		if err != nil {
+			return err
+		}
+
+		dst := path.Join(manifestPrefix, m.Digest)
+		err = upDigest.ContentType("application/vnd.oci.image.manifest.v1+json").UploadTo(dst)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("[manifest %s %s] %s -> %s\n", m.Platform.OS, m.Platform.Architecture, src, dst)
+	}
+
+	// Upload blobs from each system image
+	for _, ref := range o.multi.Manifests {
+		fullManifest := filepath.Join(multiBlobPath, ref.Digest[cutDigestPrefix:])
+		m, err := LoadOCIImageManifest(fullManifest)
+		if err != nil {
+			return err
+		}
+
+		// Upload config's blob
+		src := filepath.Join(multiBlobPath, m.Config.Digest[cutDigestPrefix:])
+		upConf, err := NewUploadFromFS(buck, src)
+		if err != nil {
+			return err
+		}
+
+		dst := path.Join(blobPrefix, m.Config.Digest)
+		err = upConf.UploadTo(dst)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("[config %s %s] %s -> %s\n", ref.Platform.OS, ref.Platform.Architecture, src, dst)
+
+		// Upload layers' blob
+		counter := 0
+		for _, lref := range m.Layers {
+			src := filepath.Join(multiBlobPath, lref.Digest[cutDigestPrefix:])
+			upLayer, err := NewUploadFromFS(buck, src)
+			if err != nil {
+				return err
+			}
+
+			dst := path.Join(blobPrefix, lref.Digest)
+			err = upLayer.UploadTo(dst)
+			if err != nil {
+				return err
+			}
+			counter += 1
+			
+		}
+		fmt.Printf("[blob %s %s] %d items sent\n", ref.Platform.OS, ref.Platform.Architecture, counter)
+	}
+	
 	return nil
 }
 
 
+type StaticRegistryManager struct {
+	name string
+	buck *blob.Bucket
+}
+
+func NewStaticRegistryManager(buck *blob.Bucket, name string) *StaticRegistryManager {
+	return &StaticRegistryManager{
+		buck: buck,
+		name: name,
+	}
+}
+
+func (l *StaticRegistryManager) UpdateTagList() error {
+	fmt.Println("Not yet implemented")
+	return nil
+}
+
 //---
 //--- Command logic
 
@@ -251,10 +516,31 @@ var containerPublishCmd = &cobra.Command{
                 os.Exit(1)
 	}
 
-	if err = oi.UploadImage(remotePath); err != nil {
+	if strings.HasPrefix(remotePath, "s3:") {
+		// open bucket
+       		bucket, err := blob.OpenBucket(context.Background(), remotePath)
+		if err != nil {
 			fmt.Println(err)
 			os.Exit(1)
 		}
+		defer bucket.Close()
+
+		// upload image
+		if err = oi.UploadImageS3(bucket); err != nil {
+                	fmt.Println(err)
+                	os.Exit(1)
+		}
+
+		// update tag	
+		if err = NewStaticRegistryManager(bucket, oi.Name).UpdateTagList(); err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+	} else {
+		fmt.Printf("Protocol not supported for remote path %s. Supported transports are s3:// and docker://\n", remotePath)
+		os.Exit(1)
+
+	}
 
 	fmt.Printf("✅ push succeeded\n")
   },

cmd/static.go

@@ -210,6 +210,7 @@ func (a *Artifact) Upload(buck *blob.Bucket) error {
 type BucketUploader struct {
 	bucket *blob.Bucket
 	reader io.ReadCloser
+	options *blob.WriterOptions
 }
 func NewUploadFromFS(buck *blob.Bucket, path string) (*BucketUploader, error) {
 	fd, err := os.Open(path)
@@ -230,6 +231,15 @@ func NewUploadFromByte(buck *blob.Bucket, content []byte) *BucketUploader {
 	}
 }
 
+func (bu *BucketUploader) ContentType(ct string) *BucketUploader {
+	if bu.options == nil {
+		bu.options = &blob.WriterOptions{}	
+	}
+	bu.options.ContentType = ct
+
+	return bu
+}
+
 func (bu *BucketUploader) UploadTo(key string) error {
 	// Checks
 	if bu.bucket == nil || bu.reader == nil {
@@ -237,7 +247,7 @@ func (bu *BucketUploader) UploadTo(key string) error {
 	}
 
 	// Open remote object
-	w, err := bu.bucket.NewWriter(context.Background(), key, nil)
+	w, err := bu.bucket.NewWriter(context.Background(), key, bu.options)
 	if err != nil {
 		return err
 	}