diff --git a/cmd/container.go b/cmd/container.go index 3eb16f8..f09a508 100644 --- a/cmd/container.go +++ b/cmd/container.go @@ -1,18 +1,24 @@ package cmd import ( + "crypto/sha256" "errors" "context" + "encoding/json" "fmt" - "io/ioutil" + "io" "os" + "path" "path/filepath" "strconv" "strings" + "github.com/containers/image/v5/transports/alltransports" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/copy" "github.com/spf13/cobra" + "gocloud.dev/blob" + _ "gocloud.dev/blob/s3blob" ) var pctx *signature.PolicyContext @@ -21,10 +27,39 @@ const distributionPrefix = "v2" //--- //--- Image converter -type OCIPlatform struct { +type OCIImageManifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config OCIRef `json:"config"` + Layers []OCIRef `json:layers"` +} +func LoadOCIImageManifest(path string) (*OCIImageManifest, error) { + fd, err := os.Open(path) + if err != nil { + return nil, err + } + defer fd.Close() + + b, err := io.ReadAll(fd) + if err != nil { + return nil, err + } + + oim := &OCIImageManifest{} + err = json.Unmarshal(b, oim) + if err != nil { + return nil, err + } + + return oim, nil } -type OCIImageManifest struct { +type OCIPlatform struct { + Architecture string `json:"architecture"` + OS string `json:"os"` +} + +type OCIRef struct { MediaType string `json:"mediaType"` Digest string `json:"digest"` Size int `json:"size"` @@ -34,10 +69,65 @@ type OCIImageManifest struct { type OCIImageIndex struct { SchemaVersion int `json:"schemaVersion"` MediaType string `json:"mediaType"` - Manifests []OCIImageManifest `json:"manifests"` + Manifests []OCIRef `json:"manifests"` +} +func NewOCIImageIndex(o *OCIMultiArch) (OCIImageIndex, error) { + idx := OCIImageIndex { + SchemaVersion: 2, + MediaType: "application/vnd.oci.image.index.v1+json", + Manifests: []OCIRef{}, + } + + for _, syst := range o.images { + if syst.index == nil { + return idx, errors.New(fmt.Sprintf("Missing index content for %s. Check that the file exists and that it can be parsed", syst.path)) + } + if len(syst.index.Manifests) != 1 { + return idx, errors.New(fmt.Sprintf("%s is not a system image as it does not contain exactly one manifest in its index", syst.path)) + } + if syst.index.Manifests[0].Platform == nil { + return idx, errors.New(fmt.Sprintf("Manifest for %s has not been enriched with its platform (os+arch). This is a logic bug.", syst.path)) + + } + idx.Manifests = append(idx.Manifests, syst.index.Manifests[0]) + } + + return idx, nil } -type OCILayout { +func LoadOCIImageIndex(path string) (*OCIImageIndex, error) { + fd, err := os.Open(filepath.Join(path, "index.json")) + if err != nil { + return nil, err + } + defer fd.Close() + + b, err := io.ReadAll(fd) + if err != nil { + return nil, err + } + + var idx OCIImageIndex + err = json.Unmarshal(b, &idx) + if err != nil { + return nil, err + } + + return &idx, nil +} + +func (i OCIImageIndex) WriteTo(root string) error { + txt, err := json.Marshal(i) + if err != nil { + return err + } + + txtPath := filepath.Join(root, "index.json") + err = os.WriteFile(txtPath, txt, 0644) + return err +} + +type OCILayout struct { ImageLayoutVersion string `json:"imageLayoutVersion"` } func NewOCILayout() OCILayout { @@ -50,23 +140,53 @@ func (o OCILayout) WriteTo(root string) error { } txtPath := filepath.Join(root, "oci-layout") - err = ioutil.WriteFile(txtPath, txt, 0644) + err = os.WriteFile(txtPath, txt, 0644) return err } +//---- Alba logic type OCISystemImage struct { path string os string arch string + index *OCIImageIndex } +func NewOCISystemImage(path, os, arch string) (OCISystemImage, error) { + si := OCISystemImage { + path: path, + os: os, + arch: arch, + index: nil, + } + + man, err := LoadOCIImageIndex(path) + if err != nil { + return si, err + } + + si.index = man + + if len(si.index.Manifests) != 1 { + return si, errors.New(fmt.Sprintf("%s index has not exactly one manifest, this can't be a system image\n", path)) + } + + // Enrich manifest + si.index.Manifests[0].Platform = &OCIPlatform { + Architecture: arch, + OS: os, + } + return si, nil +} + func (o OCISystemImage) String() string { return fmt.Sprintf("[oci system image; os:%s, arch:%s, path:%s]", o.os, o.arch, o.path) } type OCIMultiArch struct { - name string - tag string + Name string + Tag string path string + multi *OCIImageIndex images []OCISystemImage } @@ -82,8 +202,8 @@ func NewOCIMultiArch(nametag string) (*OCIMultiArch, error) { } return &OCIMultiArch { - name: ntspl[0], - tag: ntspl[1], + Name: ntspl[0], + Tag: ntspl[1], path: tmp, images: []OCISystemImage{}, }, nil @@ -100,6 +220,7 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error { return err } + // Convert all docker archives to oci images for idx, imgFile := range imgDirList { imgFilename := imgFile.Name() @@ -119,6 +240,7 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error { goos := infos[0] goarch := infos[1] + // Prepare the image conversion srcRef, err := alltransports.ParseImageName(fmt.Sprintf("docker-archive:%s", archivePath)) if err != nil { return err @@ -131,15 +253,17 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error { } + // Convert the docker archive to an oci image _, err = copy.Image(context.Background(), pctx, dstRef, srcRef, ©.Options{}) if err != nil { return err } - img := OCISystemImage { - path: ociPath, - os: goos, - arch: goarch, + + img, err := NewOCISystemImage(ociPath, goos, goarch) + if err != nil { + return err } + fmt.Printf("%s -> %s\n", archivePath, img) o.images = append(o.images, img) } @@ -151,54 +275,62 @@ func (o *OCIMultiArch) MergeSystemImages() error { fmt.Printf("-- merge system images --\n") multiArchRoot := filepath.Join(o.path, "multi") - // Create root + // Create the root folder for the OCI multi arch image err := os.Mkdir(multiArchRoot, 0750) if err != nil { return err } - // Create blob path - multiBlobPath := filepath.Join(multiArchRoot, 'blobs', 'sha256') - err := os.MkdirAll(multiBlobPath) + // Create the blob folder for the multi arch image + multiBlobPath := filepath.Join(multiArchRoot, "blobs", "sha256") + err = os.MkdirAll(multiBlobPath, 0750) if err != nil { return err } - // Create oci-layout file - err := NewOCILayout().WriteTo(multiArchRoot) + // Create the oci-layout file + err = NewOCILayout().WriteTo(multiArchRoot) if err != nil { return err } fmt.Printf("-> oci-layout\n") - // Create index.json - err, index := NewOCIIndex() + // Create the index.json + idx, err := NewOCIImageIndex(o) + if err != nil { + return err + } + err = idx.WriteTo(multiArchRoot) + if err != nil { + return err + } + o.multi = &idx fmt.Printf("-> index.json\n") // Copy blobs for _, img := range o.images { blobCounter := 0 - blobPath := filepath.Join(img.path, 'blobs', 'sha256') + blobPath := filepath.Join(img.path, "blobs", "sha256") blobList, err := os.ReadDir(blobPath) if err != nil { return err } for _, blobFile := range blobList { - src, err := io.Open(filepath.Join(blobPath, blobFile.Name())) + src, err := os.Open(filepath.Join(blobPath, blobFile.Name())) if err != nil { return err } defer src.Close() - dst, err := io.Create(filepath.Join(multiBlobPath, blobFile.Name())) + dst, err := os.Create(filepath.Join(multiBlobPath, blobFile.Name())) if err != nil { return err } defer dst.Close() - _, err := io.Copy(dst, src) + _, err = io.Copy(dst, src) if err != nil { return err } @@ -210,11 +342,144 @@ func (o *OCIMultiArch) MergeSystemImages() error { return nil } -func (o *OCIMultiArch) UploadImage(url string) error { +func (o *OCIMultiArch) UploadImageS3(buck *blob.Bucket) error { + fmt.Printf("-- push to the s3 target --\n") + + // FS paths + multiArchRoot := filepath.Join(o.path, "multi") + multiBlobPath := filepath.Join(multiArchRoot, "blobs", "sha256") + + // Registry URLs + urlPrefix := filepath.Join("v2", o.Name) + manifestPrefix := filepath.Join(urlPrefix, "manifests") + manifestUrl := filepath.Join(manifestPrefix, o.Tag) + blobPrefix := filepath.Join(urlPrefix, "blobs") + + // Utils + cutDigestPrefix := len("sha256:") + + // Checks + if o.multi == nil { + return errors.New("You try to upload a multiarch image that has not been built yet...") + } + + // Upload index + src := filepath.Join(multiArchRoot, "index.json") + up, err := NewUploadFromFS(buck, src) + if err != nil { + return err + } + + err = up.ContentType("application/vnd.oci.image.index.v1+json").UploadTo(manifestUrl) + if err != nil { + return err + } + fmt.Printf("[index] index.json -> %s\n", manifestUrl) + + // Upload the same index but with its sha256 + fd, err := os.Open(src) + if err != nil { + return err + } + defer fd.Close() + + h := sha256.New() + if _, err := io.Copy(h, fd); err != nil { + return err + } + digest := fmt.Sprintf("%x", h.Sum(nil)) + + src = filepath.Join(multiArchRoot, "index.json") + upSha, err := NewUploadFromFS(buck, src) + if err != nil { + return err + } + dst := path.Join(manifestPrefix, "sha256:"+digest) + err = upSha.ContentType("application/vnd.oci.image.index.v1+json").UploadTo(dst) + if err != nil { + return err + } + fmt.Printf("[index] index.json -> %s\n", dst) + + // Upload manifest of each system image + for _, m := range o.multi.Manifests { + src := filepath.Join(multiBlobPath, m.Digest[cutDigestPrefix:]) + upDigest, err := NewUploadFromFS(buck, src) + if err != nil { + return err + } + + dst := path.Join(manifestPrefix, m.Digest) + err = upDigest.ContentType("application/vnd.oci.image.manifest.v1+json").UploadTo(dst) + if err != nil { + return err + } + fmt.Printf("[manifest %s %s] %s -> %s\n", m.Platform.OS, m.Platform.Architecture, src, dst) + } + + // Upload blobs from each system image + for _, ref := range o.multi.Manifests { + fullManifest := filepath.Join(multiBlobPath, ref.Digest[cutDigestPrefix:]) + m, err := LoadOCIImageManifest(fullManifest) + if err != nil { + return err + } + + // Upload config's blob + src := filepath.Join(multiBlobPath, m.Config.Digest[cutDigestPrefix:]) + upConf, err := NewUploadFromFS(buck, src) + if err != nil { + return err + } + + dst := path.Join(blobPrefix, m.Config.Digest) + err = upConf.UploadTo(dst) + if err != nil { + return err + } + fmt.Printf("[config %s %s] %s -> %s\n", ref.Platform.OS, ref.Platform.Architecture, src, dst) + + // Upload layers' blob + counter := 0 + for _, lref := range m.Layers { + src := filepath.Join(multiBlobPath, lref.Digest[cutDigestPrefix:]) + upLayer, err := NewUploadFromFS(buck, src) + if err != nil { + return err + } + + dst := path.Join(blobPrefix, lref.Digest) + err = upLayer.UploadTo(dst) + if err != nil { + return err + } + counter += 1 + + } + fmt.Printf("[blob %s %s] %d items sent\n", ref.Platform.OS, ref.Platform.Architecture, counter) + } + return nil } +type StaticRegistryManager struct { + name string + buck *blob.Bucket +} + +func NewStaticRegistryManager(buck *blob.Bucket, name string) *StaticRegistryManager { + return &StaticRegistryManager{ + buck: buck, + name: name, + } +} + +func (l *StaticRegistryManager) UpdateTagList() error { + fmt.Println("Not yet implemented") + return nil +} + //--- //--- Command logic @@ -251,9 +516,30 @@ var containerPublishCmd = &cobra.Command{ os.Exit(1) } - if err = oi.UploadImage(remotePath); err != nil { - fmt.Println(err) - os.Exit(1) + if strings.HasPrefix(remotePath, "s3:") { + // open bucket + bucket, err := blob.OpenBucket(context.Background(), remotePath) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + defer bucket.Close() + + // upload image + if err = oi.UploadImageS3(bucket); err != nil { + fmt.Println(err) + os.Exit(1) + } + + // update tag + if err = NewStaticRegistryManager(bucket, oi.Name).UpdateTagList(); err != nil { + fmt.Println(err) + os.Exit(1) + } + } else { + fmt.Printf("Protocol not supported for remote path %s. Supported transports are s3:// and docker://\n", remotePath) + os.Exit(1) + } fmt.Printf("✅ push succeeded\n") diff --git a/cmd/static.go b/cmd/static.go index d80b9c9..252ffa9 100644 --- a/cmd/static.go +++ b/cmd/static.go @@ -210,6 +210,7 @@ func (a *Artifact) Upload(buck *blob.Bucket) error { type BucketUploader struct { bucket *blob.Bucket reader io.ReadCloser + options *blob.WriterOptions } func NewUploadFromFS(buck *blob.Bucket, path string) (*BucketUploader, error) { fd, err := os.Open(path) @@ -230,6 +231,15 @@ func NewUploadFromByte(buck *blob.Bucket, content []byte) *BucketUploader { } } +func (bu *BucketUploader) ContentType(ct string) *BucketUploader { + if bu.options == nil { + bu.options = &blob.WriterOptions{} + } + bu.options.ContentType = ct + + return bu +} + func (bu *BucketUploader) UploadTo(key string) error { // Checks if bu.bucket == nil || bu.reader == nil { @@ -237,7 +247,7 @@ func (bu *BucketUploader) UploadTo(key string) error { } // Open remote object - w, err := bu.bucket.NewWriter(context.Background(), key, nil) + w, err := bu.bucket.NewWriter(context.Background(), key, bu.options) if err != nil { return err }