working image push/pull
This commit is contained in:
parent
9b10486aa4
commit
29b6a209a4
GPG key ID:
E9602264D639FF68
2 changed files with 327 additions and 31 deletions
346
cmd/container.go
346
cmd/container.go
|
|
@ -1,18 +1,24 @@
|
|||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/containers/image/v5/transports/alltransports"
|
||||
"github.com/containers/image/v5/signature"
|
||||
"github.com/containers/image/v5/copy"
|
||||
"github.com/spf13/cobra"
|
||||
"gocloud.dev/blob"
|
||||
_ "gocloud.dev/blob/s3blob"
|
||||
)
|
||||
|
||||
var pctx *signature.PolicyContext
|
||||
|
|
@ -21,10 +27,39 @@ const distributionPrefix = "v2"
|
|||
|
||||
//---
|
||||
//--- Image converter
|
||||
type OCIPlatform struct {
|
||||
type OCIImageManifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Config OCIRef `json:"config"`
|
||||
Layers []OCIRef `json:layers"`
|
||||
}
|
||||
func LoadOCIImageManifest(path string) (*OCIImageManifest, error) {
|
||||
fd, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
b, err := io.ReadAll(fd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
oim := &OCIImageManifest{}
|
||||
err = json.Unmarshal(b, oim)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return oim, nil
|
||||
}
|
||||
|
||||
type OCIImageManifest struct {
|
||||
type OCIPlatform struct {
|
||||
Architecture string `json:"architecture"`
|
||||
OS string `json:"os"`
|
||||
}
|
||||
|
||||
type OCIRef struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Digest string `json:"digest"`
|
||||
Size int `json:"size"`
|
||||
|
|
@ -34,10 +69,65 @@ type OCIImageManifest struct {
|
|||
type OCIImageIndex struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
MediaType string `json:"mediaType"`
|
||||
Manifests []OCIImageManifest `json:"manifests"`
|
||||
Manifests []OCIRef `json:"manifests"`
|
||||
}
|
||||
func NewOCIImageIndex(o *OCIMultiArch) (OCIImageIndex, error) {
|
||||
idx := OCIImageIndex {
|
||||
SchemaVersion: 2,
|
||||
MediaType: "application/vnd.oci.image.index.v1+json",
|
||||
Manifests: []OCIRef{},
|
||||
}
|
||||
|
||||
for _, syst := range o.images {
|
||||
if syst.index == nil {
|
||||
return idx, errors.New(fmt.Sprintf("Missing index content for %s. Check that the file exists and that it can be parsed", syst.path))
|
||||
}
|
||||
if len(syst.index.Manifests) != 1 {
|
||||
return idx, errors.New(fmt.Sprintf("%s is not a system image as it does not contain exactly one manifest in its index", syst.path))
|
||||
}
|
||||
if syst.index.Manifests[0].Platform == nil {
|
||||
return idx, errors.New(fmt.Sprintf("Manifest for %s has not been enriched with its platform (os+arch). This is a logic bug.", syst.path))
|
||||
|
||||
}
|
||||
idx.Manifests = append(idx.Manifests, syst.index.Manifests[0])
|
||||
}
|
||||
|
||||
return idx, nil
|
||||
}
|
||||
|
||||
type OCILayout {
|
||||
func LoadOCIImageIndex(path string) (*OCIImageIndex, error) {
|
||||
fd, err := os.Open(filepath.Join(path, "index.json"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
b, err := io.ReadAll(fd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var idx OCIImageIndex
|
||||
err = json.Unmarshal(b, &idx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &idx, nil
|
||||
}
|
||||
|
||||
func (i OCIImageIndex) WriteTo(root string) error {
|
||||
txt, err := json.Marshal(i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
txtPath := filepath.Join(root, "index.json")
|
||||
err = os.WriteFile(txtPath, txt, 0644)
|
||||
return err
|
||||
}
|
||||
|
||||
type OCILayout struct {
|
||||
ImageLayoutVersion string `json:"imageLayoutVersion"`
|
||||
}
|
||||
func NewOCILayout() OCILayout {
|
||||
|
|
@ -50,23 +140,53 @@ func (o OCILayout) WriteTo(root string) error {
|
|||
}
|
||||
|
||||
txtPath := filepath.Join(root, "oci-layout")
|
||||
err = ioutil.WriteFile(txtPath, txt, 0644)
|
||||
err = os.WriteFile(txtPath, txt, 0644)
|
||||
return err
|
||||
}
|
||||
|
||||
//---- Alba logic
|
||||
type OCISystemImage struct {
|
||||
path string
|
||||
os string
|
||||
arch string
|
||||
index *OCIImageIndex
|
||||
}
|
||||
func NewOCISystemImage(path, os, arch string) (OCISystemImage, error) {
|
||||
si := OCISystemImage {
|
||||
path: path,
|
||||
os: os,
|
||||
arch: arch,
|
||||
index: nil,
|
||||
}
|
||||
|
||||
man, err := LoadOCIImageIndex(path)
|
||||
if err != nil {
|
||||
return si, err
|
||||
}
|
||||
|
||||
si.index = man
|
||||
|
||||
if len(si.index.Manifests) != 1 {
|
||||
return si, errors.New(fmt.Sprintf("%s index has not exactly one manifest, this can't be a system image\n", path))
|
||||
}
|
||||
|
||||
// Enrich manifest
|
||||
si.index.Manifests[0].Platform = &OCIPlatform {
|
||||
Architecture: arch,
|
||||
OS: os,
|
||||
}
|
||||
return si, nil
|
||||
}
|
||||
|
||||
func (o OCISystemImage) String() string {
|
||||
return fmt.Sprintf("[oci system image; os:%s, arch:%s, path:%s]", o.os, o.arch, o.path)
|
||||
}
|
||||
|
||||
type OCIMultiArch struct {
|
||||
name string
|
||||
tag string
|
||||
Name string
|
||||
Tag string
|
||||
path string
|
||||
multi *OCIImageIndex
|
||||
images []OCISystemImage
|
||||
}
|
||||
|
||||
|
|
@ -82,8 +202,8 @@ func NewOCIMultiArch(nametag string) (*OCIMultiArch, error) {
|
|||
}
|
||||
|
||||
return &OCIMultiArch {
|
||||
name: ntspl[0],
|
||||
tag: ntspl[1],
|
||||
Name: ntspl[0],
|
||||
Tag: ntspl[1],
|
||||
path: tmp,
|
||||
images: []OCISystemImage{},
|
||||
}, nil
|
||||
|
|
@ -100,6 +220,7 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Convert all docker archives to oci images
|
||||
for idx, imgFile := range imgDirList {
|
||||
imgFilename := imgFile.Name()
|
||||
|
||||
|
|
@ -119,6 +240,7 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error {
|
|||
goos := infos[0]
|
||||
goarch := infos[1]
|
||||
|
||||
// Prepare the image conversion
|
||||
srcRef, err := alltransports.ParseImageName(fmt.Sprintf("docker-archive:%s", archivePath))
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -131,15 +253,17 @@ func (o *OCIMultiArch) LoadFromDockerArchives(path string) error {
|
|||
}
|
||||
|
||||
|
||||
// Convert the docker archive to an oci image
|
||||
_, err = copy.Image(context.Background(), pctx, dstRef, srcRef, ©.Options{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
img := OCISystemImage {
|
||||
path: ociPath,
|
||||
os: goos,
|
||||
arch: goarch,
|
||||
|
||||
img, err := NewOCISystemImage(ociPath, goos, goarch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("%s -> %s\n", archivePath, img)
|
||||
o.images = append(o.images, img)
|
||||
}
|
||||
|
|
@ -151,54 +275,62 @@ func (o *OCIMultiArch) MergeSystemImages() error {
|
|||
fmt.Printf("-- merge system images --\n")
|
||||
multiArchRoot := filepath.Join(o.path, "multi")
|
||||
|
||||
// Create root
|
||||
// Create the root folder for the OCI multi arch image
|
||||
err := os.Mkdir(multiArchRoot, 0750)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create blob path
|
||||
multiBlobPath := filepath.Join(multiArchRoot, 'blobs', 'sha256')
|
||||
err := os.MkdirAll(multiBlobPath)
|
||||
// Create the blob folder for the multi arch image
|
||||
multiBlobPath := filepath.Join(multiArchRoot, "blobs", "sha256")
|
||||
err = os.MkdirAll(multiBlobPath, 0750)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
// Create oci-layout file
|
||||
err := NewOCILayout().WriteTo(multiArchRoot)
|
||||
// Create the oci-layout file
|
||||
err = NewOCILayout().WriteTo(multiArchRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("-> oci-layout\n")
|
||||
|
||||
// Create index.json
|
||||
err, index := NewOCIIndex()
|
||||
// Create the index.json
|
||||
idx, err := NewOCIImageIndex(o)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = idx.WriteTo(multiArchRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
o.multi = &idx
|
||||
fmt.Printf("-> index.json\n")
|
||||
|
||||
|
||||
// Copy blobs
|
||||
for _, img := range o.images {
|
||||
blobCounter := 0
|
||||
blobPath := filepath.Join(img.path, 'blobs', 'sha256')
|
||||
blobPath := filepath.Join(img.path, "blobs", "sha256")
|
||||
blobList, err := os.ReadDir(blobPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, blobFile := range blobList {
|
||||
src, err := io.Open(filepath.Join(blobPath, blobFile.Name()))
|
||||
src, err := os.Open(filepath.Join(blobPath, blobFile.Name()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
dst, err := io.Create(filepath.Join(multiBlobPath, blobFile.Name()))
|
||||
dst, err := os.Create(filepath.Join(multiBlobPath, blobFile.Name()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dst.Close()
|
||||
|
||||
_, err := io.Copy(dst, src)
|
||||
_, err = io.Copy(dst, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -210,11 +342,144 @@ func (o *OCIMultiArch) MergeSystemImages() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (o *OCIMultiArch) UploadImage(url string) error {
|
||||
func (o *OCIMultiArch) UploadImageS3(buck *blob.Bucket) error {
|
||||
fmt.Printf("-- push to the s3 target --\n")
|
||||
|
||||
// FS paths
|
||||
multiArchRoot := filepath.Join(o.path, "multi")
|
||||
multiBlobPath := filepath.Join(multiArchRoot, "blobs", "sha256")
|
||||
|
||||
// Registry URLs
|
||||
urlPrefix := filepath.Join("v2", o.Name)
|
||||
manifestPrefix := filepath.Join(urlPrefix, "manifests")
|
||||
manifestUrl := filepath.Join(manifestPrefix, o.Tag)
|
||||
blobPrefix := filepath.Join(urlPrefix, "blobs")
|
||||
|
||||
// Utils
|
||||
cutDigestPrefix := len("sha256:")
|
||||
|
||||
// Checks
|
||||
if o.multi == nil {
|
||||
return errors.New("You try to upload a multiarch image that has not been built yet...")
|
||||
}
|
||||
|
||||
// Upload index
|
||||
src := filepath.Join(multiArchRoot, "index.json")
|
||||
up, err := NewUploadFromFS(buck, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = up.ContentType("application/vnd.oci.image.index.v1+json").UploadTo(manifestUrl)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("[index] index.json -> %s\n", manifestUrl)
|
||||
|
||||
// Upload the same index but with its sha256
|
||||
fd, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fd.Close()
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, fd); err != nil {
|
||||
return err
|
||||
}
|
||||
digest := fmt.Sprintf("%x", h.Sum(nil))
|
||||
|
||||
src = filepath.Join(multiArchRoot, "index.json")
|
||||
upSha, err := NewUploadFromFS(buck, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dst := path.Join(manifestPrefix, "sha256:"+digest)
|
||||
err = upSha.ContentType("application/vnd.oci.image.index.v1+json").UploadTo(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("[index] index.json -> %s\n", dst)
|
||||
|
||||
// Upload manifest of each system image
|
||||
for _, m := range o.multi.Manifests {
|
||||
src := filepath.Join(multiBlobPath, m.Digest[cutDigestPrefix:])
|
||||
upDigest, err := NewUploadFromFS(buck, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dst := path.Join(manifestPrefix, m.Digest)
|
||||
err = upDigest.ContentType("application/vnd.oci.image.manifest.v1+json").UploadTo(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("[manifest %s %s] %s -> %s\n", m.Platform.OS, m.Platform.Architecture, src, dst)
|
||||
}
|
||||
|
||||
// Upload blobs from each system image
|
||||
for _, ref := range o.multi.Manifests {
|
||||
fullManifest := filepath.Join(multiBlobPath, ref.Digest[cutDigestPrefix:])
|
||||
m, err := LoadOCIImageManifest(fullManifest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload config's blob
|
||||
src := filepath.Join(multiBlobPath, m.Config.Digest[cutDigestPrefix:])
|
||||
upConf, err := NewUploadFromFS(buck, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dst := path.Join(blobPrefix, m.Config.Digest)
|
||||
err = upConf.UploadTo(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("[config %s %s] %s -> %s\n", ref.Platform.OS, ref.Platform.Architecture, src, dst)
|
||||
|
||||
// Upload layers' blob
|
||||
counter := 0
|
||||
for _, lref := range m.Layers {
|
||||
src := filepath.Join(multiBlobPath, lref.Digest[cutDigestPrefix:])
|
||||
upLayer, err := NewUploadFromFS(buck, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dst := path.Join(blobPrefix, lref.Digest)
|
||||
err = upLayer.UploadTo(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
counter += 1
|
||||
|
||||
}
|
||||
fmt.Printf("[blob %s %s] %d items sent\n", ref.Platform.OS, ref.Platform.Architecture, counter)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
type StaticRegistryManager struct {
|
||||
name string
|
||||
buck *blob.Bucket
|
||||
}
|
||||
|
||||
func NewStaticRegistryManager(buck *blob.Bucket, name string) *StaticRegistryManager {
|
||||
return &StaticRegistryManager{
|
||||
buck: buck,
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
|
||||
func (l *StaticRegistryManager) UpdateTagList() error {
|
||||
fmt.Println("Not yet implemented")
|
||||
return nil
|
||||
}
|
||||
|
||||
//---
|
||||
//--- Command logic
|
||||
|
||||
|
|
@ -251,9 +516,30 @@ var containerPublishCmd = &cobra.Command{
|
|||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err = oi.UploadImage(remotePath); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
if strings.HasPrefix(remotePath, "s3:") {
|
||||
// open bucket
|
||||
bucket, err := blob.OpenBucket(context.Background(), remotePath)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer bucket.Close()
|
||||
|
||||
// upload image
|
||||
if err = oi.UploadImageS3(bucket); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// update tag
|
||||
if err = NewStaticRegistryManager(bucket, oi.Name).UpdateTagList(); err != nil {
|
||||
fmt.Println(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("Protocol not supported for remote path %s. Supported transports are s3:// and docker://\n", remotePath)
|
||||
os.Exit(1)
|
||||
|
||||
}
|
||||
|
||||
fmt.Printf("✅ push succeeded\n")
|
||||
|
|
|
|||
|
|
@ -210,6 +210,7 @@ func (a *Artifact) Upload(buck *blob.Bucket) error {
|
|||
type BucketUploader struct {
|
||||
bucket *blob.Bucket
|
||||
reader io.ReadCloser
|
||||
options *blob.WriterOptions
|
||||
}
|
||||
func NewUploadFromFS(buck *blob.Bucket, path string) (*BucketUploader, error) {
|
||||
fd, err := os.Open(path)
|
||||
|
|
@ -230,6 +231,15 @@ func NewUploadFromByte(buck *blob.Bucket, content []byte) *BucketUploader {
|
|||
}
|
||||
}
|
||||
|
||||
func (bu *BucketUploader) ContentType(ct string) *BucketUploader {
|
||||
if bu.options == nil {
|
||||
bu.options = &blob.WriterOptions{}
|
||||
}
|
||||
bu.options.ContentType = ct
|
||||
|
||||
return bu
|
||||
}
|
||||
|
||||
func (bu *BucketUploader) UploadTo(key string) error {
|
||||
// Checks
|
||||
if bu.bucket == nil || bu.reader == nil {
|
||||
|
|
@ -237,7 +247,7 @@ func (bu *BucketUploader) UploadTo(key string) error {
|
|||
}
|
||||
|
||||
// Open remote object
|
||||
w, err := bu.bucket.NewWriter(context.Background(), key, nil)
|
||||
w, err := bu.bucket.NewWriter(context.Background(), key, bu.options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue