From 6ff2eb1c2db2d4489328c0a07fbca6c90f614774 Mon Sep 17 00:00:00 2001 From: zhouhao Date: Mon, 8 May 2017 10:47:26 +0800 Subject: [PATCH 1/2] make the image-tool work with -rc5 Signed-off-by: zhouhao --- image/descriptor.go | 43 ++++++++++++++++---------- image/image_test.go | 70 ++++++++++++++++++++++++++++++------------ image/manifest_test.go | 4 +-- 3 files changed, 78 insertions(+), 39 deletions(-) diff --git a/image/descriptor.go b/image/descriptor.go index 21bae8b..cb7aa14 100644 --- a/image/descriptor.go +++ b/image/descriptor.go @@ -23,17 +23,14 @@ import ( "strings" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) -type descriptor struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int64 `json:"size"` -} +type descriptor v1.Descriptor func (d *descriptor) algo() string { - pts := strings.SplitN(d.Digest, ":", 2) + pts := strings.SplitN(string(d.Digest), ":", 2) if len(pts) != 2 { return "" } @@ -41,7 +38,7 @@ func (d *descriptor) algo() string { } func (d *descriptor) hash() string { - pts := strings.SplitN(d.Digest, ":", 2) + pts := strings.SplitN(string(d.Digest), ":", 2) if len(pts) != 2 { return "" } @@ -50,17 +47,22 @@ func (d *descriptor) hash() string { func listReferences(w walker) (map[string]*descriptor, error) { refs := make(map[string]*descriptor) + var index v1.ImageIndex if err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || !strings.HasPrefix(path, "refs") { + if info.IsDir() || filepath.Clean(path) != "index.json" { return nil } - var d descriptor - if err := json.NewDecoder(r).Decode(&d); err != nil { + if err := json.NewDecoder(r).Decode(&index); err != nil { return err } - refs[info.Name()] = &d + + for i := 0; i < len(index.Manifests); i++ { + if index.Manifests[i].Descriptor.Annotations["org.opencontainers.ref.name"] != "" { + refs[index.Manifests[i].Descriptor.Annotations["org.opencontainers.ref.name"]] = (*descriptor)(&index.Manifests[i].Descriptor) + } + } return nil }); err != nil { @@ -71,21 +73,28 @@ func listReferences(w walker) (map[string]*descriptor, error) { func findDescriptor(w walker, name string) (*descriptor, error) { var d descriptor - dpath := filepath.Join("refs", name) + var index v1.ImageIndex switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != dpath { + if info.IsDir() || filepath.Clean(path) != "index.json" { return nil } - if err := json.NewDecoder(r).Decode(&d); err != nil { + if err := json.NewDecoder(r).Decode(&index); err != nil { return err } - return errEOW + for i := 0; i < len(index.Manifests); i++ { + if index.Manifests[i].Descriptor.Annotations["org.opencontainers.ref.name"] == name { + d = (descriptor)(index.Manifests[i].Descriptor) + return errEOW + } + } + + return nil }); err { case nil: - return nil, fmt.Errorf("%s: descriptor not found", dpath) + return nil, fmt.Errorf("index.json: descriptor not found") case errEOW: return &d, nil default: @@ -105,7 +114,7 @@ func (d *descriptor) validate(w walker, mts []string) error { return fmt.Errorf("invalid descriptor MediaType %q", d.MediaType) } - parsed, err := digest.Parse(d.Digest) + parsed, err := digest.Parse(string(d.Digest)) if err != nil { return err } diff --git a/image/image_test.go b/image/image_test.go index 35ecb29..a8182a5 100644 --- a/image/image_test.go +++ b/image/image_test.go @@ -31,8 +31,7 @@ import ( ) const ( - refTag = "latest" - + refTag = "latest" layoutStr = `{"imageLayoutVersion": "1.0.0"}` configStr = `{ @@ -91,8 +90,44 @@ const ( ) var ( - refStr = `{"digest":"","mediaType":"application/vnd.oci.image.manifest.v1+json","size":}` - + indexStr = `{ + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "size": , + "digest": "", + "annotations": { + "org.opencontainers.ref.name": "v1.0" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": , + "digest": "", + "platform": { + "architecture": "ppc64le", + "os": "linux" + }, + "annotations": { + "org.opencontainers.ref.name": "latest" + } + }, + { + "mediaType": "application/xml", + "size": , + "digest": "", + "annotations": { + "org.freedesktop.specifications.metainfo.version": "1.0", + "org.freedesktop.specifications.metainfo.type": "AppStream" + } + } + ], + "annotations": { + "com.example.index.revision": "r124356" + } +} + ` manifestStr = `{ "annotations": null, "config": { @@ -162,11 +197,6 @@ func createImageLayoutBundle(il imageLayout) error { return err } - err = os.MkdirAll(filepath.Join(il.rootDir, "refs"), 0700) - if err != nil { - return err - } - // create image layout file err = createLayoutFile(il.rootDir) if err != nil { @@ -178,14 +208,14 @@ func createImageLayoutBundle(il imageLayout) error { if err != nil { return err } - il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) + il.manifest = strings.Replace(il.manifest, "", string(desc.Digest), 1) il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) desc, err = createConfigFile(il.rootDir, il.config) if err != nil { return err } - il.manifest = strings.Replace(il.manifest, "", desc.Digest, 1) + il.manifest = strings.Replace(il.manifest, "", string(desc.Digest), 1) il.manifest = strings.Replace(il.manifest, "", strconv.FormatInt(desc.Size, 10), 1) // create manifest blob file @@ -194,7 +224,7 @@ func createImageLayoutBundle(il imageLayout) error { return err } - return createRefFile(il.rootDir, il.ref, desc) + return createIndexFile(il.rootDir, desc) } func createLayoutFile(root string) error { @@ -208,16 +238,16 @@ func createLayoutFile(root string) error { return err } -func createRefFile(root, ref string, mft descriptor) error { - refpath := filepath.Join(root, "refs", ref) - f, err := os.Create(refpath) +func createIndexFile(root string, mft descriptor) error { + indexpath := filepath.Join(root, "index.json") + f, err := os.Create(indexpath) if err != nil { return err } defer f.Close() - refStr = strings.Replace(refStr, "", mft.Digest, -1) - refStr = strings.Replace(refStr, "", strconv.FormatInt(mft.Size, 10), -1) - _, err = io.Copy(f, bytes.NewBuffer([]byte(refStr))) + indexStr = strings.Replace(indexStr, "", string(mft.Digest), -1) + indexStr = strings.Replace(indexStr, "", strconv.FormatInt(mft.Size, 10), -1) + _, err = io.Copy(f, bytes.NewBuffer([]byte(indexStr))) return err } @@ -297,7 +327,7 @@ func createHashedBlob(name string) (descriptor, error) { return descriptor{}, err } - parsed, err := digest.Parse(desc.Digest) + parsed, err := digest.Parse(string(desc.Digest)) if err != nil { return descriptor{}, err } @@ -325,7 +355,7 @@ func newDescriptor(name string) (descriptor, error) { } return descriptor{ - Digest: digester.Digest().String(), + Digest: digester.Digest(), Size: size, }, nil } diff --git a/image/manifest_test.go b/image/manifest_test.go index bf7cc26..1a6ee21 100644 --- a/image/manifest_test.go +++ b/image/manifest_test.go @@ -143,7 +143,7 @@ func testUnpackLayer(t *testing.T, compression string, invalid bool) { testManifest := manifest{ Layers: []descriptor{descriptor{ MediaType: mediatype, - Digest: digester.Digest().String(), + Digest: digester.Digest(), }}, } err = testManifest.unpack(newPathWalker(tmp1), filepath.Join(tmp1, "rootfs")) @@ -212,7 +212,7 @@ func TestUnpackLayerRemovePartialyUnpackedFile(t *testing.T) { testManifest := manifest{ Layers: []descriptor{descriptor{ MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", - Digest: digester.Digest().String(), + Digest: digester.Digest(), }}, } err = testManifest.unpack(newPathWalker(tmp1), filepath.Join(tmp1, "rootfs")) From 017b697f7114979afec0ddb296615a50b18d8fad Mon Sep 17 00:00:00 2001 From: zhouhao Date: Wed, 14 Jun 2017 13:49:06 +0800 Subject: [PATCH 2/2] image: replace the extra function Signed-off-by: zhouhao --- image/config.go | 4 ++-- image/descriptor.go | 39 +++++++++------------------------------ image/image.go | 6 +++--- image/image_test.go | 41 ++++++++++++++++++++--------------------- image/manifest.go | 16 ++++++++-------- image/manifest_test.go | 5 +++-- image/walker.go | 11 ++++++----- 7 files changed, 51 insertions(+), 71 deletions(-) diff --git a/image/config.go b/image/config.go index 7ef0ac2..3ba66b5 100644 --- a/image/config.go +++ b/image/config.go @@ -33,9 +33,9 @@ import ( type config v1.Image -func findConfig(w walker, d *descriptor) (*config, error) { +func findConfig(w walker, d *v1.Descriptor) (*config, error) { var c config - cpath := filepath.Join("blobs", d.algo(), d.hash()) + cpath := filepath.Join("blobs", string(d.Digest.Algorithm()), d.Digest.Hex()) switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { if info.IsDir() || filepath.Clean(path) != cpath { diff --git a/image/descriptor.go b/image/descriptor.go index cb7aa14..6d16fd6 100644 --- a/image/descriptor.go +++ b/image/descriptor.go @@ -20,33 +20,13 @@ import ( "io" "os" "path/filepath" - "strings" - "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) -type descriptor v1.Descriptor - -func (d *descriptor) algo() string { - pts := strings.SplitN(string(d.Digest), ":", 2) - if len(pts) != 2 { - return "" - } - return pts[0] -} - -func (d *descriptor) hash() string { - pts := strings.SplitN(string(d.Digest), ":", 2) - if len(pts) != 2 { - return "" - } - return pts[1] -} - -func listReferences(w walker) (map[string]*descriptor, error) { - refs := make(map[string]*descriptor) +func listReferences(w walker) (map[string]*v1.Descriptor, error) { + refs := make(map[string]*v1.Descriptor) var index v1.ImageIndex if err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { @@ -60,7 +40,7 @@ func listReferences(w walker) (map[string]*descriptor, error) { for i := 0; i < len(index.Manifests); i++ { if index.Manifests[i].Descriptor.Annotations["org.opencontainers.ref.name"] != "" { - refs[index.Manifests[i].Descriptor.Annotations["org.opencontainers.ref.name"]] = (*descriptor)(&index.Manifests[i].Descriptor) + refs[index.Manifests[i].Descriptor.Annotations["org.opencontainers.ref.name"]] = &index.Manifests[i].Descriptor } } @@ -71,8 +51,8 @@ func listReferences(w walker) (map[string]*descriptor, error) { return refs, nil } -func findDescriptor(w walker, name string) (*descriptor, error) { - var d descriptor +func findDescriptor(w walker, name string) (*v1.Descriptor, error) { + var d v1.Descriptor var index v1.ImageIndex switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { @@ -86,7 +66,7 @@ func findDescriptor(w walker, name string) (*descriptor, error) { for i := 0; i < len(index.Manifests); i++ { if index.Manifests[i].Descriptor.Annotations["org.opencontainers.ref.name"] == name { - d = (descriptor)(index.Manifests[i].Descriptor) + d = index.Manifests[i].Descriptor return errEOW } } @@ -102,7 +82,7 @@ func findDescriptor(w walker, name string) (*descriptor, error) { } } -func (d *descriptor) validate(w walker, mts []string) error { +func validateDescriptor(d *v1.Descriptor, w walker, mts []string) error { var found bool for _, mt := range mts { if d.MediaType == mt { @@ -114,13 +94,12 @@ func (d *descriptor) validate(w walker, mts []string) error { return fmt.Errorf("invalid descriptor MediaType %q", d.MediaType) } - parsed, err := digest.Parse(string(d.Digest)) - if err != nil { + if err := d.Digest.Validate(); err != nil { return err } // Copy the contents of the layer in to the verifier - verifier := parsed.Verifier() + verifier := d.Digest.Verifier() numBytes, err := w.get(*d, verifier) if err != nil { return err diff --git a/image/image.go b/image/image.go index 4ff345c..9318af1 100644 --- a/image/image.go +++ b/image/image.go @@ -84,7 +84,7 @@ func validate(w walker, refs []string, out *log.Logger) error { return fmt.Errorf("reference %s not found", ref) } - if err = d.validate(w, validRefMediaTypes); err != nil { + if err = validateDescriptor(d, w, validRefMediaTypes); err != nil { return err } @@ -135,7 +135,7 @@ func unpack(w walker, dest, refName string) error { return err } - if err = ref.validate(w, validRefMediaTypes); err != nil { + if err = validateDescriptor(ref, w, validRefMediaTypes); err != nil { return err } @@ -183,7 +183,7 @@ func createRuntimeBundle(w walker, dest, refName, rootfs string) error { return err } - if err = ref.validate(w, validRefMediaTypes); err != nil { + if err = validateDescriptor(ref, w, validRefMediaTypes); err != nil { return err } diff --git a/image/image_test.go b/image/image_test.go index a8182a5..e83440b 100644 --- a/image/image_test.go +++ b/image/image_test.go @@ -238,7 +238,7 @@ func createLayoutFile(root string) error { return err } -func createIndexFile(root string, mft descriptor) error { +func createIndexFile(root string, mft v1.Descriptor) error { indexpath := filepath.Join(root, "index.json") f, err := os.Create(indexpath) if err != nil { @@ -251,48 +251,48 @@ func createIndexFile(root string, mft descriptor) error { return err } -func createManifestFile(root, str string) (descriptor, error) { +func createManifestFile(root, str string) (v1.Descriptor, error) { name := filepath.Join(root, "blobs", "sha256", "test-manifest") f, err := os.Create(name) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } defer f.Close() _, err = io.Copy(f, bytes.NewBuffer([]byte(str))) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } return createHashedBlob(name) } -func createConfigFile(root, config string) (descriptor, error) { +func createConfigFile(root, config string) (v1.Descriptor, error) { name := filepath.Join(root, "blobs", "sha256", "test-config") f, err := os.Create(name) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } defer f.Close() _, err = io.Copy(f, bytes.NewBuffer([]byte(config))) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } return createHashedBlob(name) } -func createImageLayerFile(root string, list []tarContent) (descriptor, error) { +func createImageLayerFile(root string, list []tarContent) (v1.Descriptor, error) { name := filepath.Join(root, "blobs", "sha256", "test-layer") err := createTarBlob(name, list) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } desc, err := createHashedBlob(name) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } desc.MediaType = v1.MediaTypeImageLayer @@ -321,40 +321,39 @@ func createTarBlob(name string, list []tarContent) error { return nil } -func createHashedBlob(name string) (descriptor, error) { +func createHashedBlob(name string) (v1.Descriptor, error) { desc, err := newDescriptor(name) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } - parsed, err := digest.Parse(string(desc.Digest)) - if err != nil { - return descriptor{}, err + if err := desc.Digest.Validate(); err != nil { + return v1.Descriptor{}, err } // Rename the file to hashed-digest name. - err = os.Rename(name, filepath.Join(filepath.Dir(name), parsed.Hex())) + err = os.Rename(name, filepath.Join(filepath.Dir(name), desc.Digest.Hex())) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } return desc, nil } -func newDescriptor(name string) (descriptor, error) { +func newDescriptor(name string) (v1.Descriptor, error) { file, err := os.Open(name) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } defer file.Close() digester := digest.SHA256.Digester() size, err := io.Copy(digester.Hash(), file) if err != nil { - return descriptor{}, err + return v1.Descriptor{}, err } - return descriptor{ + return v1.Descriptor{ Digest: digester.Digest(), Size: size, }, nil diff --git a/image/manifest.go b/image/manifest.go index 8834c1e..a827ab6 100644 --- a/image/manifest.go +++ b/image/manifest.go @@ -36,13 +36,13 @@ import ( ) type manifest struct { - Config descriptor `json:"config"` - Layers []descriptor `json:"layers"` + Config v1.Descriptor `json:"config"` + Layers []v1.Descriptor `json:"layers"` } -func findManifest(w walker, d *descriptor) (*manifest, error) { +func findManifest(w walker, d *v1.Descriptor) (*manifest, error) { var m manifest - mpath := filepath.Join("blobs", d.algo(), d.hash()) + mpath := filepath.Join("blobs", string(d.Digest.Algorithm()), d.Digest.Hex()) switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { if info.IsDir() || filepath.Clean(path) != mpath { @@ -74,7 +74,7 @@ func findManifest(w walker, d *descriptor) (*manifest, error) { } func (m *manifest) validate(w walker) error { - if err := m.Config.validate(w, []string{v1.MediaTypeImageConfig}); err != nil { + if err := validateDescriptor(&m.Config, w, []string{v1.MediaTypeImageConfig}); err != nil { return errors.Wrap(err, "config validation failed") } @@ -86,7 +86,7 @@ func (m *manifest) validate(w walker) error { } for _, d := range m.Layers { - if err := d.validate(w, validLayerMediaTypes); err != nil { + if err := validateDescriptor(&d, w, validLayerMediaTypes); err != nil { return errors.Wrap(err, "layer validation failed") } } @@ -118,8 +118,8 @@ func (m *manifest) unpack(w walker, dest string) (retErr error) { return nil } - dd, err := filepath.Rel(filepath.Join("blobs", d.algo()), filepath.Clean(path)) - if err != nil || d.hash() != dd { + dd, err := filepath.Rel(filepath.Join("blobs", string(d.Digest.Algorithm())), filepath.Clean(path)) + if err != nil || d.Digest.Hex() != dd { return nil } diff --git a/image/manifest_test.go b/image/manifest_test.go index 1a6ee21..1657ad0 100644 --- a/image/manifest_test.go +++ b/image/manifest_test.go @@ -27,6 +27,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" bz2 "github.com/dsnet/compress/bzip2" @@ -141,7 +142,7 @@ func testUnpackLayer(t *testing.T, compression string, invalid bool) { } testManifest := manifest{ - Layers: []descriptor{descriptor{ + Layers: []v1.Descriptor{v1.Descriptor{ MediaType: mediatype, Digest: digester.Digest(), }}, @@ -210,7 +211,7 @@ func TestUnpackLayerRemovePartialyUnpackedFile(t *testing.T) { } testManifest := manifest{ - Layers: []descriptor{descriptor{ + Layers: []v1.Descriptor{v1.Descriptor{ MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", Digest: digester.Digest(), }}, diff --git a/image/walker.go b/image/walker.go index 3ef5e8b..8bf369f 100644 --- a/image/walker.go +++ b/image/walker.go @@ -22,6 +22,7 @@ import ( "path/filepath" "sync" + "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -40,7 +41,7 @@ type walker interface { // get will copy an arbitrary blob, defined by desc, in to dst. returns // the number of bytes copied on success. - get(desc descriptor, dst io.Writer) (int64, error) + get(desc v1.Descriptor, dst io.Writer) (int64, error) } // tarWalker exposes access to image layouts in a tar file. @@ -87,11 +88,11 @@ loop: return nil } -func (w *tarWalker) get(desc descriptor, dst io.Writer) (int64, error) { +func (w *tarWalker) get(desc v1.Descriptor, dst io.Writer) (int64, error) { var bytes int64 done := false - expectedPath := filepath.Join("blobs", desc.algo(), desc.hash()) + expectedPath := filepath.Join("blobs", string(desc.Digest.Algorithm()), desc.Digest.Hex()) f := func(path string, info os.FileInfo, rdr io.Reader) error { var err error @@ -161,8 +162,8 @@ func (w *pathWalker) walk(f walkFunc) error { }) } -func (w *pathWalker) get(desc descriptor, dst io.Writer) (int64, error) { - name := filepath.Join(w.root, "blobs", desc.algo(), desc.hash()) +func (w *pathWalker) get(desc v1.Descriptor, dst io.Writer) (int64, error) { + name := filepath.Join(w.root, "blobs", string(desc.Digest.Algorithm()), desc.Digest.Hex()) info, err := os.Stat(name) if err != nil {