From 49c0002a0ea0b9c68163181df11b5defd0edb7e7 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 3 Oct 2019 23:36:57 -0400 Subject: [PATCH] image derived from podman dir (not working) --- dive/filetree/file_info.go | 66 ++++++--- .../docker/{image.go => image_archive.go} | 4 +- dive/image/docker/resolver.go | 14 +- dive/image/podman/image_directory.go | 132 ++++++++++++++++++ dive/image/podman/layer.go | 68 +++++++++ dive/image/podman/resolver.go | 86 ++++++------ 6 files changed, 298 insertions(+), 72 deletions(-) rename dive/image/docker/{image.go => image_archive.go} (96%) create mode 100644 dive/image/podman/image_directory.go create mode 100644 dive/image/podman/layer.go diff --git a/dive/filetree/file_info.go b/dive/filetree/file_info.go index 4d24925..603deaf 100644 --- a/dive/filetree/file_info.go +++ b/dive/filetree/file_info.go @@ -21,24 +21,13 @@ type FileInfo struct { IsDir bool } -// NewFileInfo extracts the metadata from a tar header and file contents and generates a new FileInfo object. -func NewFileInfo(reader *tar.Reader, header *tar.Header, path string) FileInfo { - if header.Typeflag == tar.TypeDir { - return FileInfo{ - Path: path, - TypeFlag: header.Typeflag, - Linkname: header.Linkname, - hash: 0, - Size: header.FileInfo().Size(), - Mode: header.FileInfo().Mode(), - Uid: header.Uid, - Gid: header.Gid, - IsDir: header.FileInfo().IsDir(), - } +// NewFileInfoFromTarHeader extracts the metadata from a tar header and file contents and generates a new FileInfo object. +func NewFileInfoFromTarHeader(reader *tar.Reader, header *tar.Header, path string) FileInfo { + var hash uint64 + if header.Typeflag != tar.TypeDir { + hash = getHashFromReader(reader) } - hash := getHashFromReader(reader) - return FileInfo{ Path: path, TypeFlag: header.Typeflag, @@ -52,6 +41,51 @@ func NewFileInfo(reader *tar.Reader, header *tar.Header, path string) FileInfo { } } +func NewFileInfo(realPath, path string, info os.FileInfo) FileInfo { + var err error + + // todo: don't use tar types here, create our own... + var fileType byte + if info.Mode() & os.ModeSymlink != 0 { + fileType = tar.TypeSymlink + } else if info.IsDir() { + fileType = tar.TypeDir + } else { + fileType = tar.TypeReg + } + + var hash uint64 + if fileType != tar.TypeDir { + file, err := os.Open(realPath) + if err != nil { + logrus.Panic("unable to read file:", realPath) + } + defer file.Close() + hash = getHashFromReader(file) + } + + var linkName string + if fileType == tar.TypeSymlink { + linkName, err = os.Readlink(realPath) + if err != nil { + logrus.Panic("unable to read link:", realPath, err) + } + } + + return FileInfo{ + Path: path, + TypeFlag: fileType, + Linkname: linkName, + hash: hash, + Size: info.Size(), + Mode: info.Mode(), + // todo: support UID/GID + Uid: -1, + Gid: -1, + IsDir: info.IsDir(), + } +} + // Copy duplicates a FileInfo func (data *FileInfo) Copy() *FileInfo { if data == nil { diff --git a/dive/image/docker/image.go b/dive/image/docker/image_archive.go similarity index 96% rename from dive/image/docker/image.go rename to dive/image/docker/image_archive.go index a74fbb6..69b1772 100644 --- a/dive/image/docker/image.go +++ b/dive/image/docker/image_archive.go @@ -17,7 +17,7 @@ type ImageArchive struct { layerMap map[string]*filetree.FileTree } -func NewImageFromArchive(tarFile io.ReadCloser) (*ImageArchive, error) { +func NewImageArchive(tarFile io.ReadCloser) (*ImageArchive, error) { img := &ImageArchive{ layerMap: make(map[string]*filetree.FileTree), } @@ -128,7 +128,7 @@ func getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) { case tar.TypeXHeader: return nil, fmt.Errorf("unexptected tar file (XHeader): type=%v name=%s", header.Typeflag, name) default: - files = append(files, filetree.NewFileInfo(tarReader, header, name)) + files = append(files, filetree.NewFileInfoFromTarHeader(tarReader, header, name)) } } return files, nil diff --git a/dive/image/docker/resolver.go b/dive/image/docker/resolver.go index 7ce366c..38cfa45 100644 --- a/dive/image/docker/resolver.go +++ b/dive/image/docker/resolver.go @@ -13,10 +13,7 @@ import ( "golang.org/x/net/context" ) -type resolver struct { - id string - client *client.Client -} +type resolver struct {} func NewResolver() *resolver { return &resolver{} @@ -30,7 +27,7 @@ func (r *resolver) Fetch(id string) (*image.Image, error) { } defer reader.Close() - img, err := NewImageFromArchive(reader) + img, err := NewImageArchive(reader) if err != nil { return nil, err } @@ -47,6 +44,7 @@ func (r *resolver) Build(args []string) (*image.Image, error) { func (r *resolver) fetchArchive(id string) (io.ReadCloser, error) { var err error + var dockerClient *client.Client // pull the resolver if it does not exist ctx := context.Background() @@ -81,11 +79,11 @@ func (r *resolver) fetchArchive(id string) (io.ReadCloser, error) { } clientOpts = append(clientOpts, client.WithAPIVersionNegotiation()) - r.client, err = client.NewClientWithOpts(clientOpts...) + dockerClient, err = client.NewClientWithOpts(clientOpts...) if err != nil { return nil, err } - _, _, err = r.client.ImageInspectWithRaw(ctx, id) + _, _, err = dockerClient.ImageInspectWithRaw(ctx, id) if err != nil { // don't use the API, the CLI has more informative output fmt.Println("Handler not available locally. Trying to pull '" + id + "'...") @@ -95,7 +93,7 @@ func (r *resolver) fetchArchive(id string) (io.ReadCloser, error) { } } - readCloser, err := r.client.ImageSave(ctx, []string{id}) + readCloser, err := dockerClient.ImageSave(ctx, []string{id}) if err != nil { return nil, err } diff --git a/dive/image/podman/image_directory.go b/dive/image/podman/image_directory.go new file mode 100644 index 0000000..c66dbd3 --- /dev/null +++ b/dive/image/podman/image_directory.go @@ -0,0 +1,132 @@ +package podman + +import ( + "context" + "fmt" + podmanImage "github.com/containers/libpod/libpod/image" + "github.com/wagoodman/dive/dive/filetree" + "github.com/wagoodman/dive/dive/image" + "os" + "path/filepath" + "strings" +) + +type ImageDirectoryRef struct { + layerOrder []string + treeMap map[string]*filetree.FileTree + layerMap map[string]*podmanImage.Image +} + +func NewImageDirectoryRef(img *podmanImage.Image) (*ImageDirectoryRef, error) { + imgDirRef := &ImageDirectoryRef{ + layerOrder: make([]string, 0), + treeMap: make(map[string]*filetree.FileTree), + layerMap: make(map[string]*podmanImage.Image), + } + + ctx := context.TODO() + + curImg := img + for { + // h, _ := img.History(ctx) + // fmt.Printf("%+v %+v %+v\n", img.ID(), h[0].Size, h[0].CreatedBy) + + driver, err := curImg.DriverData() + if err != nil { + return nil, fmt.Errorf("graph driver error: %+v", err) + } + + if driver.Name != "overlay" { + return nil, fmt.Errorf("unsupported graph driver: %s", driver.Name) + } + + rootDir, exists := driver.Data["UpperDir"] + if !exists { + return nil, fmt.Errorf("graph has no upper dir") + } + + if _, err := os.Stat(rootDir); os.IsNotExist(err) { + return nil, fmt.Errorf("graph root dir does not exist: %s", rootDir) + } + + // build tree from directory... + tree, err := processLayer(curImg.ID(), rootDir) + if err != nil { + return nil, err + } + + // record the tree and layer info + imgDirRef.treeMap[curImg.ID()] = tree + imgDirRef.layerMap[curImg.ID()] = curImg + imgDirRef.layerOrder = append(imgDirRef.layerOrder, curImg.ID()) + + // continue to the next image + curImg, err = curImg.GetParent(ctx) + if err != nil || curImg == nil { + break + } + } + + return imgDirRef, nil +} + +func processLayer(name, rootDir string) (*filetree.FileTree, error) { + tree := filetree.NewFileTree() + tree.Name = name + + err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // add this file to the tree... + fileInfo := filetree.NewFileInfo(path, "/"+strings.TrimPrefix(path, rootDir), info) + + tree.FileSize += uint64(fileInfo.Size) + + _, _, err = tree.AddPath(fileInfo.Path, fileInfo) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("unable to walk upper directory tree") + } + + return tree, nil +} + +func (img *ImageDirectoryRef) ToImage() (*image.Image, error) { + trees := make([]*filetree.FileTree, 0) + // build the content tree + // todo: this isn't needed! + for _, id := range img.layerOrder { + tr, exists := img.treeMap[id] + if exists { + trees = append(trees, tr) + continue + } + return nil, fmt.Errorf("could not find '%s' in parsed trees", id) + } + + layers := make([]image.Layer, len(trees)) + + // note that the resolver config stores images in reverse chronological order, so iterate backwards through layers + // as you iterate chronologically through history (ignoring history items that have no layer contents) + // Note: history is not required metadata in a docker image! + tarPathIdx := 0 + for layerIdx := len(trees) - 1; layerIdx >= 0; layerIdx-- { + id := img.layerOrder[layerIdx] + layers[layerIdx] = &layer{ + obj: img.layerMap[id], + index: tarPathIdx, + tree: trees[layerIdx], + } + } + + return &image.Image{ + Trees: trees, + Layers: layers, + }, nil +} diff --git a/dive/image/podman/layer.go b/dive/image/podman/layer.go new file mode 100644 index 0000000..1323fcc --- /dev/null +++ b/dive/image/podman/layer.go @@ -0,0 +1,68 @@ +package podman + +import ( + "fmt" + podmanImage "github.com/containers/libpod/libpod/image" + "github.com/dustin/go-humanize" + "github.com/wagoodman/dive/dive/filetree" + "github.com/wagoodman/dive/dive/image" + "strings" +) + +// Layer represents a Docker image layer and metadata +type layer struct { + obj *podmanImage.Image + index int + tree *filetree.FileTree +} + +// ShortId returns the truncated id of the current layer. +func (l *layer) Id() string { + return l.obj.ID() +} + +// index returns the relative position of the layer within the image. +func (l *layer) Index() int { + return l.index +} + +// Size returns the number of bytes that this image is. +func (l *layer) Size() uint64 { + return uint64(l.obj.ImageData.Size) +} + +// Tree returns the file tree representing the current layer. +func (l *layer) Tree() *filetree.FileTree { + return l.tree +} + +// ShortId returns the truncated id of the current layer. +func (l *layer) Command() string { + // todo: is 0 right? + return strings.TrimPrefix(l.obj.ImageData.History[0].CreatedBy, "/bin/sh -c ") +} + +// ShortId returns the truncated id of the current layer. +func (l *layer) ShortId() string { + rangeBound := 15 + id := l.Id() + if length := len(id); length < 15 { + rangeBound = length + } + id = id[0:rangeBound] + + return id +} + +// String represents a layer in a columnar format. +func (l *layer) String() string { + + if l.index == 0 { + return fmt.Sprintf(image.LayerFormat, + humanize.Bytes(l.Size()), + "FROM "+l.ShortId()) + } + return fmt.Sprintf(image.LayerFormat, + humanize.Bytes(l.Size()), + l.Command()) +} diff --git a/dive/image/podman/resolver.go b/dive/image/podman/resolver.go index 0589399..2a9126d 100644 --- a/dive/image/podman/resolver.go +++ b/dive/image/podman/resolver.go @@ -31,7 +31,11 @@ func (r *resolver) Fetch(id string) (*image.Image, error) { if err == nil { return img, err } - img, err = r.resolveFromArchive(id) + + // todo: remove print of error + fmt.Println(err) + + img, err = r.resolveFromDockerArchive(id) if err == nil { return img, err } @@ -40,51 +44,41 @@ func (r *resolver) Fetch(id string) (*image.Image, error) { } func (r *resolver) resolveFromDisk(id string) (*image.Image, error) { - // var err error - return nil, fmt.Errorf("not implemented") - // - // runtime, err := libpod.NewRuntime(context.TODO()) - // if err != nil { - // return nil, err - // } - // - // images, err := runtime.ImageRuntime().GetImages() - // if err != nil { - // return nil, err - // } - // - // // cfg, _ := runtime.GetConfig() - // // cfg.StorageConfig.GraphRoot - // - // for _, item:= range images { - // for _, name := range item.Names() { - // if name == id { - // fmt.Println("Found it!") - // - // curImg := item - // for { - // h, _ := curImg.History(context.TODO()) - // fmt.Printf("%+v %+v %+v\n", curImg.ID(), h[0].Size, h[0].CreatedBy) - // x, _ := curImg.DriverData() - // fmt.Printf(" %+v\n", x.Data["UpperDir"]) - // - // - // curImg, err = curImg.GetParent(context.TODO()) - // if err != nil || curImg == nil { - // break - // } - // } - // - // } - // } - // } - // - // // os.Exit(0) - // return nil, nil + var img *ImageDirectoryRef + var err error + + runtime, err := libpod.NewRuntime(context.TODO()) + if err != nil { + return nil, err + } + + images, err := runtime.ImageRuntime().GetImages() + if err != nil { + return nil, err + } + + ImageLoop: + for _, candidateImage := range images { + for _, name := range candidateImage.Names() { + if name == id { + img, err = NewImageDirectoryRef(candidateImage) + if err != nil { + return nil, err + } + break ImageLoop + } + } + } + + if img == nil { + return nil, fmt.Errorf("could not find image by name: '%s'", id) + } + + return img.ToImage() } -func (r *resolver) resolveFromArchive(id string) (*image.Image, error) { - path, err := r.fetchArchive(id) +func (r *resolver) resolveFromDockerArchive(id string) (*image.Image, error) { + path, err := r.fetchDockerArchive(id) if err != nil { return nil, err } @@ -93,14 +87,14 @@ func (r *resolver) resolveFromArchive(id string) (*image.Image, error) { file, err := os.Open(path) defer file.Close() - img, err := docker.NewImageFromArchive(ioutil.NopCloser(bufio.NewReader(file))) + img, err := docker.NewImageArchive(ioutil.NopCloser(bufio.NewReader(file))) if err != nil { return nil, err } return img.ToImage() } -func (r *resolver) fetchArchive(id string) (string, error) { +func (r *resolver) fetchDockerArchive(id string) (string, error) { var err error var ctx = context.Background()