norohind/dive-zfs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

leveraging docker export for podman

Browse Source
This commit is contained in:
Alex Goodman 2019-10-03 09:42:27 -04:00
parent acfdd70854
commit d2c661eaf7
No known key found for this signature in database
GPG Key ID: 98AF011C5C78EB7E
10 changed files with 295 additions and 451 deletions

6
dive/get_image_handler.go

@ -32,12 +32,12 @@ func GetEngine(engine string) Engine {
}
}
func GetImageHandler(engine Engine) (image.Image, error) {
func GetImageHandler(engine Engine) (image.Handler, error) {
switch engine {
case EngineDocker:
return docker.NewDockerImage(), nil
return docker.NewHandler(), nil
case EnginePodman:
return podman.NewPodmanImage(), nil
return podman.NewHandler(), nil
}
return nil, fmt.Errorf("unable to determine image provider")

6
dive/image/docker/image_config.go → dive/image/docker/config.go

@ -5,7 +5,7 @@ import (
"github.com/sirupsen/logrus"
)
type imageConfig struct {
type config struct {
History []imageHistoryEntry `json:"history"`
RootFs rootFs `json:"rootfs"`
}
@ -15,8 +15,8 @@ type rootFs struct {
DiffIds []string `json:"diff_ids"`
}
func newDockerImageConfig(configBytes []byte) imageConfig {
var imageConfig imageConfig
func newConfig(configBytes []byte) config {
var imageConfig config
err := json.Unmarshal(configBytes, &imageConfig)
if err != nil {
logrus.Panic(err)

113
dive/image/docker/handler.go Normal file

@ -0,0 +1,113 @@
package docker
import (
"fmt"
"github.com/wagoodman/dive/dive/image"
"io"
"net/http"
"os"
"strings"
"github.com/docker/cli/cli/connhelper"
"github.com/docker/docker/client"
"golang.org/x/net/context"
)
var dockerVersion string
type handler struct {
id string
client *client.Client
image Image
}
func NewHandler() *handler {
return &handler{}
}
func (handler *handler) Get(id string) error {
handler.id = id
reader, err := handler.fetchArchive()
if err != nil {
return err
}
defer reader.Close()
img, err := NewImageFromArchive(reader)
if err != nil {
return err
}
handler.image = img
return nil
}
func (handler *handler) Build(args []string) (string, error) {
var err error
handler.id, err = buildImageFromCli(args)
return handler.id, err
}
func (handler *handler) fetchArchive() (io.ReadCloser, error) {
var err error
// pull the handler if it does not exist
ctx := context.Background()
host := os.Getenv("DOCKER_HOST")
var clientOpts []client.Opt
switch strings.Split(host, ":")[0] {
case "ssh":
helper, err := connhelper.GetConnectionHelper(host)
if err != nil {
fmt.Println("docker host", err)
}
clientOpts = append(clientOpts, func(c *client.Client) error {
httpClient := &http.Client{
Transport: &http.Transport{
DialContext: helper.Dialer,
},
}
return client.WithHTTPClient(httpClient)(c)
})
clientOpts = append(clientOpts, client.WithHost(helper.Host))
clientOpts = append(clientOpts, client.WithDialContext(helper.Dialer))
default:
if os.Getenv("DOCKER_TLS_VERIFY") != "" && os.Getenv("DOCKER_CERT_PATH") == "" {
os.Setenv("DOCKER_CERT_PATH", "~/.docker")
}
clientOpts = append(clientOpts, client.FromEnv)
}
clientOpts = append(clientOpts, client.WithVersion(dockerVersion))
handler.client, err = client.NewClientWithOpts(clientOpts...)
if err != nil {
return nil, err
}
_, _, err = handler.client.ImageInspectWithRaw(ctx, handler.id)
if err != nil {
// don't use the API, the CLI has more informative output
fmt.Println("Handler not available locally. Trying to pull '" + handler.id + "'...")
err = runDockerCmd("pull", handler.id)
if err != nil {
return nil, err
}
}
readCloser, err := handler.client.ImageSave(ctx, []string{handler.id})
if err != nil {
return nil, err
}
return readCloser, nil
}
func (handler *handler) Analyze() (*image.AnalysisResult, error) {
return handler.image.Analyze()
}

232
dive/image/docker/image.go

@ -3,116 +3,28 @@ package docker
import (
"archive/tar"
"fmt"
"github.com/wagoodman/dive/dive/filetree"
"github.com/wagoodman/dive/dive/image"
"github.com/wagoodman/dive/utils"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
"github.com/docker/cli/cli/connhelper"
"github.com/docker/docker/client"
"github.com/wagoodman/dive/dive/filetree"
"github.com/wagoodman/dive/utils"
"golang.org/x/net/context"
)
var dockerVersion string
type dockerImage struct {
id string
client *client.Client
type Image struct {
jsonFiles map[string][]byte
trees []*filetree.FileTree
layerMap map[string]*filetree.FileTree
layers []*dockerLayer
}
func NewDockerImage() *dockerImage {
return &dockerImage{
func NewImageFromArchive(tarFile io.ReadCloser) (Image, error) {
img := Image{
// store discovered json files in a map so we can read the image in one pass
jsonFiles: make(map[string][]byte),
layerMap: make(map[string]*filetree.FileTree),
}
}
func (img *dockerImage) Get(id string) error {
img.id = id
reader, err := img.fetch()
if err != nil {
return err
}
defer reader.Close()
return img.parse(reader)
}
func (img *dockerImage) Build(args []string) (string, error) {
var err error
img.id, err = buildImageFromCli(args)
return img.id, err
}
func (img *dockerImage) fetch() (io.ReadCloser, error) {
var err error
// pull the img if it does not exist
ctx := context.Background()
host := os.Getenv("DOCKER_HOST")
var clientOpts []client.Opt
switch strings.Split(host, ":")[0] {
case "ssh":
helper, err := connhelper.GetConnectionHelper(host)
if err != nil {
fmt.Println("docker host", err)
}
clientOpts = append(clientOpts, func(c *client.Client) error {
httpClient := &http.Client{
Transport: &http.Transport{
DialContext: helper.Dialer,
},
}
return client.WithHTTPClient(httpClient)(c)
})
clientOpts = append(clientOpts, client.WithHost(helper.Host))
clientOpts = append(clientOpts, client.WithDialContext(helper.Dialer))
default:
if os.Getenv("DOCKER_TLS_VERIFY") != "" && os.Getenv("DOCKER_CERT_PATH") == "" {
os.Setenv("DOCKER_CERT_PATH", "~/.docker")
}
clientOpts = append(clientOpts, client.FromEnv)
}
clientOpts = append(clientOpts, client.WithVersion(dockerVersion))
img.client, err = client.NewClientWithOpts(clientOpts...)
if err != nil {
return nil, err
}
_, _, err = img.client.ImageInspectWithRaw(ctx, img.id)
if err != nil {
// don't use the API, the CLI has more informative output
fmt.Println("Image not available locally. Trying to pull '" + img.id + "'...")
err = runDockerCmd("pull", img.id)
if err != nil {
return nil, err
}
}
readCloser, err := img.client.ImageSave(ctx, []string{img.id})
if err != nil {
return nil, err
}
return readCloser, nil
}
func (img *dockerImage) parse(tarFile io.ReadCloser) error {
tarReader := tar.NewReader(tarFile)
var currentLayer uint
@ -136,43 +48,107 @@ func (img *dockerImage) parse(tarFile io.ReadCloser) error {
if strings.HasSuffix(name, "layer.tar") {
currentLayer++
if err != nil {
return err
return img, err
}
layerReader := tar.NewReader(tarReader)
err := img.processLayerTar(name, currentLayer, layerReader)
tree, err := processLayerTar(name, layerReader)
if err != nil {
return err
return img, err
}
// add the layer to the image
img.layerMap[tree.Name] = tree
} else if strings.HasSuffix(name, ".json") {
fileBuffer, err := ioutil.ReadAll(tarReader)
if err != nil {
return err
return img, err
}
img.jsonFiles[name] = fileBuffer
}
}
}
return nil
return img, nil
}
func (img *dockerImage) Analyze() (*image.AnalysisResult, error) {
func processLayerTar(name string, reader *tar.Reader) (*filetree.FileTree, error) {
tree := filetree.NewFileTree()
tree.Name = pathToLayerId(name)
fileInfos, err := getFileList(reader)
if err != nil {
return nil, err
}
for _, element := range fileInfos {
tree.FileSize += uint64(element.Size)
_, _, err := tree.AddPath(element.Path, element)
if err != nil {
return nil, err
}
}
return tree, nil
}
func pathToLayerId(name string) string {
return strings.TrimSuffix(strings.TrimSuffix(name, ".tar"), "/layer")
}
func getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) {
var files []filetree.FileInfo
for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
fmt.Println(err)
utils.Exit(1)
}
name := header.Name
switch header.Typeflag {
case tar.TypeXGlobalHeader:
return nil, fmt.Errorf("unexptected tar file: (XGlobalHeader): type=%v name=%s", header.Typeflag, name)
case tar.TypeXHeader:
return nil, fmt.Errorf("unexptected tar file (XHeader): type=%v name=%s", header.Typeflag, name)
default:
files = append(files, filetree.NewFileInfo(tarReader, header, name))
}
}
return files, nil
}
func (img *Image) Analyze() (*image.AnalysisResult, error) {
img.trees = make([]*filetree.FileTree, 0)
manifest := newDockerImageManifest(img.jsonFiles["manifest.json"])
config := newDockerImageConfig(img.jsonFiles[manifest.ConfigPath])
manifest := newManifest(img.jsonFiles["manifest.json"])
config := newConfig(img.jsonFiles[manifest.ConfigPath])
// build the content tree
for _, treeName := range manifest.LayerTarPaths {
img.trees = append(img.trees, img.layerMap[treeName])
key := pathToLayerId(treeName)
tr, exists := img.layerMap[key]
if exists {
img.trees = append(img.trees, tr)
continue
}
return nil, fmt.Errorf("could not find '%s' in parsed layers", treeName)
}
// build the layers array
img.layers = make([]*dockerLayer, len(img.trees))
// note that the img config stores images in reverse chronological order, so iterate backwards through layers
// note that the handler config stores images in reverse chronological order, so iterate backwards through layers
// as you iterate chronologically through history (ignoring history items that have no layer contents)
// Note: history is not required metadata in a docker img!
// Note: history is not required metadata in a docker image!
tarPathIdx := 0
histIdx := 0
for layerIdx := len(img.trees) - 1; layerIdx >= 0; layerIdx-- {
@ -234,51 +210,3 @@ func (img *dockerImage) Analyze() (*image.AnalysisResult, error) {
Inefficiencies: inefficiencies,
}, nil
}
func (img *dockerImage) processLayerTar(name string, layerIdx uint, reader *tar.Reader) error {
tree := filetree.NewFileTree()
tree.Name = name
fileInfos, err := img.getFileList(reader)
if err != nil {
return err
}
for _, element := range fileInfos {
tree.FileSize += uint64(element.Size)
_, _, err := tree.AddPath(element.Path, element)
if err != nil {
return err
}
}
img.layerMap[tree.Name] = tree
return nil
}
func (img *dockerImage) getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) {
var files []filetree.FileInfo
for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
fmt.Println(err)
utils.Exit(1)
}
name := header.Name
switch header.Typeflag {
case tar.TypeXGlobalHeader:
return nil, fmt.Errorf("unexptected tar file: (XGlobalHeader): type=%v name=%s", header.Typeflag, name)
case tar.TypeXHeader:
return nil, fmt.Errorf("unexptected tar file (XHeader): type=%v name=%s", header.Typeflag, name)
default:
files = append(files, filetree.NewFileInfo(tarReader, header, name))
}
}
return files, nil
}

13
dive/image/docker/layer.go

@ -65,31 +65,18 @@ func (layer *dockerLayer) ShortId() string {
}
id = id[0:rangeBound]
// show the tagged image as the last layer
// if len(layer.History.Tags) > 0 {
// id = "[" + strings.Join(layer.History.Tags, ",") + "]"
// }
return id
}
func (layer *dockerLayer) StringFormat() string {
return image.LayerFormat
}
// String represents a layer in a columnar format.
func (layer *dockerLayer) String() string {
if layer.index == 0 {
return fmt.Sprintf(image.LayerFormat,
// layer.ShortId(),
// fmt.Sprintf("%d",layer.Index()),
humanize.Bytes(layer.Size()),
"FROM "+layer.ShortId())
}
return fmt.Sprintf(image.LayerFormat,
// layer.ShortId(),
// fmt.Sprintf("%d",layer.Index()),
humanize.Bytes(layer.Size()),
layer.Command())
}

6
dive/image/docker/image_manifest.go → dive/image/docker/manifest.go

@ -5,14 +5,14 @@ import (
"github.com/sirupsen/logrus"
)
type imageManifest struct {
type manifest struct {
ConfigPath string `json:"Config"`
RepoTags []string `json:"RepoTags"`
LayerTarPaths []string `json:"Layers"`
}
func newDockerImageManifest(manifestBytes []byte) imageManifest {
var manifest []imageManifest
func newManifest(manifestBytes []byte) manifest {
var manifest []manifest
err := json.Unmarshal(manifestBytes, &manifest)
if err != nil {
logrus.Panic(err)

10
dive/image/docker/testing.go

@ -12,15 +12,11 @@ func TestLoadDockerImageTar(tarPath string) (*image.AnalysisResult, error) {
}
defer f.Close()
img := NewDockerImage()
err = img.Get("dive-test:latest")
handler := NewHandler()
err = handler.Get("dive-test:latest")
if err != nil {
return nil, err
}
err = img.parse(f)
if err != nil {
return nil, err
}
return img.Analyze()
return handler.Analyze()
}

2
dive/image/image.go → dive/image/handler.go

@ -1,6 +1,6 @@
package image
type Image interface {
type Handler interface {
Resolver
Analyzer
}

89
dive/image/podman/handler.go Normal file

@ -0,0 +1,89 @@
package podman
import (
"bufio"
"context"
"fmt"
"github.com/containers/libpod/libpod"
"github.com/wagoodman/dive/dive/image"
"github.com/wagoodman/dive/dive/image/docker"
"io/ioutil"
"os"
)
type handler struct {
id string
// note: podman supports saving docker formatted archives, we're leveraging this here
// todo: add oci parser and image/layer objects
image docker.Image
}
func NewHandler() *handler {
return &handler{}
}
func (handler *handler) Get(id string) error {
handler.id = id
path, err := handler.fetchArchive()
if err != nil {
return err
}
defer os.Remove(path)
file, err := os.Open(path)
// we use podman to extract a docker-formatted image
img, err := docker.NewImageFromArchive(ioutil.NopCloser(bufio.NewReader(file)))
if err != nil {
return err
}
handler.image = img
return nil
}
func (handler *handler) Build(args []string) (string, error) {
var err error
handler.id, err = buildImageFromCli(args)
return handler.id, err
}
func (handler *handler) fetchArchive() (string, error) {
var err error
var ctx = context.Background()
runtime, err := libpod.NewRuntime(ctx)
if err != nil {
return "", err
}
images, err := runtime.ImageRuntime().GetImages()
if err != nil {
return "", err
}
for _, item:= range images {
for _, name := range item.Names() {
if name == handler.id {
file, err := ioutil.TempFile(os.TempDir(), "dive-handler-tar")
if err != nil {
return "", err
}
err = item.Save(ctx, "dive-export", "docker-archive", file.Name(), []string{}, false, false)
if err != nil {
return "", err
}
return file.Name(), nil
}
}
}
return "", fmt.Errorf("image could not be found")
}
func (handler *handler) Analyze() (*image.AnalysisResult, error) {
return handler.image.Analyze()
}

269
dive/image/podman/image.go

@ -1,269 +0,0 @@
package podman
import (
"context"
"fmt"
"github.com/containers/libpod/libpod"
"os"
// "github.com/containers/libpod/libpod"
// libpodImage "github.com/containers/libpod/libpod/image"
// "github.com/containers/storage"
"github.com/wagoodman/dive/dive/filetree"
"github.com/wagoodman/dive/dive/image"
"io"
)
type podmanImage struct {
id string
jsonFiles map[string][]byte
trees []*filetree.FileTree
layerMap map[string]*filetree.FileTree
// layers []*podmanLayer
}
func NewPodmanImage() *podmanImage {
return &podmanImage{
// store discovered json files in a map so we can read the image in one pass
jsonFiles: make(map[string][]byte),
layerMap: make(map[string]*filetree.FileTree),
}
}
func (img *podmanImage) Get(id string) error {
img.id = id
reader, err := img.fetch()
if err != nil {
return err
}
defer reader.Close()
return img.parse(reader)
}
func (img *podmanImage) Build(args []string) (string, error) {
var err error
img.id, err = buildImageFromCli(args)
return img.id, err
}
func (img *podmanImage) fetch() (io.ReadCloser, error) {
var err error
runtime, err := libpod.NewRuntime(context.TODO())
if err != nil {
return nil, err
}
images, err := runtime.ImageRuntime().GetImages()
if err != nil {
return nil, err
}
// cfg, _ := runtime.GetConfig()
// cfg.StorageConfig.GraphRoot
for _, item:= range images {
for _, name := range item.Names() {
if name == img.id {
fmt.Println("Found it!")
curImg := item
for {
h, _ := curImg.History(context.TODO())
fmt.Printf("%+v %+v %+v\n", curImg.ID(), h[0].Size, h[0].CreatedBy)
x, _ := curImg.DriverData()
fmt.Printf(" %+v\n", x)
// for _, i := range x {
// fmt.Printf(" %+v\n", i)
// }
curImg, err = curImg.GetParent(context.TODO())
if err != nil || curImg == nil {
break
}
}
}
}
}
os.Exit(0)
return nil, err
}
func (img *podmanImage) parse(tarFile io.ReadCloser) error {
// tarReader := tar.NewReader(tarFile)
//
// var currentLayer uint
// for {
// header, err := tarReader.Next()
//
// if err == io.EOF {
// break
// }
//
// if err != nil {
// fmt.Println(err)
// utils.Exit(1)
// }
//
// name := header.Name
//
// // some layer tars can be relative layer symlinks to other layer tars
// if header.Typeflag == tar.TypeSymlink || header.Typeflag == tar.TypeReg {
//
// if strings.HasSuffix(name, "layer.tar") {
// currentLayer++
// if err != nil {
// return err
// }
// layerReader := tar.NewReader(tarReader)
// err := img.processLayerTar(name, currentLayer, layerReader)
// if err != nil {
// return err
// }
// } else if strings.HasSuffix(name, ".json") {
// fileBuffer, err := ioutil.ReadAll(tarReader)
// if err != nil {
// return err
// }
// img.jsonFiles[name] = fileBuffer
// }
// }
// }
//
// return nil
return nil
}
func (img *podmanImage) Analyze() (*image.AnalysisResult, error) {
// img.trees = make([]*filetree.FileTree, 0)
//
// manifest := newDockerImageManifest(img.jsonFiles["manifest.json"])
// config := newDockerImageConfig(img.jsonFiles[manifest.ConfigPath])
//
// // build the content tree
// for _, treeName := range manifest.LayerTarPaths {
// img.trees = append(img.trees, img.layerMap[treeName])
// }
//
// // build the layers array
// img.layers = make([]*dockerLayer, len(img.trees))
//
// // note that the img config stores images in reverse chronological order, so iterate backwards through layers
// // as you iterate chronologically through history (ignoring history items that have no layer contents)
// // Note: history is not required metadata in a docker img!
// tarPathIdx := 0
// histIdx := 0
// for layerIdx := len(img.trees) - 1; layerIdx >= 0; layerIdx-- {
//
// tree := img.trees[(len(img.trees)-1)-layerIdx]
//
// // ignore empty layers, we are only observing layers with content
// historyObj := imageHistoryEntry{
// CreatedBy: "(missing)",
// }
// for nextHistIdx := histIdx; nextHistIdx < len(config.History); nextHistIdx++ {
// if !config.History[nextHistIdx].EmptyLayer {
// histIdx = nextHistIdx
// break
// }
// }
// if histIdx < len(config.History) && !config.History[histIdx].EmptyLayer {
// historyObj = config.History[histIdx]
// histIdx++
// }
//
// img.layers[layerIdx] = &dockerLayer{
// history: historyObj,
// index: tarPathIdx,
// tree: img.trees[layerIdx],
// tarPath: manifest.LayerTarPaths[tarPathIdx],
// }
// img.layers[layerIdx].history.Size = tree.FileSize
//
// tarPathIdx++
// }
//
// efficiency, inefficiencies := filetree.Efficiency(img.trees)
//
// var sizeBytes, userSizeBytes uint64
// layers := make([]image.Layer, len(img.layers))
// for i, v := range img.layers {
// layers[i] = v
// sizeBytes += v.Size()
// if i != 0 {
// userSizeBytes += v.Size()
// }
// }
//
// var wastedBytes uint64
// for idx := 0; idx < len(inefficiencies); idx++ {
// fileData := inefficiencies[len(inefficiencies)-1-idx]
// wastedBytes += uint64(fileData.CumulativeSize)
// }
//
// return &image.AnalysisResult{
// Layers: layers,
// RefTrees: img.trees,
// Efficiency: efficiency,
// UserSizeByes: userSizeBytes,
// SizeBytes: sizeBytes,
// WastedBytes: wastedBytes,
// WastedUserPercent: float64(wastedBytes) / float64(userSizeBytes),
// Inefficiencies: inefficiencies,
// }, nil
return nil, nil
}
// func (img *podmanImage) processLayerTar(name string, layerIdx uint, reader *tar.Reader) error {
// tree := filetree.NewFileTree()
// tree.Name = name
//
// fileInfos, err := img.getFileList(reader)
// if err != nil {
// return err
// }
//
// for _, element := range fileInfos {
// tree.FileSize += uint64(element.Size)
//
// _, _, err := tree.AddPath(element.Path, element)
// if err != nil {
// return err
// }
// }
//
// img.layerMap[tree.Name] = tree
// return nil
// }
//
// func (img *podmanImage) getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) {
// var files []filetree.FileInfo
//
// for {
// header, err := tarReader.Next()
// if err == io.EOF {
// break
// } else if err != nil {
// fmt.Println(err)
// utils.Exit(1)
// }
//
// name := header.Name
//
// switch header.Typeflag {
// case tar.TypeXGlobalHeader:
// return nil, fmt.Errorf("unexptected tar file: (XGlobalHeader): type=%v name=%s", header.Typeflag, name)
// case tar.TypeXHeader:
// return nil, fmt.Errorf("unexptected tar file (XHeader): type=%v name=%s", header.Typeflag, name)
// default:
// files = append(files, filetree.NewFileInfo(tarReader, header, name))
// }
// }
// return files, nil
// }