diff --git a/dive/get_image_handler.go b/dive/get_image_handler.go index 834ed11..a3eb5a1 100644 --- a/dive/get_image_handler.go +++ b/dive/get_image_handler.go @@ -32,12 +32,12 @@ func GetEngine(engine string) Engine { } } -func GetImageHandler(engine Engine) (image.Image, error) { +func GetImageHandler(engine Engine) (image.Handler, error) { switch engine { case EngineDocker: - return docker.NewDockerImage(), nil + return docker.NewHandler(), nil case EnginePodman: - return podman.NewPodmanImage(), nil + return podman.NewHandler(), nil } return nil, fmt.Errorf("unable to determine image provider") diff --git a/dive/image/docker/image_config.go b/dive/image/docker/config.go similarity index 84% rename from dive/image/docker/image_config.go rename to dive/image/docker/config.go index 704299a..9d2acb9 100644 --- a/dive/image/docker/image_config.go +++ b/dive/image/docker/config.go @@ -5,7 +5,7 @@ import ( "github.com/sirupsen/logrus" ) -type imageConfig struct { +type config struct { History []imageHistoryEntry `json:"history"` RootFs rootFs `json:"rootfs"` } @@ -15,8 +15,8 @@ type rootFs struct { DiffIds []string `json:"diff_ids"` } -func newDockerImageConfig(configBytes []byte) imageConfig { - var imageConfig imageConfig +func newConfig(configBytes []byte) config { + var imageConfig config err := json.Unmarshal(configBytes, &imageConfig) if err != nil { logrus.Panic(err) diff --git a/dive/image/docker/handler.go b/dive/image/docker/handler.go new file mode 100644 index 0000000..d8b1457 --- /dev/null +++ b/dive/image/docker/handler.go @@ -0,0 +1,113 @@ +package docker + +import ( + "fmt" + "github.com/wagoodman/dive/dive/image" + "io" + "net/http" + "os" + "strings" + + "github.com/docker/cli/cli/connhelper" + "github.com/docker/docker/client" + "golang.org/x/net/context" +) + +var dockerVersion string + +type handler struct { + id string + client *client.Client + image Image +} + +func NewHandler() *handler { + return &handler{} +} + +func (handler *handler) Get(id string) error { + handler.id = id + + reader, err := handler.fetchArchive() + if err != nil { + return err + } + defer reader.Close() + + img, err := NewImageFromArchive(reader) + if err != nil { + return err + } + handler.image = img + + return nil +} + +func (handler *handler) Build(args []string) (string, error) { + var err error + handler.id, err = buildImageFromCli(args) + return handler.id, err +} + +func (handler *handler) fetchArchive() (io.ReadCloser, error) { + var err error + + // pull the handler if it does not exist + ctx := context.Background() + + host := os.Getenv("DOCKER_HOST") + var clientOpts []client.Opt + + switch strings.Split(host, ":")[0] { + case "ssh": + helper, err := connhelper.GetConnectionHelper(host) + if err != nil { + fmt.Println("docker host", err) + } + clientOpts = append(clientOpts, func(c *client.Client) error { + httpClient := &http.Client{ + Transport: &http.Transport{ + DialContext: helper.Dialer, + }, + } + return client.WithHTTPClient(httpClient)(c) + }) + clientOpts = append(clientOpts, client.WithHost(helper.Host)) + clientOpts = append(clientOpts, client.WithDialContext(helper.Dialer)) + + default: + + if os.Getenv("DOCKER_TLS_VERIFY") != "" && os.Getenv("DOCKER_CERT_PATH") == "" { + os.Setenv("DOCKER_CERT_PATH", "~/.docker") + } + + clientOpts = append(clientOpts, client.FromEnv) + } + + clientOpts = append(clientOpts, client.WithVersion(dockerVersion)) + handler.client, err = client.NewClientWithOpts(clientOpts...) + if err != nil { + return nil, err + } + _, _, err = handler.client.ImageInspectWithRaw(ctx, handler.id) + if err != nil { + // don't use the API, the CLI has more informative output + fmt.Println("Handler not available locally. Trying to pull '" + handler.id + "'...") + err = runDockerCmd("pull", handler.id) + if err != nil { + return nil, err + } + } + + readCloser, err := handler.client.ImageSave(ctx, []string{handler.id}) + if err != nil { + return nil, err + } + + return readCloser, nil +} + + +func (handler *handler) Analyze() (*image.AnalysisResult, error) { + return handler.image.Analyze() +} diff --git a/dive/image/docker/image.go b/dive/image/docker/image.go index b346bd8..7523c04 100644 --- a/dive/image/docker/image.go +++ b/dive/image/docker/image.go @@ -3,116 +3,28 @@ package docker import ( "archive/tar" "fmt" + "github.com/wagoodman/dive/dive/filetree" "github.com/wagoodman/dive/dive/image" + "github.com/wagoodman/dive/utils" "io" "io/ioutil" - "net/http" - "os" "strings" - - "github.com/docker/cli/cli/connhelper" - "github.com/docker/docker/client" - "github.com/wagoodman/dive/dive/filetree" - "github.com/wagoodman/dive/utils" - "golang.org/x/net/context" ) -var dockerVersion string - -type dockerImage struct { - id string - client *client.Client +type Image struct { jsonFiles map[string][]byte trees []*filetree.FileTree layerMap map[string]*filetree.FileTree layers []*dockerLayer } -func NewDockerImage() *dockerImage { - return &dockerImage{ +func NewImageFromArchive(tarFile io.ReadCloser) (Image, error) { + img := Image{ // store discovered json files in a map so we can read the image in one pass jsonFiles: make(map[string][]byte), layerMap: make(map[string]*filetree.FileTree), } -} -func (img *dockerImage) Get(id string) error { - img.id = id - - reader, err := img.fetch() - if err != nil { - return err - } - defer reader.Close() - - return img.parse(reader) -} - -func (img *dockerImage) Build(args []string) (string, error) { - var err error - img.id, err = buildImageFromCli(args) - return img.id, err -} - -func (img *dockerImage) fetch() (io.ReadCloser, error) { - var err error - - // pull the img if it does not exist - ctx := context.Background() - - host := os.Getenv("DOCKER_HOST") - var clientOpts []client.Opt - - switch strings.Split(host, ":")[0] { - case "ssh": - helper, err := connhelper.GetConnectionHelper(host) - if err != nil { - fmt.Println("docker host", err) - } - clientOpts = append(clientOpts, func(c *client.Client) error { - httpClient := &http.Client{ - Transport: &http.Transport{ - DialContext: helper.Dialer, - }, - } - return client.WithHTTPClient(httpClient)(c) - }) - clientOpts = append(clientOpts, client.WithHost(helper.Host)) - clientOpts = append(clientOpts, client.WithDialContext(helper.Dialer)) - - default: - - if os.Getenv("DOCKER_TLS_VERIFY") != "" && os.Getenv("DOCKER_CERT_PATH") == "" { - os.Setenv("DOCKER_CERT_PATH", "~/.docker") - } - - clientOpts = append(clientOpts, client.FromEnv) - } - - clientOpts = append(clientOpts, client.WithVersion(dockerVersion)) - img.client, err = client.NewClientWithOpts(clientOpts...) - if err != nil { - return nil, err - } - _, _, err = img.client.ImageInspectWithRaw(ctx, img.id) - if err != nil { - // don't use the API, the CLI has more informative output - fmt.Println("Image not available locally. Trying to pull '" + img.id + "'...") - err = runDockerCmd("pull", img.id) - if err != nil { - return nil, err - } - } - - readCloser, err := img.client.ImageSave(ctx, []string{img.id}) - if err != nil { - return nil, err - } - - return readCloser, nil -} - -func (img *dockerImage) parse(tarFile io.ReadCloser) error { tarReader := tar.NewReader(tarFile) var currentLayer uint @@ -136,43 +48,107 @@ func (img *dockerImage) parse(tarFile io.ReadCloser) error { if strings.HasSuffix(name, "layer.tar") { currentLayer++ if err != nil { - return err + return img, err } layerReader := tar.NewReader(tarReader) - err := img.processLayerTar(name, currentLayer, layerReader) + tree, err := processLayerTar(name, layerReader) + if err != nil { - return err + return img, err } + + // add the layer to the image + img.layerMap[tree.Name] = tree + } else if strings.HasSuffix(name, ".json") { fileBuffer, err := ioutil.ReadAll(tarReader) if err != nil { - return err + return img, err } img.jsonFiles[name] = fileBuffer } } } - return nil + return img, nil } -func (img *dockerImage) Analyze() (*image.AnalysisResult, error) { +func processLayerTar(name string, reader *tar.Reader) (*filetree.FileTree, error) { + tree := filetree.NewFileTree() + tree.Name = pathToLayerId(name) + + fileInfos, err := getFileList(reader) + if err != nil { + return nil, err + } + + for _, element := range fileInfos { + tree.FileSize += uint64(element.Size) + + _, _, err := tree.AddPath(element.Path, element) + if err != nil { + return nil, err + } + } + + + return tree, nil +} + +func pathToLayerId(name string) string { + return strings.TrimSuffix(strings.TrimSuffix(name, ".tar"), "/layer") +} + +func getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) { + var files []filetree.FileInfo + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + fmt.Println(err) + utils.Exit(1) + } + + name := header.Name + + switch header.Typeflag { + case tar.TypeXGlobalHeader: + return nil, fmt.Errorf("unexptected tar file: (XGlobalHeader): type=%v name=%s", header.Typeflag, name) + case tar.TypeXHeader: + return nil, fmt.Errorf("unexptected tar file (XHeader): type=%v name=%s", header.Typeflag, name) + default: + files = append(files, filetree.NewFileInfo(tarReader, header, name)) + } + } + return files, nil +} + +func (img *Image) Analyze() (*image.AnalysisResult, error) { + img.trees = make([]*filetree.FileTree, 0) - manifest := newDockerImageManifest(img.jsonFiles["manifest.json"]) - config := newDockerImageConfig(img.jsonFiles[manifest.ConfigPath]) + manifest := newManifest(img.jsonFiles["manifest.json"]) + config := newConfig(img.jsonFiles[manifest.ConfigPath]) // build the content tree for _, treeName := range manifest.LayerTarPaths { - img.trees = append(img.trees, img.layerMap[treeName]) + key := pathToLayerId(treeName) + tr, exists := img.layerMap[key] + if exists { + img.trees = append(img.trees, tr) + continue + } + return nil, fmt.Errorf("could not find '%s' in parsed layers", treeName) } // build the layers array img.layers = make([]*dockerLayer, len(img.trees)) - // note that the img config stores images in reverse chronological order, so iterate backwards through layers + // note that the handler config stores images in reverse chronological order, so iterate backwards through layers // as you iterate chronologically through history (ignoring history items that have no layer contents) - // Note: history is not required metadata in a docker img! + // Note: history is not required metadata in a docker image! tarPathIdx := 0 histIdx := 0 for layerIdx := len(img.trees) - 1; layerIdx >= 0; layerIdx-- { @@ -234,51 +210,3 @@ func (img *dockerImage) Analyze() (*image.AnalysisResult, error) { Inefficiencies: inefficiencies, }, nil } - -func (img *dockerImage) processLayerTar(name string, layerIdx uint, reader *tar.Reader) error { - tree := filetree.NewFileTree() - tree.Name = name - - fileInfos, err := img.getFileList(reader) - if err != nil { - return err - } - - for _, element := range fileInfos { - tree.FileSize += uint64(element.Size) - - _, _, err := tree.AddPath(element.Path, element) - if err != nil { - return err - } - } - - img.layerMap[tree.Name] = tree - return nil -} - -func (img *dockerImage) getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) { - var files []filetree.FileInfo - - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } else if err != nil { - fmt.Println(err) - utils.Exit(1) - } - - name := header.Name - - switch header.Typeflag { - case tar.TypeXGlobalHeader: - return nil, fmt.Errorf("unexptected tar file: (XGlobalHeader): type=%v name=%s", header.Typeflag, name) - case tar.TypeXHeader: - return nil, fmt.Errorf("unexptected tar file (XHeader): type=%v name=%s", header.Typeflag, name) - default: - files = append(files, filetree.NewFileInfo(tarReader, header, name)) - } - } - return files, nil -} diff --git a/dive/image/docker/layer.go b/dive/image/docker/layer.go index 08ff582..9997d3c 100644 --- a/dive/image/docker/layer.go +++ b/dive/image/docker/layer.go @@ -65,31 +65,18 @@ func (layer *dockerLayer) ShortId() string { } id = id[0:rangeBound] - // show the tagged image as the last layer - // if len(layer.History.Tags) > 0 { - // id = "[" + strings.Join(layer.History.Tags, ",") + "]" - // } - return id } -func (layer *dockerLayer) StringFormat() string { - return image.LayerFormat -} - // String represents a layer in a columnar format. func (layer *dockerLayer) String() string { if layer.index == 0 { return fmt.Sprintf(image.LayerFormat, - // layer.ShortId(), - // fmt.Sprintf("%d",layer.Index()), humanize.Bytes(layer.Size()), "FROM "+layer.ShortId()) } return fmt.Sprintf(image.LayerFormat, - // layer.ShortId(), - // fmt.Sprintf("%d",layer.Index()), humanize.Bytes(layer.Size()), layer.Command()) } diff --git a/dive/image/docker/image_manifest.go b/dive/image/docker/manifest.go similarity index 71% rename from dive/image/docker/image_manifest.go rename to dive/image/docker/manifest.go index 084dc72..b08ee05 100644 --- a/dive/image/docker/image_manifest.go +++ b/dive/image/docker/manifest.go @@ -5,14 +5,14 @@ import ( "github.com/sirupsen/logrus" ) -type imageManifest struct { +type manifest struct { ConfigPath string `json:"Config"` RepoTags []string `json:"RepoTags"` LayerTarPaths []string `json:"Layers"` } -func newDockerImageManifest(manifestBytes []byte) imageManifest { - var manifest []imageManifest +func newManifest(manifestBytes []byte) manifest { + var manifest []manifest err := json.Unmarshal(manifestBytes, &manifest) if err != nil { logrus.Panic(err) diff --git a/dive/image/docker/testing.go b/dive/image/docker/testing.go index ed7c6b2..48e465a 100644 --- a/dive/image/docker/testing.go +++ b/dive/image/docker/testing.go @@ -12,15 +12,11 @@ func TestLoadDockerImageTar(tarPath string) (*image.AnalysisResult, error) { } defer f.Close() - img := NewDockerImage() - err = img.Get("dive-test:latest") + handler := NewHandler() + err = handler.Get("dive-test:latest") if err != nil { return nil, err } - err = img.parse(f) - if err != nil { - return nil, err - } - return img.Analyze() + return handler.Analyze() } diff --git a/dive/image/image.go b/dive/image/handler.go similarity index 59% rename from dive/image/image.go rename to dive/image/handler.go index 73d802e..924f5f3 100644 --- a/dive/image/image.go +++ b/dive/image/handler.go @@ -1,6 +1,6 @@ package image -type Image interface { +type Handler interface { Resolver Analyzer } diff --git a/dive/image/podman/handler.go b/dive/image/podman/handler.go new file mode 100644 index 0000000..719f0ba --- /dev/null +++ b/dive/image/podman/handler.go @@ -0,0 +1,89 @@ +package podman + +import ( + "bufio" + "context" + "fmt" + "github.com/containers/libpod/libpod" + "github.com/wagoodman/dive/dive/image" + "github.com/wagoodman/dive/dive/image/docker" + "io/ioutil" + "os" +) + +type handler struct { + id string + // note: podman supports saving docker formatted archives, we're leveraging this here + // todo: add oci parser and image/layer objects + image docker.Image +} + +func NewHandler() *handler { + return &handler{} +} + +func (handler *handler) Get(id string) error { + handler.id = id + + path, err := handler.fetchArchive() + if err != nil { + return err + } + defer os.Remove(path) + + file, err := os.Open(path) + + // we use podman to extract a docker-formatted image + img, err := docker.NewImageFromArchive(ioutil.NopCloser(bufio.NewReader(file))) + if err != nil { + return err + } + + handler.image = img + return nil +} + +func (handler *handler) Build(args []string) (string, error) { + var err error + handler.id, err = buildImageFromCli(args) + return handler.id, err +} + +func (handler *handler) fetchArchive() (string, error) { + var err error + var ctx = context.Background() + + runtime, err := libpod.NewRuntime(ctx) + if err != nil { + return "", err + } + + images, err := runtime.ImageRuntime().GetImages() + if err != nil { + return "", err + } + + for _, item:= range images { + for _, name := range item.Names() { + if name == handler.id { + file, err := ioutil.TempFile(os.TempDir(), "dive-handler-tar") + if err != nil { + return "", err + } + + err = item.Save(ctx, "dive-export", "docker-archive", file.Name(), []string{}, false, false) + if err != nil { + return "", err + } + + return file.Name(), nil + } + } + } + + return "", fmt.Errorf("image could not be found") +} + +func (handler *handler) Analyze() (*image.AnalysisResult, error) { + return handler.image.Analyze() +} diff --git a/dive/image/podman/image.go b/dive/image/podman/image.go deleted file mode 100644 index 4769dd0..0000000 --- a/dive/image/podman/image.go +++ /dev/null @@ -1,269 +0,0 @@ -package podman - -import ( - "context" - "fmt" - "github.com/containers/libpod/libpod" - "os" - - // "github.com/containers/libpod/libpod" - // libpodImage "github.com/containers/libpod/libpod/image" - // "github.com/containers/storage" - "github.com/wagoodman/dive/dive/filetree" - "github.com/wagoodman/dive/dive/image" - "io" -) - -type podmanImage struct { - id string - jsonFiles map[string][]byte - trees []*filetree.FileTree - layerMap map[string]*filetree.FileTree - // layers []*podmanLayer -} - -func NewPodmanImage() *podmanImage { - return &podmanImage{ - // store discovered json files in a map so we can read the image in one pass - jsonFiles: make(map[string][]byte), - layerMap: make(map[string]*filetree.FileTree), - } -} - -func (img *podmanImage) Get(id string) error { - img.id = id - - reader, err := img.fetch() - if err != nil { - return err - } - defer reader.Close() - - return img.parse(reader) -} - -func (img *podmanImage) Build(args []string) (string, error) { - var err error - img.id, err = buildImageFromCli(args) - return img.id, err -} - -func (img *podmanImage) fetch() (io.ReadCloser, error) { - var err error - - runtime, err := libpod.NewRuntime(context.TODO()) - if err != nil { - return nil, err - } - - images, err := runtime.ImageRuntime().GetImages() - if err != nil { - return nil, err - } - - // cfg, _ := runtime.GetConfig() - // cfg.StorageConfig.GraphRoot - - for _, item:= range images { - for _, name := range item.Names() { - if name == img.id { - fmt.Println("Found it!") - - curImg := item - for { - h, _ := curImg.History(context.TODO()) - fmt.Printf("%+v %+v %+v\n", curImg.ID(), h[0].Size, h[0].CreatedBy) - x, _ := curImg.DriverData() - fmt.Printf(" %+v\n", x) - // for _, i := range x { - // fmt.Printf(" %+v\n", i) - // } - - curImg, err = curImg.GetParent(context.TODO()) - if err != nil || curImg == nil { - break - } - } - - } - } - } - - os.Exit(0) - - return nil, err -} - -func (img *podmanImage) parse(tarFile io.ReadCloser) error { - // tarReader := tar.NewReader(tarFile) - // - // var currentLayer uint - // for { - // header, err := tarReader.Next() - // - // if err == io.EOF { - // break - // } - // - // if err != nil { - // fmt.Println(err) - // utils.Exit(1) - // } - // - // name := header.Name - // - // // some layer tars can be relative layer symlinks to other layer tars - // if header.Typeflag == tar.TypeSymlink || header.Typeflag == tar.TypeReg { - // - // if strings.HasSuffix(name, "layer.tar") { - // currentLayer++ - // if err != nil { - // return err - // } - // layerReader := tar.NewReader(tarReader) - // err := img.processLayerTar(name, currentLayer, layerReader) - // if err != nil { - // return err - // } - // } else if strings.HasSuffix(name, ".json") { - // fileBuffer, err := ioutil.ReadAll(tarReader) - // if err != nil { - // return err - // } - // img.jsonFiles[name] = fileBuffer - // } - // } - // } - // - // return nil - return nil -} - -func (img *podmanImage) Analyze() (*image.AnalysisResult, error) { - // img.trees = make([]*filetree.FileTree, 0) - // - // manifest := newDockerImageManifest(img.jsonFiles["manifest.json"]) - // config := newDockerImageConfig(img.jsonFiles[manifest.ConfigPath]) - // - // // build the content tree - // for _, treeName := range manifest.LayerTarPaths { - // img.trees = append(img.trees, img.layerMap[treeName]) - // } - // - // // build the layers array - // img.layers = make([]*dockerLayer, len(img.trees)) - // - // // note that the img config stores images in reverse chronological order, so iterate backwards through layers - // // as you iterate chronologically through history (ignoring history items that have no layer contents) - // // Note: history is not required metadata in a docker img! - // tarPathIdx := 0 - // histIdx := 0 - // for layerIdx := len(img.trees) - 1; layerIdx >= 0; layerIdx-- { - // - // tree := img.trees[(len(img.trees)-1)-layerIdx] - // - // // ignore empty layers, we are only observing layers with content - // historyObj := imageHistoryEntry{ - // CreatedBy: "(missing)", - // } - // for nextHistIdx := histIdx; nextHistIdx < len(config.History); nextHistIdx++ { - // if !config.History[nextHistIdx].EmptyLayer { - // histIdx = nextHistIdx - // break - // } - // } - // if histIdx < len(config.History) && !config.History[histIdx].EmptyLayer { - // historyObj = config.History[histIdx] - // histIdx++ - // } - // - // img.layers[layerIdx] = &dockerLayer{ - // history: historyObj, - // index: tarPathIdx, - // tree: img.trees[layerIdx], - // tarPath: manifest.LayerTarPaths[tarPathIdx], - // } - // img.layers[layerIdx].history.Size = tree.FileSize - // - // tarPathIdx++ - // } - // - // efficiency, inefficiencies := filetree.Efficiency(img.trees) - // - // var sizeBytes, userSizeBytes uint64 - // layers := make([]image.Layer, len(img.layers)) - // for i, v := range img.layers { - // layers[i] = v - // sizeBytes += v.Size() - // if i != 0 { - // userSizeBytes += v.Size() - // } - // } - // - // var wastedBytes uint64 - // for idx := 0; idx < len(inefficiencies); idx++ { - // fileData := inefficiencies[len(inefficiencies)-1-idx] - // wastedBytes += uint64(fileData.CumulativeSize) - // } - // - // return &image.AnalysisResult{ - // Layers: layers, - // RefTrees: img.trees, - // Efficiency: efficiency, - // UserSizeByes: userSizeBytes, - // SizeBytes: sizeBytes, - // WastedBytes: wastedBytes, - // WastedUserPercent: float64(wastedBytes) / float64(userSizeBytes), - // Inefficiencies: inefficiencies, - // }, nil - - return nil, nil -} - -// func (img *podmanImage) processLayerTar(name string, layerIdx uint, reader *tar.Reader) error { -// tree := filetree.NewFileTree() -// tree.Name = name -// -// fileInfos, err := img.getFileList(reader) -// if err != nil { -// return err -// } -// -// for _, element := range fileInfos { -// tree.FileSize += uint64(element.Size) -// -// _, _, err := tree.AddPath(element.Path, element) -// if err != nil { -// return err -// } -// } -// -// img.layerMap[tree.Name] = tree -// return nil -// } -// -// func (img *podmanImage) getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) { -// var files []filetree.FileInfo -// -// for { -// header, err := tarReader.Next() -// if err == io.EOF { -// break -// } else if err != nil { -// fmt.Println(err) -// utils.Exit(1) -// } -// -// name := header.Name -// -// switch header.Typeflag { -// case tar.TypeXGlobalHeader: -// return nil, fmt.Errorf("unexptected tar file: (XGlobalHeader): type=%v name=%s", header.Typeflag, name) -// case tar.TypeXHeader: -// return nil, fmt.Errorf("unexptected tar file (XHeader): type=%v name=%s", header.Typeflag, name) -// default: -// files = append(files, filetree.NewFileInfo(tarReader, header, name)) -// } -// } -// return files, nil -// }