From 8053a8d1aae7e93611e48d9c77b3f99cd7ee0555 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 3 Oct 2019 16:46:29 -0400 Subject: [PATCH] introduced common image format and analyzer --- cmd/analyze.go | 2 +- cmd/build.go | 10 ++-- cmd/root.go | 9 ++-- dive/image/docker/config.go | 13 ++++- dive/image/docker/image.go | 64 +++++++--------------- dive/image/docker/layer.go | 55 +++++++------------ dive/image/docker/resolver.go | 31 ++++++----- dive/image/docker/testing.go | 4 +- dive/image/image.go | 40 ++++++++++++++ dive/image/podman/resolver.go | 93 ++++++++++++++++++++++++-------- dive/image/resolver.go | 4 +- runtime/run.go | 22 ++++---- runtime/ui/details_controller.go | 2 - 13 files changed, 203 insertions(+), 146 deletions(-) create mode 100644 dive/image/image.go diff --git a/cmd/analyze.go b/cmd/analyze.go index e521fc0..1b181cf 100644 --- a/cmd/analyze.go +++ b/cmd/analyze.go @@ -42,7 +42,7 @@ func doAnalyzeCmd(cmd *cobra.Command, args []string) { engine, err := cmd.PersistentFlags().GetString("engine") if err != nil { - fmt.Printf("unable to determine eingine: %v\n", err) + fmt.Printf("unable to determine engine: %v\n", err) utils.Exit(1) } diff --git a/cmd/build.go b/cmd/build.go index dbd25d2..cc6e09a 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -1,8 +1,8 @@ package cmd import ( - "fmt" "github.com/spf13/cobra" + "github.com/spf13/viper" "github.com/wagoodman/dive/dive" "github.com/wagoodman/dive/runtime" "github.com/wagoodman/dive/utils" @@ -26,11 +26,9 @@ func doBuildCmd(cmd *cobra.Command, args []string) { initLogging() - engine, err := cmd.PersistentFlags().GetString("engine") - if err != nil { - fmt.Printf("unable to determine eingine: %v\n", err) - utils.Exit(1) - } + // there is no cli options allowed, only config can be supplied + // todo: allow for an engine flag to be passed to dive but not the container engine + engine := viper.GetString("container-engine") runtime.Run(runtime.Options{ Ci: isCi, diff --git a/cmd/root.go b/cmd/root.go index 6bfb23a..b09b5aa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -65,7 +65,7 @@ func initCli() { rootCmd.PersistentFlags().String("engine", "docker", "The container engine to fetch the image from. Allowed values: "+strings.Join(dive.AllowedEngines, ", ")) - if err := ciConfig.BindPFlag("container-engine.default", rootCmd.PersistentFlags().Lookup("engine")); err != nil { + if err := viper.BindPFlag("container-engine", rootCmd.PersistentFlags().Lookup("engine")); err != nil { log.Fatal("Unable to bind 'engine' flag:", err) } } @@ -104,9 +104,12 @@ func initConfig() { viper.SetDefault("filetree.pane-width", 0.5) viper.SetDefault("filetree.show-attributes", true) - viper.SetDefault("container-engine.default", "docker") + viper.SetDefault("container-engine", "docker") - viper.AutomaticEnv() // read in environment variables that match + viper.SetEnvPrefix("DIVE") + // replace all - with _ when looking for matching environment variables + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + viper.AutomaticEnv() // If a config file is found, read it in. if err := viper.ReadInConfig(); err == nil { diff --git a/dive/image/docker/config.go b/dive/image/docker/config.go index 9d2acb9..0efcb57 100644 --- a/dive/image/docker/config.go +++ b/dive/image/docker/config.go @@ -6,8 +6,8 @@ import ( ) type config struct { - History []imageHistoryEntry `json:"history"` - RootFs rootFs `json:"rootfs"` + History []historyEntry `json:"history"` + RootFs rootFs `json:"rootfs"` } type rootFs struct { @@ -15,6 +15,15 @@ type rootFs struct { DiffIds []string `json:"diff_ids"` } +type historyEntry struct { + ID string + Size uint64 + Created string `json:"created"` + Author string `json:"author"` + CreatedBy string `json:"created_by"` + EmptyLayer bool `json:"empty_layer"` +} + func newConfig(configBytes []byte) config { var imageConfig config err := json.Unmarshal(configBytes, &imageConfig) diff --git a/dive/image/docker/image.go b/dive/image/docker/image.go index f2b8cbd..a74fbb6 100644 --- a/dive/image/docker/image.go +++ b/dive/image/docker/image.go @@ -11,16 +11,14 @@ import ( "strings" ) -type Image struct { +type ImageArchive struct { manifest manifest config config - trees []*filetree.FileTree layerMap map[string]*filetree.FileTree - layers []*dockerLayer } -func NewImageFromArchive(tarFile io.ReadCloser) (*Image, error) { - img := &Image{ +func NewImageFromArchive(tarFile io.ReadCloser) (*ImageArchive, error) { + img := &ImageArchive{ layerMap: make(map[string]*filetree.FileTree), } @@ -110,7 +108,6 @@ func processLayerTar(name string, reader *tar.Reader) (*filetree.FileTree, error return tree, nil } - func getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) { var files []filetree.FileInfo @@ -137,34 +134,32 @@ func getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) { return files, nil } -func (img *Image) Analyze() (*image.AnalysisResult, error) { - - img.trees = make([]*filetree.FileTree, 0) +func (img *ImageArchive) ToImage() (*image.Image, error) { + trees := make([]*filetree.FileTree, 0) // build the content tree for _, treeName := range img.manifest.LayerTarPaths { tr, exists := img.layerMap[treeName] if exists { - img.trees = append(img.trees, tr) + trees = append(trees, tr) continue } return nil, fmt.Errorf("could not find '%s' in parsed layers", treeName) } // build the layers array - img.layers = make([]*dockerLayer, len(img.trees)) + layers := make([]image.Layer, len(trees)) // note that the resolver config stores images in reverse chronological order, so iterate backwards through layers // as you iterate chronologically through history (ignoring history items that have no layer contents) // Note: history is not required metadata in a docker image! tarPathIdx := 0 histIdx := 0 - for layerIdx := len(img.trees) - 1; layerIdx >= 0; layerIdx-- { - - tree := img.trees[(len(img.trees)-1)-layerIdx] + for layerIdx := len(trees) - 1; layerIdx >= 0; layerIdx-- { + tree := trees[(len(trees)-1)-layerIdx] // ignore empty layers, we are only observing layers with content - historyObj := imageHistoryEntry{ + historyObj := historyEntry{ CreatedBy: "(missing)", } for nextHistIdx := histIdx; nextHistIdx < len(img.config.History); nextHistIdx++ { @@ -178,43 +173,20 @@ func (img *Image) Analyze() (*image.AnalysisResult, error) { histIdx++ } - img.layers[layerIdx] = &dockerLayer{ + historyObj.Size = tree.FileSize + + layers[layerIdx] = &layer{ history: historyObj, index: tarPathIdx, - tree: img.trees[layerIdx], - tarPath: img.manifest.LayerTarPaths[tarPathIdx], + tree: trees[layerIdx], } - img.layers[layerIdx].history.Size = tree.FileSize tarPathIdx++ } - efficiency, inefficiencies := filetree.Efficiency(img.trees) - - var sizeBytes, userSizeBytes uint64 - layers := make([]image.Layer, len(img.layers)) - for i, v := range img.layers { - layers[i] = v - sizeBytes += v.Size() - if i != 0 { - userSizeBytes += v.Size() - } - } - - var wastedBytes uint64 - for idx := 0; idx < len(inefficiencies); idx++ { - fileData := inefficiencies[len(inefficiencies)-1-idx] - wastedBytes += uint64(fileData.CumulativeSize) - } - - return &image.AnalysisResult{ - Layers: layers, - RefTrees: img.trees, - Efficiency: efficiency, - UserSizeByes: userSizeBytes, - SizeBytes: sizeBytes, - WastedBytes: wastedBytes, - WastedUserPercent: float64(wastedBytes) / float64(userSizeBytes), - Inefficiencies: inefficiencies, + return &image.Image{ + Trees: trees, + Layers: layers, }, nil + } diff --git a/dive/image/docker/layer.go b/dive/image/docker/layer.go index 9997d3c..9c423b5 100644 --- a/dive/image/docker/layer.go +++ b/dive/image/docker/layer.go @@ -10,56 +10,41 @@ import ( ) // Layer represents a Docker image layer and metadata -type dockerLayer struct { - tarPath string - history imageHistoryEntry +type layer struct { + history historyEntry index int tree *filetree.FileTree } -type imageHistoryEntry struct { - ID string - Size uint64 - Created string `json:"created"` - Author string `json:"author"` - CreatedBy string `json:"created_by"` - EmptyLayer bool `json:"empty_layer"` -} - // ShortId returns the truncated id of the current layer. -func (layer *dockerLayer) TarId() string { - return strings.TrimSuffix(layer.tarPath, "/layer.tar") -} - -// ShortId returns the truncated id of the current layer. -func (layer *dockerLayer) Id() string { - return layer.history.ID +func (l *layer) Id() string { + return l.history.ID } // index returns the relative position of the layer within the image. -func (layer *dockerLayer) Index() int { - return layer.index +func (l *layer) Index() int { + return l.index } // Size returns the number of bytes that this image is. -func (layer *dockerLayer) Size() uint64 { - return layer.history.Size +func (l *layer) Size() uint64 { + return l.history.Size } // Tree returns the file tree representing the current layer. -func (layer *dockerLayer) Tree() *filetree.FileTree { - return layer.tree +func (l *layer) Tree() *filetree.FileTree { + return l.tree } // ShortId returns the truncated id of the current layer. -func (layer *dockerLayer) Command() string { - return strings.TrimPrefix(layer.history.CreatedBy, "/bin/sh -c ") +func (l *layer) Command() string { + return strings.TrimPrefix(l.history.CreatedBy, "/bin/sh -c ") } // ShortId returns the truncated id of the current layer. -func (layer *dockerLayer) ShortId() string { +func (l *layer) ShortId() string { rangeBound := 15 - id := layer.Id() + id := l.Id() if length := len(id); length < 15 { rangeBound = length } @@ -69,14 +54,14 @@ func (layer *dockerLayer) ShortId() string { } // String represents a layer in a columnar format. -func (layer *dockerLayer) String() string { +func (l *layer) String() string { - if layer.index == 0 { + if l.index == 0 { return fmt.Sprintf(image.LayerFormat, - humanize.Bytes(layer.Size()), - "FROM "+layer.ShortId()) + humanize.Bytes(l.Size()), + "FROM "+l.ShortId()) } return fmt.Sprintf(image.LayerFormat, - humanize.Bytes(layer.Size()), - layer.Command()) + humanize.Bytes(l.Size()), + l.Command()) } diff --git a/dive/image/docker/resolver.go b/dive/image/docker/resolver.go index 5d0d3d9..7ce366c 100644 --- a/dive/image/docker/resolver.go +++ b/dive/image/docker/resolver.go @@ -13,8 +13,6 @@ import ( "golang.org/x/net/context" ) -var dockerVersion string - type resolver struct { id string client *client.Client @@ -24,10 +22,9 @@ func NewResolver() *resolver { return &resolver{} } -func (r *resolver) Resolve(id string) (image.Analyzer, error) { - r.id = id +func (r *resolver) Fetch(id string) (*image.Image, error) { - reader, err := r.fetchArchive() + reader, err := r.fetchArchive(id) if err != nil { return nil, err } @@ -37,16 +34,18 @@ func (r *resolver) Resolve(id string) (image.Analyzer, error) { if err != nil { return nil, err } - return img, nil + return img.ToImage() } -func (r *resolver) Build(args []string) (string, error) { - var err error - r.id, err = buildImageFromCli(args) - return r.id, err +func (r *resolver) Build(args []string) (*image.Image, error) { + id, err := buildImageFromCli(args) + if err != nil { + return nil, err + } + return r.Fetch(id) } -func (r *resolver) fetchArchive() (io.ReadCloser, error) { +func (r *resolver) fetchArchive(id string) (io.ReadCloser, error) { var err error // pull the resolver if it does not exist @@ -81,22 +80,22 @@ func (r *resolver) fetchArchive() (io.ReadCloser, error) { clientOpts = append(clientOpts, client.FromEnv) } - clientOpts = append(clientOpts, client.WithVersion(dockerVersion)) + clientOpts = append(clientOpts, client.WithAPIVersionNegotiation()) r.client, err = client.NewClientWithOpts(clientOpts...) if err != nil { return nil, err } - _, _, err = r.client.ImageInspectWithRaw(ctx, r.id) + _, _, err = r.client.ImageInspectWithRaw(ctx, id) if err != nil { // don't use the API, the CLI has more informative output - fmt.Println("Handler not available locally. Trying to pull '" + r.id + "'...") - err = runDockerCmd("pull", r.id) + fmt.Println("Handler not available locally. Trying to pull '" + id + "'...") + err = runDockerCmd("pull", id) if err != nil { return nil, err } } - readCloser, err := r.client.ImageSave(ctx, []string{r.id}) + readCloser, err := r.client.ImageSave(ctx, []string{id}) if err != nil { return nil, err } diff --git a/dive/image/docker/testing.go b/dive/image/docker/testing.go index 0354a29..423d8ad 100644 --- a/dive/image/docker/testing.go +++ b/dive/image/docker/testing.go @@ -12,8 +12,8 @@ func TestLoadDockerImageTar(tarPath string) (*image.AnalysisResult, error) { } defer f.Close() - handler := NewResolver() - img, err := handler.Resolve("dive-test:latest") + resolver := NewResolver() + img, err := resolver.Fetch("dive-test:latest") if err != nil { return nil, err } diff --git a/dive/image/image.go b/dive/image/image.go new file mode 100644 index 0000000..66ed49a --- /dev/null +++ b/dive/image/image.go @@ -0,0 +1,40 @@ +package image + +import ( + "github.com/wagoodman/dive/dive/filetree" +) + +type Image struct { + Trees []*filetree.FileTree + Layers []Layer +} + +func (img *Image) Analyze() (*AnalysisResult, error) { + + efficiency, inefficiencies := filetree.Efficiency(img.Trees) + var sizeBytes, userSizeBytes uint64 + + for i, v := range img.Layers { + sizeBytes += v.Size() + if i != 0 { + userSizeBytes += v.Size() + } + } + + var wastedBytes uint64 + for idx := 0; idx < len(inefficiencies); idx++ { + fileData := inefficiencies[len(inefficiencies)-1-idx] + wastedBytes += uint64(fileData.CumulativeSize) + } + + return &AnalysisResult{ + Layers: img.Layers, + RefTrees: img.Trees, + Efficiency: efficiency, + UserSizeByes: userSizeBytes, + SizeBytes: sizeBytes, + WastedBytes: wastedBytes, + WastedUserPercent: float64(wastedBytes) / float64(userSizeBytes), + Inefficiencies: inefficiencies, + }, nil +} diff --git a/dive/image/podman/resolver.go b/dive/image/podman/resolver.go index 87da3a8..0589399 100644 --- a/dive/image/podman/resolver.go +++ b/dive/image/podman/resolver.go @@ -11,43 +11,96 @@ import ( "os" ) -type resolver struct { - id string - // note: podman supports saving docker formatted archives, we're leveraging this here - // todo: add oci parser and image/layer objects - image docker.Image -} +type resolver struct {} func NewResolver() *resolver { return &resolver{} } -func (handler *resolver) Resolve(id string) (image.Analyzer, error) { - handler.id = id +func (r *resolver) Build(args []string) (*image.Image, error) { + id, err := buildImageFromCli(args) + if err != nil { + return nil, err + } + return r.Fetch(id) +} - path, err := handler.fetchArchive() + +func (r *resolver) Fetch(id string) (*image.Image, error) { + img, err := r.resolveFromDisk(id) + if err == nil { + return img, err + } + img, err = r.resolveFromArchive(id) + if err == nil { + return img, err + } + + return nil, fmt.Errorf("unable to resolve image '%s'", id) +} + +func (r *resolver) resolveFromDisk(id string) (*image.Image, error) { + // var err error + return nil, fmt.Errorf("not implemented") + // + // runtime, err := libpod.NewRuntime(context.TODO()) + // if err != nil { + // return nil, err + // } + // + // images, err := runtime.ImageRuntime().GetImages() + // if err != nil { + // return nil, err + // } + // + // // cfg, _ := runtime.GetConfig() + // // cfg.StorageConfig.GraphRoot + // + // for _, item:= range images { + // for _, name := range item.Names() { + // if name == id { + // fmt.Println("Found it!") + // + // curImg := item + // for { + // h, _ := curImg.History(context.TODO()) + // fmt.Printf("%+v %+v %+v\n", curImg.ID(), h[0].Size, h[0].CreatedBy) + // x, _ := curImg.DriverData() + // fmt.Printf(" %+v\n", x.Data["UpperDir"]) + // + // + // curImg, err = curImg.GetParent(context.TODO()) + // if err != nil || curImg == nil { + // break + // } + // } + // + // } + // } + // } + // + // // os.Exit(0) + // return nil, nil +} + +func (r *resolver) resolveFromArchive(id string) (*image.Image, error) { + path, err := r.fetchArchive(id) if err != nil { return nil, err } defer os.Remove(path) file, err := os.Open(path) + defer file.Close() img, err := docker.NewImageFromArchive(ioutil.NopCloser(bufio.NewReader(file))) if err != nil { return nil, err } - - return img, nil + return img.ToImage() } -func (handler *resolver) Build(args []string) (string, error) { - var err error - handler.id, err = buildImageFromCli(args) - return handler.id, err -} - -func (handler *resolver) fetchArchive() (string, error) { +func (r *resolver) fetchArchive(id string) (string, error) { var err error var ctx = context.Background() @@ -63,7 +116,7 @@ func (handler *resolver) fetchArchive() (string, error) { for _, item:= range images { for _, name := range item.Names() { - if name == handler.id { + if name == id { file, err := ioutil.TempFile(os.TempDir(), "dive-resolver-tar") if err != nil { return "", err @@ -74,8 +127,6 @@ func (handler *resolver) fetchArchive() (string, error) { return "", err } - fmt.Println(file.Name()) - return file.Name(), nil } } diff --git a/dive/image/resolver.go b/dive/image/resolver.go index 825f557..aaa2407 100644 --- a/dive/image/resolver.go +++ b/dive/image/resolver.go @@ -1,6 +1,6 @@ package image type Resolver interface { - Resolve(id string) (Analyzer, error) - Build(options []string) (string, error) + Fetch(id string) (*Image, error) + Build(options []string) (*Image, error) } diff --git a/runtime/run.go b/runtime/run.go index 66c04e6..a6fca8d 100644 --- a/runtime/run.go +++ b/runtime/run.go @@ -40,29 +40,31 @@ func Run(options Options) { // if build is given, get the handler based off of either the explicit runtime - imageHandler, err := dive.GetImageHandler(options.Engine) + imageResolver, err := dive.GetImageHandler(options.Engine) if err != nil { fmt.Printf("cannot determine image provider: %v\n", err) utils.Exit(1) } + var img *image.Image + if doBuild { fmt.Println(utils.TitleFormat("Building image...")) - options.ImageId, err = imageHandler.Build(options.BuildArgs) + img, err = imageResolver.Build(options.BuildArgs) if err != nil { fmt.Printf("cannot build image: %v\n", err) utils.Exit(1) } - } - - imgAnalyzer, err := imageHandler.Resolve(options.ImageId) - if err != nil { - fmt.Printf("cannot fetch image: %v\n", err) - utils.Exit(1) + } else { + img, err = imageResolver.Fetch(options.ImageId) + if err != nil { + fmt.Printf("cannot fetch image: %v\n", err) + utils.Exit(1) + } } // todo, cleanup on error - // todo: image get shold return error for cleanup? + // todo: image get should return error for cleanup? if doExport { fmt.Println(utils.TitleFormat(fmt.Sprintf("Analyzing image... (export to '%s')", options.ExportFile))) @@ -70,7 +72,7 @@ func Run(options Options) { fmt.Println(utils.TitleFormat("Analyzing image...")) } - result, err := imgAnalyzer.Analyze() + result, err := img.Analyze() if err != nil { fmt.Printf("cannot analyze image: %v\n", err) utils.Exit(1) diff --git a/runtime/ui/details_controller.go b/runtime/ui/details_controller.go index ab9d7ec..92d7a6a 100644 --- a/runtime/ui/details_controller.go +++ b/runtime/ui/details_controller.go @@ -126,8 +126,6 @@ func (controller *DetailsController) Render() error { // update contents controller.view.Clear() _, _ = fmt.Fprintln(controller.view, Formatting.Header("Digest: ")+currentLayer.Id()) - // TODO: add back in with controller model - // fmt.Fprintln(view.view, Formatting.Header("Tar ID: ")+currentLayer.TarId()) _, _ = fmt.Fprintln(controller.view, Formatting.Header("Command:")) _, _ = fmt.Fprintln(controller.view, currentLayer.Command())