norohind/dive-zfs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

introduced common image format and analyzer

Browse Source
This commit is contained in:
Alex Goodman 2019-10-03 16:46:29 -04:00
parent 50d776e845
commit 8053a8d1aa
No known key found for this signature in database
GPG Key ID: 98AF011C5C78EB7E
13 changed files with 203 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/analyze.go
View File

@ -42,7 +42,7 @@ func doAnalyzeCmd(cmd *cobra.Command, args []string) {
engine, err := cmd.PersistentFlags().GetString("engine")
if err != nil {
fmt.Printf("unable to determine eingine: %v\n", err)
fmt.Printf("unable to determine engine: %v\n", err)
utils.Exit(1)
}

10
cmd/build.go
View File

@ -1,8 +1,8 @@
package cmd
import (
"fmt"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/wagoodman/dive/dive"
"github.com/wagoodman/dive/runtime"
"github.com/wagoodman/dive/utils"
@ -26,11 +26,9 @@ func doBuildCmd(cmd *cobra.Command, args []string) {
initLogging()
engine, err := cmd.PersistentFlags().GetString("engine")
if err != nil {
fmt.Printf("unable to determine eingine: %v\n", err)
utils.Exit(1)
}
// there is no cli options allowed, only config can be supplied
// todo: allow for an engine flag to be passed to dive but not the container engine
engine := viper.GetString("container-engine")
runtime.Run(runtime.Options{
Ci: isCi,

9
cmd/root.go
View File

@ -65,7 +65,7 @@ func initCli() {
rootCmd.PersistentFlags().String("engine", "docker", "The container engine to fetch the image from. Allowed values: "+strings.Join(dive.AllowedEngines, ", "))
if err := ciConfig.BindPFlag("container-engine.default", rootCmd.PersistentFlags().Lookup("engine")); err != nil {
if err := viper.BindPFlag("container-engine", rootCmd.PersistentFlags().Lookup("engine")); err != nil {
log.Fatal("Unable to bind 'engine' flag:", err)
}
}
@ -104,9 +104,12 @@ func initConfig() {
viper.SetDefault("filetree.pane-width", 0.5)
viper.SetDefault("filetree.show-attributes", true)
viper.SetDefault("container-engine.default", "docker")
viper.SetDefault("container-engine", "docker")
viper.AutomaticEnv() // read in environment variables that match
viper.SetEnvPrefix("DIVE")
// replace all - with _ when looking for matching environment variables
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
viper.AutomaticEnv()
// If a config file is found, read it in.
if err := viper.ReadInConfig(); err == nil {

13
dive/image/docker/config.go
View File

@ -6,8 +6,8 @@ import (
)
type config struct {
History []imageHistoryEntry `json:"history"`
RootFs rootFs `json:"rootfs"`
History []historyEntry `json:"history"`
RootFs rootFs `json:"rootfs"`
}
type rootFs struct {
@ -15,6 +15,15 @@ type rootFs struct {
DiffIds []string `json:"diff_ids"`
}
type historyEntry struct {
ID string
Size uint64
Created string `json:"created"`
Author string `json:"author"`
CreatedBy string `json:"created_by"`
EmptyLayer bool `json:"empty_layer"`
}
func newConfig(configBytes []byte) config {
var imageConfig config
err := json.Unmarshal(configBytes, &imageConfig)

64
dive/image/docker/image.go
View File

@ -11,16 +11,14 @@ import (
"strings"
)
type Image struct {
type ImageArchive struct {
manifest manifest
config config
trees []*filetree.FileTree
layerMap map[string]*filetree.FileTree
layers []*dockerLayer
}
func NewImageFromArchive(tarFile io.ReadCloser) (*Image, error) {
img := &Image{
func NewImageFromArchive(tarFile io.ReadCloser) (*ImageArchive, error) {
img := &ImageArchive{
layerMap: make(map[string]*filetree.FileTree),
}
@ -110,7 +108,6 @@ func processLayerTar(name string, reader *tar.Reader) (*filetree.FileTree, error
return tree, nil
}
func getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) {
var files []filetree.FileInfo
@ -137,34 +134,32 @@ func getFileList(tarReader *tar.Reader) ([]filetree.FileInfo, error) {
return files, nil
}
func (img *Image) Analyze() (*image.AnalysisResult, error) {
img.trees = make([]*filetree.FileTree, 0)
func (img *ImageArchive) ToImage() (*image.Image, error) {
trees := make([]*filetree.FileTree, 0)
// build the content tree
for _, treeName := range img.manifest.LayerTarPaths {
tr, exists := img.layerMap[treeName]
if exists {
img.trees = append(img.trees, tr)
trees = append(trees, tr)
continue
}
return nil, fmt.Errorf("could not find '%s' in parsed layers", treeName)
}
// build the layers array
img.layers = make([]*dockerLayer, len(img.trees))
layers := make([]image.Layer, len(trees))
// note that the resolver config stores images in reverse chronological order, so iterate backwards through layers
// as you iterate chronologically through history (ignoring history items that have no layer contents)
// Note: history is not required metadata in a docker image!
tarPathIdx := 0
histIdx := 0
for layerIdx := len(img.trees) - 1; layerIdx >= 0; layerIdx-- {
tree := img.trees[(len(img.trees)-1)-layerIdx]
for layerIdx := len(trees) - 1; layerIdx >= 0; layerIdx-- {
tree := trees[(len(trees)-1)-layerIdx]
// ignore empty layers, we are only observing layers with content
historyObj := imageHistoryEntry{
historyObj := historyEntry{
CreatedBy: "(missing)",
}
for nextHistIdx := histIdx; nextHistIdx < len(img.config.History); nextHistIdx++ {
@ -178,43 +173,20 @@ func (img *Image) Analyze() (*image.AnalysisResult, error) {
histIdx++
}
img.layers[layerIdx] = &dockerLayer{
historyObj.Size = tree.FileSize
layers[layerIdx] = &layer{
history: historyObj,
index: tarPathIdx,
tree: img.trees[layerIdx],
tarPath: img.manifest.LayerTarPaths[tarPathIdx],
tree: trees[layerIdx],
}
img.layers[layerIdx].history.Size = tree.FileSize
tarPathIdx++
}
efficiency, inefficiencies := filetree.Efficiency(img.trees)
var sizeBytes, userSizeBytes uint64
layers := make([]image.Layer, len(img.layers))
for i, v := range img.layers {
layers[i] = v
sizeBytes += v.Size()
if i != 0 {
userSizeBytes += v.Size()
}
}
var wastedBytes uint64
for idx := 0; idx < len(inefficiencies); idx++ {
fileData := inefficiencies[len(inefficiencies)-1-idx]
wastedBytes += uint64(fileData.CumulativeSize)
}
return &image.AnalysisResult{
Layers: layers,
RefTrees: img.trees,
Efficiency: efficiency,
UserSizeByes: userSizeBytes,
SizeBytes: sizeBytes,
WastedBytes: wastedBytes,
WastedUserPercent: float64(wastedBytes) / float64(userSizeBytes),
Inefficiencies: inefficiencies,
return &image.Image{
Trees: trees,
Layers: layers,
}, nil
}

55
dive/image/docker/layer.go
View File

@ -10,56 +10,41 @@ import (
)
// Layer represents a Docker image layer and metadata
type dockerLayer struct {
tarPath string
history imageHistoryEntry
type layer struct {
history historyEntry
index int
tree *filetree.FileTree
}
type imageHistoryEntry struct {
ID string
Size uint64
Created string `json:"created"`
Author string `json:"author"`
CreatedBy string `json:"created_by"`
EmptyLayer bool `json:"empty_layer"`
}
// ShortId returns the truncated id of the current layer.
func (layer *dockerLayer) TarId() string {
return strings.TrimSuffix(layer.tarPath, "/layer.tar")
}
// ShortId returns the truncated id of the current layer.
func (layer *dockerLayer) Id() string {
return layer.history.ID
func (l *layer) Id() string {
return l.history.ID
}
// index returns the relative position of the layer within the image.
func (layer *dockerLayer) Index() int {
return layer.index
func (l *layer) Index() int {
return l.index
}
// Size returns the number of bytes that this image is.
func (layer *dockerLayer) Size() uint64 {
return layer.history.Size
func (l *layer) Size() uint64 {
return l.history.Size
}
// Tree returns the file tree representing the current layer.
func (layer *dockerLayer) Tree() *filetree.FileTree {
return layer.tree
func (l *layer) Tree() *filetree.FileTree {
return l.tree
}
// ShortId returns the truncated id of the current layer.
func (layer *dockerLayer) Command() string {
return strings.TrimPrefix(layer.history.CreatedBy, "/bin/sh -c ")
func (l *layer) Command() string {
return strings.TrimPrefix(l.history.CreatedBy, "/bin/sh -c ")
}
// ShortId returns the truncated id of the current layer.
func (layer *dockerLayer) ShortId() string {
func (l *layer) ShortId() string {
rangeBound := 15
id := layer.Id()
id := l.Id()
if length := len(id); length < 15 {
rangeBound = length
}
@ -69,14 +54,14 @@ func (layer *dockerLayer) ShortId() string {
}
// String represents a layer in a columnar format.
func (layer *dockerLayer) String() string {
func (l *layer) String() string {
if layer.index == 0 {
if l.index == 0 {
return fmt.Sprintf(image.LayerFormat,
humanize.Bytes(layer.Size()),
"FROM "+layer.ShortId())
humanize.Bytes(l.Size()),
"FROM "+l.ShortId())
}
return fmt.Sprintf(image.LayerFormat,
humanize.Bytes(layer.Size()),
layer.Command())
humanize.Bytes(l.Size()),
l.Command())
}

31
dive/image/docker/resolver.go
View File

@ -13,8 +13,6 @@ import (
"golang.org/x/net/context"
)
var dockerVersion string
type resolver struct {
id string
client *client.Client
@ -24,10 +22,9 @@ func NewResolver() *resolver {
return &resolver{}
}
func (r *resolver) Resolve(id string) (image.Analyzer, error) {
r.id = id
func (r *resolver) Fetch(id string) (*image.Image, error) {
reader, err := r.fetchArchive()
reader, err := r.fetchArchive(id)
if err != nil {
return nil, err
}
@ -37,16 +34,18 @@ func (r *resolver) Resolve(id string) (image.Analyzer, error) {
if err != nil {
return nil, err
}
return img, nil
return img.ToImage()
}
func (r *resolver) Build(args []string) (string, error) {
var err error
r.id, err = buildImageFromCli(args)
return r.id, err
func (r *resolver) Build(args []string) (*image.Image, error) {
id, err := buildImageFromCli(args)
if err != nil {
return nil, err
}
return r.Fetch(id)
}
func (r *resolver) fetchArchive() (io.ReadCloser, error) {
func (r *resolver) fetchArchive(id string) (io.ReadCloser, error) {
var err error
// pull the resolver if it does not exist
@ -81,22 +80,22 @@ func (r *resolver) fetchArchive() (io.ReadCloser, error) {
clientOpts = append(clientOpts, client.FromEnv)
}
clientOpts = append(clientOpts, client.WithVersion(dockerVersion))
clientOpts = append(clientOpts, client.WithAPIVersionNegotiation())
r.client, err = client.NewClientWithOpts(clientOpts...)
if err != nil {
return nil, err
}
_, _, err = r.client.ImageInspectWithRaw(ctx, r.id)
_, _, err = r.client.ImageInspectWithRaw(ctx, id)
if err != nil {
// don't use the API, the CLI has more informative output
fmt.Println("Handler not available locally. Trying to pull '" + r.id + "'...")
err = runDockerCmd("pull", r.id)
fmt.Println("Handler not available locally. Trying to pull '" + id + "'...")
err = runDockerCmd("pull", id)
if err != nil {
return nil, err
}
}
readCloser, err := r.client.ImageSave(ctx, []string{r.id})
readCloser, err := r.client.ImageSave(ctx, []string{id})
if err != nil {
return nil, err
}

4
dive/image/docker/testing.go
View File

@ -12,8 +12,8 @@ func TestLoadDockerImageTar(tarPath string) (*image.AnalysisResult, error) {
}
defer f.Close()
handler := NewResolver()
img, err := handler.Resolve("dive-test:latest")
resolver := NewResolver()
img, err := resolver.Fetch("dive-test:latest")
if err != nil {
return nil, err
}

40
dive/image/image.go Normal file
View File

@ -0,0 +1,40 @@
package image
import (
"github.com/wagoodman/dive/dive/filetree"
)
type Image struct {
Trees []*filetree.FileTree
Layers []Layer
}
func (img *Image) Analyze() (*AnalysisResult, error) {
efficiency, inefficiencies := filetree.Efficiency(img.Trees)
var sizeBytes, userSizeBytes uint64
for i, v := range img.Layers {
sizeBytes += v.Size()
if i != 0 {
userSizeBytes += v.Size()
}
}
var wastedBytes uint64
for idx := 0; idx < len(inefficiencies); idx++ {
fileData := inefficiencies[len(inefficiencies)-1-idx]
wastedBytes += uint64(fileData.CumulativeSize)
}
return &AnalysisResult{
Layers: img.Layers,
RefTrees: img.Trees,
Efficiency: efficiency,
UserSizeByes: userSizeBytes,
SizeBytes: sizeBytes,
WastedBytes: wastedBytes,
WastedUserPercent: float64(wastedBytes) / float64(userSizeBytes),
Inefficiencies: inefficiencies,
}, nil
}

93
dive/image/podman/resolver.go
View File

@ -11,43 +11,96 @@ import (
"os"
)
type resolver struct {
id string
// note: podman supports saving docker formatted archives, we're leveraging this here
// todo: add oci parser and image/layer objects
image docker.Image
}
type resolver struct {}
func NewResolver() *resolver {
return &resolver{}
}
func (handler *resolver) Resolve(id string) (image.Analyzer, error) {
handler.id = id
func (r *resolver) Build(args []string) (*image.Image, error) {
id, err := buildImageFromCli(args)
if err != nil {
return nil, err
}
return r.Fetch(id)
}
path, err := handler.fetchArchive()
func (r *resolver) Fetch(id string) (*image.Image, error) {
img, err := r.resolveFromDisk(id)
if err == nil {
return img, err
}
img, err = r.resolveFromArchive(id)
if err == nil {
return img, err
}
return nil, fmt.Errorf("unable to resolve image '%s'", id)
}
func (r *resolver) resolveFromDisk(id string) (*image.Image, error) {
// var err error
return nil, fmt.Errorf("not implemented")
//
// runtime, err := libpod.NewRuntime(context.TODO())
// if err != nil {
// return nil, err
// }
//
// images, err := runtime.ImageRuntime().GetImages()
// if err != nil {
// return nil, err
// }
//
// // cfg, _ := runtime.GetConfig()
// // cfg.StorageConfig.GraphRoot
//
// for _, item:= range images {
// for _, name := range item.Names() {
// if name == id {
// fmt.Println("Found it!")
//
// curImg := item
// for {
// h, _ := curImg.History(context.TODO())
// fmt.Printf("%+v %+v %+v\n", curImg.ID(), h[0].Size, h[0].CreatedBy)
// x, _ := curImg.DriverData()
// fmt.Printf(" %+v\n", x.Data["UpperDir"])
//
//
// curImg, err = curImg.GetParent(context.TODO())
// if err != nil || curImg == nil {
// break
// }
// }
//
// }
// }
// }
//
// // os.Exit(0)
// return nil, nil
}
func (r *resolver) resolveFromArchive(id string) (*image.Image, error) {
path, err := r.fetchArchive(id)
if err != nil {
return nil, err
}
defer os.Remove(path)
file, err := os.Open(path)
defer file.Close()
img, err := docker.NewImageFromArchive(ioutil.NopCloser(bufio.NewReader(file)))
if err != nil {
return nil, err
}
return img, nil
return img.ToImage()
}
func (handler *resolver) Build(args []string) (string, error) {
var err error
handler.id, err = buildImageFromCli(args)
return handler.id, err
}
func (handler *resolver) fetchArchive() (string, error) {
func (r *resolver) fetchArchive(id string) (string, error) {
var err error
var ctx = context.Background()
@ -63,7 +116,7 @@ func (handler *resolver) fetchArchive() (string, error) {
for _, item:= range images {
for _, name := range item.Names() {
if name == handler.id {
if name == id {
file, err := ioutil.TempFile(os.TempDir(), "dive-resolver-tar")
if err != nil {
return "", err
@ -74,8 +127,6 @@ func (handler *resolver) fetchArchive() (string, error) {
return "", err
}
fmt.Println(file.Name())
return file.Name(), nil
}
}

4
dive/image/resolver.go
View File

@ -1,6 +1,6 @@
package image
type Resolver interface {
Resolve(id string) (Analyzer, error)
Build(options []string) (string, error)
Fetch(id string) (*Image, error)
Build(options []string) (*Image, error)
}

22
runtime/run.go
View File

@ -40,29 +40,31 @@ func Run(options Options) {
// if build is given, get the handler based off of either the explicit runtime
imageHandler, err := dive.GetImageHandler(options.Engine)
imageResolver, err := dive.GetImageHandler(options.Engine)
if err != nil {
fmt.Printf("cannot determine image provider: %v\n", err)
utils.Exit(1)
}
var img *image.Image
if doBuild {
fmt.Println(utils.TitleFormat("Building image..."))
options.ImageId, err = imageHandler.Build(options.BuildArgs)
img, err = imageResolver.Build(options.BuildArgs)
if err != nil {
fmt.Printf("cannot build image: %v\n", err)
utils.Exit(1)
}
}
imgAnalyzer, err := imageHandler.Resolve(options.ImageId)
if err != nil {
fmt.Printf("cannot fetch image: %v\n", err)
utils.Exit(1)
} else {
img, err = imageResolver.Fetch(options.ImageId)
if err != nil {
fmt.Printf("cannot fetch image: %v\n", err)
utils.Exit(1)
}
}
// todo, cleanup on error
// todo: image get shold return error for cleanup?
// todo: image get should return error for cleanup?
if doExport {
fmt.Println(utils.TitleFormat(fmt.Sprintf("Analyzing image... (export to '%s')", options.ExportFile)))
@ -70,7 +72,7 @@ func Run(options Options) {
fmt.Println(utils.TitleFormat("Analyzing image..."))
}
result, err := imgAnalyzer.Analyze()
result, err := img.Analyze()
if err != nil {
fmt.Printf("cannot analyze image: %v\n", err)
utils.Exit(1)

2
runtime/ui/details_controller.go
View File

@ -126,8 +126,6 @@ func (controller *DetailsController) Render() error {
// update contents
controller.view.Clear()
_, _ = fmt.Fprintln(controller.view, Formatting.Header("Digest: ")+currentLayer.Id())
// TODO: add back in with controller model
// fmt.Fprintln(view.view, Formatting.Header("Tar ID: ")+currentLayer.TarId())
_, _ = fmt.Fprintln(controller.view, Formatting.Header("Command:"))
_, _ = fmt.Fprintln(controller.view, currentLayer.Command())