Reconcile history from image manifest and config (#17)

.gitignore

@@ -17,3 +17,4 @@
 /_vendor*
 /vendor
 /.image
+*.log

cmd/root.go

@@ -3,10 +3,10 @@ package cmd
 import (
 	"fmt"
 	"os"
-
-	homedir "github.com/mitchellh/go-homedir"
+	"github.com/mitchellh/go-homedir"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	log "github.com/sirupsen/logrus"
 )
 
 var cfgFile string
@@ -31,6 +31,7 @@ func Execute() {
 
 func init() {
 	cobra.OnInitialize(initConfig)
+	cobra.OnInitialize(initLogging)
 
 	// Here you will define your flags and configuration settings.
 	// Cobra supports persistent flags, which, if defined here,
@@ -67,3 +68,21 @@ func initConfig() {
 		fmt.Println("Using config file:", viper.ConfigFileUsed())
 	}
 }
+
+func initLogging() {
+	// TODO: clean this up and make more configurable
+	var filename string = "dive.log"
+	// create the log file if doesn't exist. And append to it if it already exists.
+	f, err := os.OpenFile(filename, os.O_WRONLY | os.O_APPEND | os.O_CREATE, 0644)
+	Formatter := new(log.TextFormatter)
+	Formatter.DisableTimestamp = true
+	log.SetFormatter(Formatter)
+	log.SetLevel(log.DebugLevel)
+	if err != nil {
+		// cannot open log file. Logging to stderr
+		fmt.Println(err)
+	}else{
+		log.SetOutput(f)
+	}
+	log.Info("Starting Dive...")
+}

filetree/tree.go

@@ -20,6 +20,7 @@ const (
 type FileTree struct {
 	Root *FileNode
 	Size int
+	FileSize uint64
 	Name string
 	Id   uuid.UUID
 }
@@ -41,6 +42,7 @@ func (tree *FileTree) String(showAttributes bool) string {
 func (tree *FileTree) Copy() *FileTree {
 	newTree := NewFileTree()
 	newTree.Size = tree.Size
+	newTree.FileSize = tree.FileSize
 	newTree.Root = tree.Root.Copy(newTree.Root)
 
 	// update the tree pointers

image/image.go

@@ -33,7 +33,26 @@ type ImageManifest struct {
 	LayerTarPaths []string `json:"Layers"`
 }
 
-func NewManifest(reader *tar.Reader, header *tar.Header) ImageManifest {
+type ImageConfig struct {
+	History []ImageHistoryEntry `json:"history"`
+	RootFs RootFs `json:"rootfs"`
+}
+
+type RootFs struct {
+	Type string `json:"type"`
+	DiffIds []string `json:"diff_ids"`
+}
+
+type ImageHistoryEntry struct {
+	ID string
+	Size uint64
+	Created string `json:"created"`
+	Author string `json:"author"`
+	CreatedBy string `json:"created_by"`
+	EmptyLayer bool `json:"empty_layer"`
+}
+
+func NewImageManifest(reader *tar.Reader, header *tar.Header) ImageManifest {
 	size := header.Size
 	manifestBytes := make([]byte, size)
 	_, err := reader.Read(manifestBytes)
@@ -48,21 +67,36 @@ func NewManifest(reader *tar.Reader, header *tar.Header) ImageManifest {
 	return manifest[0]
 }
 
-func InitializeData(imageID string) ([]*Layer, []*filetree.FileTree) {
-	var manifest ImageManifest
-	var layerMap = make(map[string]*filetree.FileTree)
-	var trees []*filetree.FileTree = make([]*filetree.FileTree, 0)
+func NewImageConfig(reader *tar.Reader, header *tar.Header) ImageConfig {
+	size := header.Size
+	configBytes := make([]byte, size)
+	_, err := reader.Read(configBytes)
+	if err != nil && err != io.EOF {
+		panic(err)
+	}
+	var imageConfig ImageConfig
+	err = json.Unmarshal(configBytes, &imageConfig)
+	if err != nil {
+		panic(err)
+	}
 
-	// save this image to disk temporarily to get the content info
-	fmt.Println("Fetching image...")
-	// imageTarPath, tmpDir := saveImage(imageID)
-	imageTarPath := "/tmp/dive031537738/image.tar"
-	// tmpDir := "/tmp/dive031537738"
-	// fmt.Println(tmpDir)
-	// defer os.RemoveAll(tmpDir)
+	layerIdx := 0
+	for idx := range imageConfig.History {
+		if imageConfig.History[idx].EmptyLayer {
+			imageConfig.History[idx].ID = "<missing>"
+		} else {
+			imageConfig.History[idx].ID = imageConfig.RootFs.DiffIds[layerIdx]
+			layerIdx++
+		}
+	}
 
+	return imageConfig
+}
+
+func GetImageConfig(imageTarPath string, manifest ImageManifest) ImageConfig{
+	var config ImageConfig
 	// read through the image contents and build a tree
-	fmt.Println("Reading image...")
+	fmt.Println("Fetching image config...")
 	tarFile, err := os.Open(imageTarPath)
 	if err != nil {
 		fmt.Println(err)
@@ -83,20 +117,69 @@ func InitializeData(imageID string) ([]*Layer, []*filetree.FileTree) {
 			os.Exit(1)
 		}
 
+		name := header.Name
+		if name == manifest.ConfigPath {
+			config = NewImageConfig(tarReader, header)
+		}
+	}
+
+	// obtain the image history
+	return config
+}
+
+func InitializeData(imageID string) ([]*Layer, []*filetree.FileTree) {
+	var manifest ImageManifest
+	var layerMap = make(map[string]*filetree.FileTree)
+	var trees []*filetree.FileTree = make([]*filetree.FileTree, 0)
+
+	// save this image to disk temporarily to get the content info
+	// fmt.Println("Fetching image...")
+	imageTarPath, tmpDir := saveImage(imageID)
+	// imageTarPath := "/tmp/dive932744808/image.tar"
+	// tmpDir := "/tmp/dive031537738"
+	// fmt.Println(tmpDir)
+	defer os.RemoveAll(tmpDir)
+
+	// read through the image contents and build a tree
+	fmt.Println("Reading image...")
+	tarFile, err := os.Open(imageTarPath)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	defer tarFile.Close()
+
+	tarReader := tar.NewReader(tarFile)
+	for {
+		header, err := tarReader.Next()
+
+		// log.Debug(header.Name)
+
+		if err == io.EOF {
+			break
+		}
+
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
 		name := header.Name
 		if name == "manifest.json" {
-			manifest = NewManifest(tarReader, header)
+			manifest = NewImageManifest(tarReader, header)
 		}
 
 		switch header.Typeflag {
 		case tar.TypeDir:
 			continue
 		case tar.TypeReg:
+			// todo: process this loop in parallel, visualize with jotframe
 			if strings.HasSuffix(name, "layer.tar") {
 				tree := filetree.NewFileTree()
 				tree.Name = name
 				fileInfos := getFileList(tarReader, header)
 				for _, element := range fileInfos {
+					tree.FileSize += uint64(element.TarHeader.FileInfo().Size())
 					tree.AddPath(element.Path, element)
 				}
 				layerMap[tree.Name] = tree
@@ -106,33 +189,41 @@ func InitializeData(imageID string) ([]*Layer, []*filetree.FileTree) {
 		}
 	}
 
+	// obtain the image history
+	config := GetImageConfig(imageTarPath, manifest)
+
 	// build the content tree
 	fmt.Println("Building tree...")
 	for _, treeName := range manifest.LayerTarPaths {
 		trees = append(trees, layerMap[treeName])
 	}
 
-	// get the history of this image
-	ctx := context.Background()
-	dockerClient, err := client.NewClientWithOpts()
-	if err != nil {
-		panic(err)
-	}
-
-	history, err := dockerClient.ImageHistory(ctx, imageID)
 
 	// build the layers array
 	layers := make([]*Layer, len(trees))
-	for idx := 0; idx < len(trees); idx++ {
-		layers[idx] = &Layer{
-			History: history[idx],
-			Index: idx,
-			Tree: trees[idx],
+
+	// note that the image config stores images in reverse chronological order, so iterate backwards through layers
+	// as you iterate chronologically through history (ignoring history items that have no layer contents)
+	layerIdx := len(trees)-1
+	for idx := 0; idx < len(config.History); idx++ {
+		// ignore empty layers, we are only observing layers with content
+		if config.History[idx].EmptyLayer {
+			continue
+		}
+
+		config.History[idx].Size = uint64(trees[(len(trees)-1)-layerIdx].FileSize)
+
+		layers[layerIdx] = &Layer{
+			History: config.History[idx],
+			Index: layerIdx,
+			Tree: trees[layerIdx],
 			RefTrees: trees,
 		}
+
 		if len(manifest.LayerTarPaths) > idx {
-			layers[idx].TarPath = manifest.LayerTarPaths[idx]
+			layers[layerIdx].TarPath = manifest.LayerTarPaths[layerIdx]
 		}
+		layerIdx--
 	}
 
 	return layers, trees

image/layer.go

@@ -1,7 +1,6 @@
 package image
 
 import (
-	"github.com/docker/docker/api/types/image"
 	"github.com/wagoodman/dive/filetree"
 	"strings"
 	"fmt"
@@ -11,7 +10,7 @@ import (
 
 type Layer struct {
 	TarPath  string
-	History  image.HistoryResponseItem
+	History ImageHistoryEntry
 	Index    int
 	Tree     *filetree.FileTree
 	RefTrees []*filetree.FileTree

ui/layerview.go

@@ -6,6 +6,7 @@ import (
 	"github.com/jroimartin/gocui"
 	"github.com/wagoodman/dive/image"
 	"github.com/lunixbochs/vtclean"
+	"github.com/dustin/go-humanize"
 )
 
 type LayerView struct {
@@ -151,8 +152,7 @@ func (view *LayerView) Render() error {
 					layerId = fmt.Sprintf("%-25s", layer.History.ID)
 				}
 
-				// TODO: add size
-				layerStr = fmt.Sprintf(image.LayerFormat, layerId, "", "", "FROM "+layer.Id())
+				layerStr = fmt.Sprintf(image.LayerFormat, layerId, "", humanize.Bytes(uint64(layer.History.Size)), "FROM "+layer.Id())
 			}
 
 			compareBar := view.renderCompareBar(idx)

ui/ui.go

@@ -14,7 +14,7 @@ import (
 	"runtime/pprof"
 )
 
-const debug = true
+const debug = false
 const profile = false
 
 func debugPrint(s string) {